Unverified Commit cd7071e7 authored by Steven Liu's avatar Steven Liu Committed by GitHub
Browse files

[docs] Add safetensors flag (#4245)

* add safetensors flag

* apply review
parent e31f38b5
...@@ -51,6 +51,7 @@ from diffusers import DiffusionPipeline ...@@ -51,6 +51,7 @@ from diffusers import DiffusionPipeline
pipe = DiffusionPipeline.from_pretrained( pipe = DiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
) )
pipe = pipe.to("cuda") pipe = pipe.to("cuda")
...@@ -80,6 +81,7 @@ from diffusers import StableDiffusionPipeline ...@@ -80,6 +81,7 @@ from diffusers import StableDiffusionPipeline
pipe = StableDiffusionPipeline.from_pretrained( pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
) )
pipe = pipe.to("cuda") pipe = pipe.to("cuda")
...@@ -106,6 +108,7 @@ from diffusers import StableDiffusionPipeline, UniPCMultistepScheduler ...@@ -106,6 +108,7 @@ from diffusers import StableDiffusionPipeline, UniPCMultistepScheduler
pipe = StableDiffusionPipeline.from_pretrained( pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
) )
pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
pipe = pipe.to("cuda") pipe = pipe.to("cuda")
...@@ -133,6 +136,7 @@ from diffusers import StableDiffusionPipeline ...@@ -133,6 +136,7 @@ from diffusers import StableDiffusionPipeline
pipe = StableDiffusionPipeline.from_pretrained( pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
) )
prompt = "a photo of an astronaut riding a horse on mars" prompt = "a photo of an astronaut riding a horse on mars"
...@@ -157,6 +161,7 @@ from diffusers import StableDiffusionPipeline ...@@ -157,6 +161,7 @@ from diffusers import StableDiffusionPipeline
pipe = StableDiffusionPipeline.from_pretrained( pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
) )
prompt = "a photo of an astronaut riding a horse on mars" prompt = "a photo of an astronaut riding a horse on mars"
...@@ -189,6 +194,7 @@ from diffusers import StableDiffusionPipeline ...@@ -189,6 +194,7 @@ from diffusers import StableDiffusionPipeline
pipe = StableDiffusionPipeline.from_pretrained( pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
) )
prompt = "a photo of an astronaut riding a horse on mars" prompt = "a photo of an astronaut riding a horse on mars"
...@@ -205,6 +211,7 @@ from diffusers import StableDiffusionPipeline ...@@ -205,6 +211,7 @@ from diffusers import StableDiffusionPipeline
pipe = StableDiffusionPipeline.from_pretrained( pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
) )
prompt = "a photo of an astronaut riding a horse on mars" prompt = "a photo of an astronaut riding a horse on mars"
...@@ -267,6 +274,7 @@ def generate_inputs(): ...@@ -267,6 +274,7 @@ def generate_inputs():
pipe = StableDiffusionPipeline.from_pretrained( pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
).to("cuda") ).to("cuda")
unet = pipe.unet unet = pipe.unet
unet.eval() unet.eval()
...@@ -330,6 +338,7 @@ class UNet2DConditionOutput: ...@@ -330,6 +338,7 @@ class UNet2DConditionOutput:
pipe = StableDiffusionPipeline.from_pretrained( pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
).to("cuda") ).to("cuda")
# use jitted unet # use jitted unet
...@@ -389,6 +398,7 @@ import torch ...@@ -389,6 +398,7 @@ import torch
pipe = DiffusionPipeline.from_pretrained( pipe = DiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
).to("cuda") ).to("cuda")
pipe.enable_xformers_memory_efficient_attention() pipe.enable_xformers_memory_efficient_attention()
......
...@@ -39,7 +39,7 @@ pip install --upgrade torch diffusers ...@@ -39,7 +39,7 @@ pip install --upgrade torch diffusers
import torch import torch
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16, use_safetensors=True)
pipe = pipe.to("cuda") pipe = pipe.to("cuda")
prompt = "a photo of an astronaut riding a horse on mars" prompt = "a photo of an astronaut riding a horse on mars"
...@@ -53,7 +53,7 @@ pip install --upgrade torch diffusers ...@@ -53,7 +53,7 @@ pip install --upgrade torch diffusers
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
+ from diffusers.models.attention_processor import AttnProcessor2_0 + from diffusers.models.attention_processor import AttnProcessor2_0
pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16).to("cuda") pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16, use_safetensors=True).to("cuda")
+ pipe.unet.set_attn_processor(AttnProcessor2_0()) + pipe.unet.set_attn_processor(AttnProcessor2_0())
prompt = "a photo of an astronaut riding a horse on mars" prompt = "a photo of an astronaut riding a horse on mars"
...@@ -69,7 +69,7 @@ pip install --upgrade torch diffusers ...@@ -69,7 +69,7 @@ pip install --upgrade torch diffusers
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
from diffusers.models.attention_processor import AttnProcessor from diffusers.models.attention_processor import AttnProcessor
pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16).to("cuda") pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16, use_safetensors=True).to("cuda")
pipe.unet.set_default_attn_processor() pipe.unet.set_default_attn_processor()
prompt = "a photo of an astronaut riding a horse on mars" prompt = "a photo of an astronaut riding a horse on mars"
...@@ -107,7 +107,7 @@ path = "runwayml/stable-diffusion-v1-5" ...@@ -107,7 +107,7 @@ path = "runwayml/stable-diffusion-v1-5"
run_compile = True # Set True / False run_compile = True # Set True / False
pipe = DiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16) pipe = DiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16, use_safetensors=True)
pipe = pipe.to("cuda") pipe = pipe.to("cuda")
pipe.unet.to(memory_format=torch.channels_last) pipe.unet.to(memory_format=torch.channels_last)
...@@ -140,7 +140,7 @@ path = "runwayml/stable-diffusion-v1-5" ...@@ -140,7 +140,7 @@ path = "runwayml/stable-diffusion-v1-5"
run_compile = True # Set True / False run_compile = True # Set True / False
pipe = StableDiffusionImg2ImgPipeline.from_pretrained(path, torch_dtype=torch.float16) pipe = StableDiffusionImg2ImgPipeline.from_pretrained(path, torch_dtype=torch.float16, use_safetensors=True)
pipe = pipe.to("cuda") pipe = pipe.to("cuda")
pipe.unet.to(memory_format=torch.channels_last) pipe.unet.to(memory_format=torch.channels_last)
...@@ -180,7 +180,7 @@ path = "runwayml/stable-diffusion-inpainting" ...@@ -180,7 +180,7 @@ path = "runwayml/stable-diffusion-inpainting"
run_compile = True # Set True / False run_compile = True # Set True / False
pipe = StableDiffusionInpaintPipeline.from_pretrained(path, torch_dtype=torch.float16) pipe = StableDiffusionInpaintPipeline.from_pretrained(path, torch_dtype=torch.float16, use_safetensors=True)
pipe = pipe.to("cuda") pipe = pipe.to("cuda")
pipe.unet.to(memory_format=torch.channels_last) pipe.unet.to(memory_format=torch.channels_last)
...@@ -212,9 +212,9 @@ init_image = init_image.resize((512, 512)) ...@@ -212,9 +212,9 @@ init_image = init_image.resize((512, 512))
path = "runwayml/stable-diffusion-v1-5" path = "runwayml/stable-diffusion-v1-5"
run_compile = True # Set True / False run_compile = True # Set True / False
controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16) controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16, use_safetensors=True)
pipe = StableDiffusionControlNetPipeline.from_pretrained( pipe = StableDiffusionControlNetPipeline.from_pretrained(
path, controlnet=controlnet, torch_dtype=torch.float16 path, controlnet=controlnet, torch_dtype=torch.float16, use_safetensors=True
) )
pipe = pipe.to("cuda") pipe = pipe.to("cuda")
...@@ -240,11 +240,11 @@ import torch ...@@ -240,11 +240,11 @@ import torch
run_compile = True # Set True / False run_compile = True # Set True / False
pipe = DiffusionPipeline.from_pretrained("DeepFloyd/IF-I-M-v1.0", variant="fp16", text_encoder=None, torch_dtype=torch.float16) pipe = DiffusionPipeline.from_pretrained("DeepFloyd/IF-I-M-v1.0", variant="fp16", text_encoder=None, torch_dtype=torch.float16, use_safetensors=True)
pipe.to("cuda") pipe.to("cuda")
pipe_2 = DiffusionPipeline.from_pretrained("DeepFloyd/IF-II-M-v1.0", variant="fp16", text_encoder=None, torch_dtype=torch.float16) pipe_2 = DiffusionPipeline.from_pretrained("DeepFloyd/IF-II-M-v1.0", variant="fp16", text_encoder=None, torch_dtype=torch.float16, use_safetensors=True)
pipe_2.to("cuda") pipe_2.to("cuda")
pipe_3 = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-x4-upscaler", torch_dtype=torch.float16) pipe_3 = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-x4-upscaler", torch_dtype=torch.float16, use_safetensors=True)
pipe_3.to("cuda") pipe_3.to("cuda")
......
...@@ -67,7 +67,7 @@ Load the model with the [`~DiffusionPipeline.from_pretrained`] method: ...@@ -67,7 +67,7 @@ Load the model with the [`~DiffusionPipeline.from_pretrained`] method:
```python ```python
>>> from diffusers import DiffusionPipeline >>> from diffusers import DiffusionPipeline
>>> pipeline = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") >>> pipeline = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", use_safetensors=True)
``` ```
The [`DiffusionPipeline`] downloads and caches all modeling, tokenization, and scheduling components. You'll see that the Stable Diffusion pipeline is composed of the [`UNet2DConditionModel`] and [`PNDMScheduler`] among other things: The [`DiffusionPipeline`] downloads and caches all modeling, tokenization, and scheduling components. You'll see that the Stable Diffusion pipeline is composed of the [`UNet2DConditionModel`] and [`PNDMScheduler`] among other things:
...@@ -130,7 +130,7 @@ You can also use the pipeline locally. The only difference is you need to downlo ...@@ -130,7 +130,7 @@ You can also use the pipeline locally. The only difference is you need to downlo
Then load the saved weights into the pipeline: Then load the saved weights into the pipeline:
```python ```python
>>> pipeline = DiffusionPipeline.from_pretrained("./stable-diffusion-v1-5") >>> pipeline = DiffusionPipeline.from_pretrained("./stable-diffusion-v1-5", use_safetensors=True)
``` ```
Now you can run the pipeline as you would in the section above. Now you can run the pipeline as you would in the section above.
...@@ -142,7 +142,7 @@ Different schedulers come with different denoising speeds and quality trade-offs ...@@ -142,7 +142,7 @@ Different schedulers come with different denoising speeds and quality trade-offs
```py ```py
>>> from diffusers import EulerDiscreteScheduler >>> from diffusers import EulerDiscreteScheduler
>>> pipeline = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") >>> pipeline = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", use_safetensors=True)
>>> pipeline.scheduler = EulerDiscreteScheduler.from_config(pipeline.scheduler.config) >>> pipeline.scheduler = EulerDiscreteScheduler.from_config(pipeline.scheduler.config)
``` ```
...@@ -160,7 +160,7 @@ Models are initiated with the [`~ModelMixin.from_pretrained`] method which also ...@@ -160,7 +160,7 @@ Models are initiated with the [`~ModelMixin.from_pretrained`] method which also
>>> from diffusers import UNet2DModel >>> from diffusers import UNet2DModel
>>> repo_id = "google/ddpm-cat-256" >>> repo_id = "google/ddpm-cat-256"
>>> model = UNet2DModel.from_pretrained(repo_id) >>> model = UNet2DModel.from_pretrained(repo_id, use_safetensors=True)
``` ```
To access the model parameters, call `model.config`: To access the model parameters, call `model.config`:
......
...@@ -26,7 +26,7 @@ Begin by loading the [`runwayml/stable-diffusion-v1-5`](https://huggingface.co/r ...@@ -26,7 +26,7 @@ Begin by loading the [`runwayml/stable-diffusion-v1-5`](https://huggingface.co/r
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
model_id = "runwayml/stable-diffusion-v1-5" model_id = "runwayml/stable-diffusion-v1-5"
pipeline = DiffusionPipeline.from_pretrained(model_id) pipeline = DiffusionPipeline.from_pretrained(model_id, use_safetensors=True)
``` ```
The example prompt you'll use is a portrait of an old warrior chief, but feel free to use your own prompt: The example prompt you'll use is a portrait of an old warrior chief, but feel free to use your own prompt:
...@@ -75,7 +75,7 @@ Let's start by loading the model in `float16` and generate an image: ...@@ -75,7 +75,7 @@ Let's start by loading the model in `float16` and generate an image:
```python ```python
import torch import torch
pipeline = DiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16) pipeline = DiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16, use_safetensors=True)
pipeline = pipeline.to("cuda") pipeline = pipeline.to("cuda")
generator = torch.Generator("cuda").manual_seed(0) generator = torch.Generator("cuda").manual_seed(0)
image = pipeline(prompt, generator=generator).images[0] image = pipeline(prompt, generator=generator).images[0]
......
...@@ -11,7 +11,7 @@ A [`UNet2DConditionModel`] by default accepts 4 channels in the [input sample](h ...@@ -11,7 +11,7 @@ A [`UNet2DConditionModel`] by default accepts 4 channels in the [input sample](h
```py ```py
from diffusers import StableDiffusionPipeline from diffusers import StableDiffusionPipeline
pipeline = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") pipeline = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", use_safetensors=True)
pipeline.unet.config["in_channels"] pipeline.unet.config["in_channels"]
4 4
``` ```
...@@ -21,7 +21,7 @@ Inpainting requires 9 channels in the input sample. You can check this value in ...@@ -21,7 +21,7 @@ Inpainting requires 9 channels in the input sample. You can check this value in
```py ```py
from diffusers import StableDiffusionPipeline from diffusers import StableDiffusionPipeline
pipeline = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") pipeline = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-inpainting", use_safetensors=True)
pipeline.unet.config["in_channels"] pipeline.unet.config["in_channels"]
9 9
``` ```
...@@ -35,7 +35,12 @@ from diffusers import UNet2DConditionModel ...@@ -35,7 +35,12 @@ from diffusers import UNet2DConditionModel
model_id = "runwayml/stable-diffusion-v1-5" model_id = "runwayml/stable-diffusion-v1-5"
unet = UNet2DConditionModel.from_pretrained( unet = UNet2DConditionModel.from_pretrained(
model_id, subfolder="unet", in_channels=9, low_cpu_mem_usage=False, ignore_mismatched_sizes=True model_id,
subfolder="unet",
in_channels=9,
low_cpu_mem_usage=False,
ignore_mismatched_sizes=True,
use_safetensors=True,
) )
``` ```
......
...@@ -306,9 +306,9 @@ import torch ...@@ -306,9 +306,9 @@ import torch
base_model_path = "path to model" base_model_path = "path to model"
controlnet_path = "path to controlnet" controlnet_path = "path to controlnet"
controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=torch.float16) controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=torch.float16, use_safetensors=True)
pipe = StableDiffusionControlNetPipeline.from_pretrained( pipe = StableDiffusionControlNetPipeline.from_pretrained(
base_model_path, controlnet=controlnet, torch_dtype=torch.float16 base_model_path, controlnet=controlnet, torch_dtype=torch.float16, use_safetensors=True
) )
# speed up diffusion process with faster scheduler and memory optimization # speed up diffusion process with faster scheduler and memory optimization
......
...@@ -222,7 +222,9 @@ Once you have trained a model using the above command, you can run inference usi ...@@ -222,7 +222,9 @@ Once you have trained a model using the above command, you can run inference usi
import torch import torch
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
pipe = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16).to("cuda") pipe = DiffusionPipeline.from_pretrained(
"CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16, use_safetensors=True
).to("cuda")
pipe.unet.load_attn_procs("path-to-save-model", weight_name="pytorch_custom_diffusion_weights.bin") pipe.unet.load_attn_procs("path-to-save-model", weight_name="pytorch_custom_diffusion_weights.bin")
pipe.load_textual_inversion("path-to-save-model", weight_name="<new1>.bin") pipe.load_textual_inversion("path-to-save-model", weight_name="<new1>.bin")
...@@ -246,7 +248,7 @@ model_id = "sayakpaul/custom-diffusion-cat" ...@@ -246,7 +248,7 @@ model_id = "sayakpaul/custom-diffusion-cat"
card = RepoCard.load(model_id) card = RepoCard.load(model_id)
base_model_id = card.data.to_dict()["base_model"] base_model_id = card.data.to_dict()["base_model"]
pipe = DiffusionPipeline.from_pretrained(base_model_id, torch_dtype=torch.float16).to("cuda") pipe = DiffusionPipeline.from_pretrained(base_model_id, torch_dtype=torch.float16, use_safetensors=True).to("cuda")
pipe.unet.load_attn_procs(model_id, weight_name="pytorch_custom_diffusion_weights.bin") pipe.unet.load_attn_procs(model_id, weight_name="pytorch_custom_diffusion_weights.bin")
pipe.load_textual_inversion(model_id, weight_name="<new1>.bin") pipe.load_textual_inversion(model_id, weight_name="<new1>.bin")
...@@ -270,7 +272,7 @@ model_id = "sayakpaul/custom-diffusion-cat-wooden-pot" ...@@ -270,7 +272,7 @@ model_id = "sayakpaul/custom-diffusion-cat-wooden-pot"
card = RepoCard.load(model_id) card = RepoCard.load(model_id)
base_model_id = card.data.to_dict()["base_model"] base_model_id = card.data.to_dict()["base_model"]
pipe = DiffusionPipeline.from_pretrained(base_model_id, torch_dtype=torch.float16).to("cuda") pipe = DiffusionPipeline.from_pretrained(base_model_id, torch_dtype=torch.float16, use_safetensors=True).to("cuda")
pipe.unet.load_attn_procs(model_id, weight_name="pytorch_custom_diffusion_weights.bin") pipe.unet.load_attn_procs(model_id, weight_name="pytorch_custom_diffusion_weights.bin")
pipe.load_textual_inversion(model_id, weight_name="<new1>.bin") pipe.load_textual_inversion(model_id, weight_name="<new1>.bin")
pipe.load_textual_inversion(model_id, weight_name="<new2>.bin") pipe.load_textual_inversion(model_id, weight_name="<new2>.bin")
......
...@@ -16,7 +16,9 @@ Now use the [`~accelerate.PartialState.split_between_processes`] utility as a co ...@@ -16,7 +16,9 @@ Now use the [`~accelerate.PartialState.split_between_processes`] utility as a co
from accelerate import PartialState from accelerate import PartialState
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
pipeline = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) pipeline = DiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16, use_safetensors=True
)
distributed_state = PartialState() distributed_state = PartialState()
pipeline.to(distributed_state.device) pipeline.to(distributed_state.device)
...@@ -50,7 +52,9 @@ import torch.multiprocessing as mp ...@@ -50,7 +52,9 @@ import torch.multiprocessing as mp
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
sd = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) sd = DiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16, use_safetensors=True
)
``` ```
You'll want to create a function to run inference; [`init_process_group`](https://pytorch.org/docs/stable/distributed.html?highlight=init_process_group#torch.distributed.init_process_group) handles creating a distributed environment with the type of backend to use, the `rank` of the current process, and the `world_size` or the number of processes participating. If you're running inference in parallel over 2 GPUs, then the `world_size` is 2. You'll want to create a function to run inference; [`init_process_group`](https://pytorch.org/docs/stable/distributed.html?highlight=init_process_group#torch.distributed.init_process_group) handles creating a distributed environment with the type of backend to use, the `rank` of the current process, and the `world_size` or the number of processes participating. If you're running inference in parallel over 2 GPUs, then the `world_size` is 2.
......
...@@ -303,7 +303,9 @@ unet = UNet2DConditionModel.from_pretrained("/sddata/dreambooth/daruma-v2-1/chec ...@@ -303,7 +303,9 @@ unet = UNet2DConditionModel.from_pretrained("/sddata/dreambooth/daruma-v2-1/chec
# if you have trained with `--args.train_text_encoder` make sure to also load the text encoder # if you have trained with `--args.train_text_encoder` make sure to also load the text encoder
text_encoder = CLIPTextModel.from_pretrained("/sddata/dreambooth/daruma-v2-1/checkpoint-100/text_encoder") text_encoder = CLIPTextModel.from_pretrained("/sddata/dreambooth/daruma-v2-1/checkpoint-100/text_encoder")
pipeline = DiffusionPipeline.from_pretrained(model_id, unet=unet, text_encoder=text_encoder, dtype=torch.float16) pipeline = DiffusionPipeline.from_pretrained(
model_id, unet=unet, text_encoder=text_encoder, dtype=torch.float16, use_safetensors=True
)
pipeline.to("cuda") pipeline.to("cuda")
# Perform inference, or save, or push to the hub # Perform inference, or save, or push to the hub
...@@ -318,7 +320,7 @@ from diffusers import DiffusionPipeline ...@@ -318,7 +320,7 @@ from diffusers import DiffusionPipeline
# Load the pipeline with the same arguments (model, revision) that were used for training # Load the pipeline with the same arguments (model, revision) that were used for training
model_id = "CompVis/stable-diffusion-v1-4" model_id = "CompVis/stable-diffusion-v1-4"
pipeline = DiffusionPipeline.from_pretrained(model_id) pipeline = DiffusionPipeline.from_pretrained(model_id, use_safetensors=True)
accelerator = Accelerator() accelerator = Accelerator()
...@@ -333,6 +335,7 @@ pipeline = DiffusionPipeline.from_pretrained( ...@@ -333,6 +335,7 @@ pipeline = DiffusionPipeline.from_pretrained(
model_id, model_id,
unet=accelerator.unwrap_model(unet), unet=accelerator.unwrap_model(unet),
text_encoder=accelerator.unwrap_model(text_encoder), text_encoder=accelerator.unwrap_model(text_encoder),
use_safetensors=True,
) )
# Perform inference, or save, or push to the hub # Perform inference, or save, or push to the hub
...@@ -488,7 +491,7 @@ from diffusers import DiffusionPipeline ...@@ -488,7 +491,7 @@ from diffusers import DiffusionPipeline
import torch import torch
model_id = "path_to_saved_model" model_id = "path_to_saved_model"
pipe = DiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda") pipe = DiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16, use_safetensors=True).to("cuda")
prompt = "A photo of sks dog in a bucket" prompt = "A photo of sks dog in a bucket"
image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5).images[0] image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5).images[0]
...@@ -510,7 +513,7 @@ must also update the pipeline's scheduler config. ...@@ -510,7 +513,7 @@ must also update the pipeline's scheduler config.
```py ```py
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
pipe = DiffusionPipeline.from_pretrained("DeepFloyd/IF-I-XL-v1.0") pipe = DiffusionPipeline.from_pretrained("DeepFloyd/IF-I-XL-v1.0", use_safetensors=True)
pipe.load_lora_weights("<lora weights path>") pipe.load_lora_weights("<lora weights path>")
......
...@@ -165,7 +165,9 @@ import torch ...@@ -165,7 +165,9 @@ import torch
from diffusers import StableDiffusionInstructPix2PixPipeline from diffusers import StableDiffusionInstructPix2PixPipeline
model_id = "your_model_id" # <- replace this model_id = "your_model_id" # <- replace this
pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda") pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained(
model_id, torch_dtype=torch.float16, use_safetensors=True
).to("cuda")
generator = torch.Generator("cuda").manual_seed(0) generator = torch.Generator("cuda").manual_seed(0)
url = "https://huggingface.co/datasets/sayakpaul/sample-datasets/resolve/main/test_pix2pix_4.png" url = "https://huggingface.co/datasets/sayakpaul/sample-datasets/resolve/main/test_pix2pix_4.png"
......
...@@ -98,7 +98,7 @@ Now you can use the model for inference by loading the base model in the [`Stabl ...@@ -98,7 +98,7 @@ Now you can use the model for inference by loading the base model in the [`Stabl
>>> model_base = "runwayml/stable-diffusion-v1-5" >>> model_base = "runwayml/stable-diffusion-v1-5"
>>> pipe = StableDiffusionPipeline.from_pretrained(model_base, torch_dtype=torch.float16) >>> pipe = StableDiffusionPipeline.from_pretrained(model_base, torch_dtype=torch.float16, use_safetensors=True)
>>> pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) >>> pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
``` ```
...@@ -137,7 +137,7 @@ lora_model_id = "sayakpaul/sd-model-finetuned-lora-t4" ...@@ -137,7 +137,7 @@ lora_model_id = "sayakpaul/sd-model-finetuned-lora-t4"
card = RepoCard.load(lora_model_id) card = RepoCard.load(lora_model_id)
base_model_id = card.data.to_dict()["base_model"] base_model_id = card.data.to_dict()["base_model"]
pipe = StableDiffusionPipeline.from_pretrained(base_model_id, torch_dtype=torch.float16) pipe = StableDiffusionPipeline.from_pretrained(base_model_id, torch_dtype=torch.float16, use_safetensors=True)
... ...
``` ```
...@@ -211,7 +211,7 @@ Now you can use the model for inference by loading the base model in the [`Stabl ...@@ -211,7 +211,7 @@ Now you can use the model for inference by loading the base model in the [`Stabl
>>> model_base = "runwayml/stable-diffusion-v1-5" >>> model_base = "runwayml/stable-diffusion-v1-5"
>>> pipe = StableDiffusionPipeline.from_pretrained(model_base, torch_dtype=torch.float16) >>> pipe = StableDiffusionPipeline.from_pretrained(model_base, torch_dtype=torch.float16, use_safetensors=True)
``` ```
Load the LoRA weights from your finetuned DreamBooth model *on top of the base model weights*, and then move the pipeline to a GPU for faster inference. When you merge the LoRA weights with the frozen pretrained model weights, you can optionally adjust how much of the weights to merge with the `scale` parameter: Load the LoRA weights from your finetuned DreamBooth model *on top of the base model weights*, and then move the pipeline to a GPU for faster inference. When you merge the LoRA weights with the frozen pretrained model weights, you can optionally adjust how much of the weights to merge with the `scale` parameter:
...@@ -251,7 +251,7 @@ lora_model_id = "sayakpaul/dreambooth-text-encoder-test" ...@@ -251,7 +251,7 @@ lora_model_id = "sayakpaul/dreambooth-text-encoder-test"
card = RepoCard.load(lora_model_id) card = RepoCard.load(lora_model_id)
base_model_id = card.data.to_dict()["base_model"] base_model_id = card.data.to_dict()["base_model"]
pipe = StableDiffusionPipeline.from_pretrained(base_model_id, torch_dtype=torch.float16) pipe = StableDiffusionPipeline.from_pretrained(base_model_id, torch_dtype=torch.float16, use_safetensors=True)
pipe = pipe.to("cuda") pipe = pipe.to("cuda")
pipe.load_lora_weights(lora_model_id) pipe.load_lora_weights(lora_model_id)
image = pipe("A picture of a sks dog in a bucket", num_inference_steps=25).images[0] image = pipe("A picture of a sks dog in a bucket", num_inference_steps=25).images[0]
...@@ -307,7 +307,7 @@ import torch ...@@ -307,7 +307,7 @@ import torch
from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler
pipeline = StableDiffusionPipeline.from_pretrained( pipeline = StableDiffusionPipeline.from_pretrained(
"gsdf/Counterfeit-V2.5", torch_dtype=torch.float16, safety_checker=None "gsdf/Counterfeit-V2.5", torch_dtype=torch.float16, safety_checker=None, use_safetensors=True
).to("cuda") ).to("cuda")
pipeline.scheduler = DPMSolverMultistepScheduler.from_config( pipeline.scheduler = DPMSolverMultistepScheduler.from_config(
pipeline.scheduler.config, use_karras_sigmas=True pipeline.scheduler.config, use_karras_sigmas=True
......
...@@ -238,7 +238,7 @@ Now you can load the fine-tuned model for inference by passing the model path or ...@@ -238,7 +238,7 @@ Now you can load the fine-tuned model for inference by passing the model path or
from diffusers import StableDiffusionPipeline from diffusers import StableDiffusionPipeline
model_path = "path_to_saved_model" model_path = "path_to_saved_model"
pipe = StableDiffusionPipeline.from_pretrained(model_path, torch_dtype=torch.float16) pipe = StableDiffusionPipeline.from_pretrained(model_path, torch_dtype=torch.float16, use_safetensors=True)
pipe.to("cuda") pipe.to("cuda")
image = pipe(prompt="yoda").images[0] image = pipe(prompt="yoda").images[0]
......
...@@ -204,7 +204,7 @@ from diffusers import StableDiffusionPipeline ...@@ -204,7 +204,7 @@ from diffusers import StableDiffusionPipeline
import torch import torch
model_id = "runwayml/stable-diffusion-v1-5" model_id = "runwayml/stable-diffusion-v1-5"
pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda") pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16, use_safetensors=True).to("cuda")
``` ```
Next, we need to load the textual inversion embedding vector which can be done via the [`TextualInversionLoaderMixin.load_textual_inversion`] Next, we need to load the textual inversion embedding vector which can be done via the [`TextualInversionLoaderMixin.load_textual_inversion`]
......
...@@ -25,7 +25,7 @@ In this guide, you'll use [`DiffusionPipeline`] for text-to-image generation wit ...@@ -25,7 +25,7 @@ In this guide, you'll use [`DiffusionPipeline`] for text-to-image generation wit
```python ```python
>>> from diffusers import DiffusionPipeline >>> from diffusers import DiffusionPipeline
>>> generator = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") >>> generator = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", use_safetensors=True)
``` ```
The [`DiffusionPipeline`] downloads and caches all modeling, tokenization, and scheduling components. The [`DiffusionPipeline`] downloads and caches all modeling, tokenization, and scheduling components.
......
...@@ -94,7 +94,7 @@ output = pipeline() ...@@ -94,7 +94,7 @@ output = pipeline()
But what's even better is you can load pre-existing weights into the pipeline if the pipeline structure is identical. For example, you can load the [`google/ddpm-cifar10-32`](https://huggingface.co/google/ddpm-cifar10-32) weights into the one-step pipeline: But what's even better is you can load pre-existing weights into the pipeline if the pipeline structure is identical. For example, you can load the [`google/ddpm-cifar10-32`](https://huggingface.co/google/ddpm-cifar10-32) weights into the one-step pipeline:
```python ```python
pipeline = UnetSchedulerOneForwardPipeline.from_pretrained("google/ddpm-cifar10-32") pipeline = UnetSchedulerOneForwardPipeline.from_pretrained("google/ddpm-cifar10-32", use_safetensors=True)
output = pipeline() output = pipeline()
``` ```
...@@ -108,7 +108,9 @@ Once it is merged, anyone with `diffusers >= 0.4.0` installed can use this pipel ...@@ -108,7 +108,9 @@ Once it is merged, anyone with `diffusers >= 0.4.0` installed can use this pipel
```python ```python
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
pipe = DiffusionPipeline.from_pretrained("google/ddpm-cifar10-32", custom_pipeline="one_step_unet") pipe = DiffusionPipeline.from_pretrained(
"google/ddpm-cifar10-32", custom_pipeline="one_step_unet", use_safetensors=True
)
pipe() pipe()
``` ```
...@@ -117,7 +119,9 @@ Another way to share your community pipeline is to upload the `one_step_unet.py` ...@@ -117,7 +119,9 @@ Another way to share your community pipeline is to upload the `one_step_unet.py`
```python ```python
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
pipeline = DiffusionPipeline.from_pretrained("google/ddpm-cifar10-32", custom_pipeline="stevhliu/one_step_unet") pipeline = DiffusionPipeline.from_pretrained(
"google/ddpm-cifar10-32", custom_pipeline="stevhliu/one_step_unet", use_safetensors=True
)
``` ```
Take a look at the following table to compare the two sharing workflows to help you decide the best option for you: Take a look at the following table to compare the two sharing workflows to help you decide the best option for you:
...@@ -161,6 +165,7 @@ pipeline = DiffusionPipeline.from_pretrained( ...@@ -161,6 +165,7 @@ pipeline = DiffusionPipeline.from_pretrained(
feature_extractor=feature_extractor, feature_extractor=feature_extractor,
scheduler=scheduler, scheduler=scheduler,
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
) )
``` ```
......
...@@ -24,7 +24,7 @@ Next, configure the following parameters in the [`DDIMScheduler`]: ...@@ -24,7 +24,7 @@ Next, configure the following parameters in the [`DDIMScheduler`]:
```py ```py
>>> from diffusers import DiffusionPipeline, DDIMScheduler >>> from diffusers import DiffusionPipeline, DDIMScheduler
>>> pipeline = DiffusionPipeline.from_pretrained("ptx0/pseudo-journey-v2") >>> pipeline = DiffusionPipeline.from_pretrained("ptx0/pseudo-journey-v2", use_safetensors=True)
# switch the scheduler in the pipeline to use the DDIMScheduler # switch the scheduler in the pipeline to use the DDIMScheduler
>>> pipeline.scheduler = DDIMScheduler.from_config( >>> pipeline.scheduler = DDIMScheduler.from_config(
......
...@@ -32,7 +32,7 @@ If a community doesn't work as expected, please open an issue and ping the autho ...@@ -32,7 +32,7 @@ If a community doesn't work as expected, please open an issue and ping the autho
To load a custom pipeline you just need to pass the `custom_pipeline` argument to `DiffusionPipeline`, as one of the files in `diffusers/examples/community`. Feel free to send a PR with your own pipelines, we will merge them quickly. To load a custom pipeline you just need to pass the `custom_pipeline` argument to `DiffusionPipeline`, as one of the files in `diffusers/examples/community`. Feel free to send a PR with your own pipelines, we will merge them quickly.
```py ```py
pipe = DiffusionPipeline.from_pretrained( pipe = DiffusionPipeline.from_pretrained(
"CompVis/stable-diffusion-v1-4", custom_pipeline="filename_in_the_community_folder" "CompVis/stable-diffusion-v1-4", custom_pipeline="filename_in_the_community_folder", use_safetensors=True
) )
``` ```
...@@ -61,6 +61,7 @@ guided_pipeline = DiffusionPipeline.from_pretrained( ...@@ -61,6 +61,7 @@ guided_pipeline = DiffusionPipeline.from_pretrained(
clip_model=clip_model, clip_model=clip_model,
feature_extractor=feature_extractor, feature_extractor=feature_extractor,
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
) )
guided_pipeline.enable_attention_slicing() guided_pipeline.enable_attention_slicing()
guided_pipeline = guided_pipeline.to("cuda") guided_pipeline = guided_pipeline.to("cuda")
...@@ -117,6 +118,7 @@ pipe = DiffusionPipeline.from_pretrained( ...@@ -117,6 +118,7 @@ pipe = DiffusionPipeline.from_pretrained(
torch_dtype=torch.float16, torch_dtype=torch.float16,
safety_checker=None, # Very important for videos...lots of false positives while interpolating safety_checker=None, # Very important for videos...lots of false positives while interpolating
custom_pipeline="interpolate_stable_diffusion", custom_pipeline="interpolate_stable_diffusion",
use_safetensors=True,
).to("cuda") ).to("cuda")
pipe.enable_attention_slicing() pipe.enable_attention_slicing()
...@@ -159,6 +161,7 @@ pipe = DiffusionPipeline.from_pretrained( ...@@ -159,6 +161,7 @@ pipe = DiffusionPipeline.from_pretrained(
"CompVis/stable-diffusion-v1-4", "CompVis/stable-diffusion-v1-4",
custom_pipeline="stable_diffusion_mega", custom_pipeline="stable_diffusion_mega",
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
) )
pipe.to("cuda") pipe.to("cuda")
pipe.enable_attention_slicing() pipe.enable_attention_slicing()
...@@ -203,7 +206,7 @@ from diffusers import DiffusionPipeline ...@@ -203,7 +206,7 @@ from diffusers import DiffusionPipeline
import torch import torch
pipe = DiffusionPipeline.from_pretrained( pipe = DiffusionPipeline.from_pretrained(
"hakurei/waifu-diffusion", custom_pipeline="lpw_stable_diffusion", torch_dtype=torch.float16 "hakurei/waifu-diffusion", custom_pipeline="lpw_stable_diffusion", torch_dtype=torch.float16, use_safetensors=True
) )
pipe = pipe.to("cuda") pipe = pipe.to("cuda")
...@@ -224,6 +227,7 @@ pipe = DiffusionPipeline.from_pretrained( ...@@ -224,6 +227,7 @@ pipe = DiffusionPipeline.from_pretrained(
custom_pipeline="lpw_stable_diffusion_onnx", custom_pipeline="lpw_stable_diffusion_onnx",
revision="onnx", revision="onnx",
provider="CUDAExecutionProvider", provider="CUDAExecutionProvider",
use_safetensors=True,
) )
prompt = "a photo of an astronaut riding a horse on mars, best quality" prompt = "a photo of an astronaut riding a horse on mars, best quality"
...@@ -267,8 +271,8 @@ diffuser_pipeline = DiffusionPipeline.from_pretrained( ...@@ -267,8 +271,8 @@ diffuser_pipeline = DiffusionPipeline.from_pretrained(
custom_pipeline="speech_to_image_diffusion", custom_pipeline="speech_to_image_diffusion",
speech_model=model, speech_model=model,
speech_processor=processor, speech_processor=processor,
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
) )
diffuser_pipeline.enable_attention_slicing() diffuser_pipeline.enable_attention_slicing()
......
...@@ -30,7 +30,7 @@ To load any community pipeline on the Hub, pass the repository id of the communi ...@@ -30,7 +30,7 @@ To load any community pipeline on the Hub, pass the repository id of the communi
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
pipeline = DiffusionPipeline.from_pretrained( pipeline = DiffusionPipeline.from_pretrained(
"google/ddpm-cifar10-32", custom_pipeline="hf-internal-testing/diffusers-dummy-pipeline" "google/ddpm-cifar10-32", custom_pipeline="hf-internal-testing/diffusers-dummy-pipeline", use_safetensors=True
) )
``` ```
...@@ -50,6 +50,7 @@ pipeline = DiffusionPipeline.from_pretrained( ...@@ -50,6 +50,7 @@ pipeline = DiffusionPipeline.from_pretrained(
custom_pipeline="clip_guided_stable_diffusion", custom_pipeline="clip_guided_stable_diffusion",
clip_model=clip_model, clip_model=clip_model,
feature_extractor=feature_extractor, feature_extractor=feature_extractor,
use_safetensors=True,
) )
``` ```
......
...@@ -28,6 +28,7 @@ from diffusers import StableDiffusionDepth2ImgPipeline ...@@ -28,6 +28,7 @@ from diffusers import StableDiffusionDepth2ImgPipeline
pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( pipe = StableDiffusionDepth2ImgPipeline.from_pretrained(
"stabilityai/stable-diffusion-2-depth", "stabilityai/stable-diffusion-2-depth",
torch_dtype=torch.float16, torch_dtype=torch.float16,
use_safetensors=True,
).to("cuda") ).to("cuda")
``` ```
......
...@@ -33,9 +33,9 @@ from io import BytesIO ...@@ -33,9 +33,9 @@ from io import BytesIO
from diffusers import StableDiffusionImg2ImgPipeline from diffusers import StableDiffusionImg2ImgPipeline
device = "cuda" device = "cuda"
pipe = StableDiffusionImg2ImgPipeline.from_pretrained("nitrosocke/Ghibli-Diffusion", torch_dtype=torch.float16).to( pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
device "nitrosocke/Ghibli-Diffusion", torch_dtype=torch.float16, use_safetensors=True
) ).to(device)
``` ```
Download and preprocess an initial image so you can pass it to the pipeline: Download and preprocess an initial image so you can pass it to the pipeline:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment