"vscode:/vscode.git/clone" did not exist on "58095cb00aec2c08f3cdaf0109ea6b323c68f046"
Unverified Commit b52119ae authored by suzukimain's avatar suzukimain Committed by GitHub
Browse files

[docs] Replace runwayml/stable-diffusion-v1-5 with Lykon/dreamshaper-8 (#9428)



* [docs] Replace runwayml/stable-diffusion-v1-5 with Lykon/dreamshaper-8

Updated documentation as runwayml/stable-diffusion-v1-5 has been removed from Huggingface.

* Update docs/source/en/using-diffusers/inpaint.md
Co-authored-by: default avatarSteven Liu <59462357+stevhliu@users.noreply.github.com>

* Replace with stable-diffusion-v1-5/stable-diffusion-v1-5

* Update inpaint.md

---------
Co-authored-by: default avatarSteven Liu <59462357+stevhliu@users.noreply.github.com>
parent 8336405e
...@@ -22,12 +22,12 @@ specific language governing permissions and limitations under the License. ...@@ -22,12 +22,12 @@ specific language governing permissions and limitations under the License.
本教程将指导您如何通过 [`DiffusionPipeline`] 更快、更好地生成图像。 本教程将指导您如何通过 [`DiffusionPipeline`] 更快、更好地生成图像。
首先,加载 [`runwayml/stable-diffusion-v1-5`](https://huggingface.co/runwayml/stable-diffusion-v1-5) 模型: 首先,加载 [`stable-diffusion-v1-5/stable-diffusion-v1-5`](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5) 模型:
```python ```python
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
model_id = "runwayml/stable-diffusion-v1-5" model_id = "stable-diffusion-v1-5/stable-diffusion-v1-5"
pipeline = DiffusionPipeline.from_pretrained(model_id, use_safetensors=True) pipeline = DiffusionPipeline.from_pretrained(model_id, use_safetensors=True)
``` ```
......
...@@ -76,7 +76,7 @@ Please also check out our [Community Scripts](https://github.com/huggingface/dif ...@@ -76,7 +76,7 @@ Please also check out our [Community Scripts](https://github.com/huggingface/dif
To load a custom pipeline you just need to pass the `custom_pipeline` argument to `DiffusionPipeline`, as one of the files in `diffusers/examples/community`. Feel free to send a PR with your own pipelines, we will merge them quickly. To load a custom pipeline you just need to pass the `custom_pipeline` argument to `DiffusionPipeline`, as one of the files in `diffusers/examples/community`. Feel free to send a PR with your own pipelines, we will merge them quickly.
```py ```py
pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", custom_pipeline="filename_in_the_community_folder") pipe = DiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5", custom_pipeline="filename_in_the_community_folder")
``` ```
## Example usages ## Example usages
...@@ -363,7 +363,7 @@ clip_model = CLIPModel.from_pretrained("laion/CLIP-ViT-B-32-laion2B-s34B-b79K", ...@@ -363,7 +363,7 @@ clip_model = CLIPModel.from_pretrained("laion/CLIP-ViT-B-32-laion2B-s34B-b79K",
guided_pipeline = DiffusionPipeline.from_pretrained( guided_pipeline = DiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "stable-diffusion-v1-5/stable-diffusion-v1-5",
custom_pipeline="clip_guided_stable_diffusion", custom_pipeline="clip_guided_stable_diffusion",
clip_model=clip_model, clip_model=clip_model,
feature_extractor=feature_extractor, feature_extractor=feature_extractor,
...@@ -462,7 +462,7 @@ def download_image(url): ...@@ -462,7 +462,7 @@ def download_image(url):
response = requests.get(url) response = requests.get(url)
return PIL.Image.open(BytesIO(response.content)).convert("RGB") return PIL.Image.open(BytesIO(response.content)).convert("RGB")
pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", custom_pipeline="stable_diffusion_mega", torch_dtype=torch.float16, variant="fp16") pipe = DiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5", custom_pipeline="stable_diffusion_mega", torch_dtype=torch.float16, variant="fp16")
pipe.to("cuda") pipe.to("cuda")
pipe.enable_attention_slicing() pipe.enable_attention_slicing()
...@@ -921,7 +921,7 @@ if __name__ == "__main__": ...@@ -921,7 +921,7 @@ if __name__ == "__main__":
tensor_norm = torch.Tensor([[43.8203],[28.3668],[27.9345],[28.0084],[28.2958],[28.2576],[28.3373],[28.2695],[28.4097],[28.2790],[28.2825],[28.2807],[28.2775],[28.2708],[28.2682],[28.2624],[28.2589],[28.2611],[28.2616],[28.2639],[28.2613],[28.2566],[28.2615],[28.2665],[28.2799],[28.2885],[28.2852],[28.2863],[28.2780],[28.2818],[28.2764],[28.2532],[28.2412],[28.2336],[28.2514],[28.2734],[28.2763],[28.2977],[28.2971],[28.2948],[28.2818],[28.2676],[28.2831],[28.2890],[28.2979],[28.2999],[28.3117],[28.3363],[28.3554],[28.3626],[28.3589],[28.3597],[28.3543],[28.3660],[28.3731],[28.3717],[28.3812],[28.3753],[28.3810],[28.3777],[28.3693],[28.3713],[28.3670],[28.3691],[28.3679],[28.3624],[28.3703],[28.3703],[28.3720],[28.3594],[28.3576],[28.3562],[28.3438],[28.3376],[28.3389],[28.3433],[28.3191]]) tensor_norm = torch.Tensor([[43.8203],[28.3668],[27.9345],[28.0084],[28.2958],[28.2576],[28.3373],[28.2695],[28.4097],[28.2790],[28.2825],[28.2807],[28.2775],[28.2708],[28.2682],[28.2624],[28.2589],[28.2611],[28.2616],[28.2639],[28.2613],[28.2566],[28.2615],[28.2665],[28.2799],[28.2885],[28.2852],[28.2863],[28.2780],[28.2818],[28.2764],[28.2532],[28.2412],[28.2336],[28.2514],[28.2734],[28.2763],[28.2977],[28.2971],[28.2948],[28.2818],[28.2676],[28.2831],[28.2890],[28.2979],[28.2999],[28.3117],[28.3363],[28.3554],[28.3626],[28.3589],[28.3597],[28.3543],[28.3660],[28.3731],[28.3717],[28.3812],[28.3753],[28.3810],[28.3777],[28.3693],[28.3713],[28.3670],[28.3691],[28.3679],[28.3624],[28.3703],[28.3703],[28.3720],[28.3594],[28.3576],[28.3562],[28.3438],[28.3376],[28.3389],[28.3433],[28.3191]])
pipeline = DiffusionPipeline.from_pretrained( pipeline = DiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "stable-diffusion-v1-5/stable-diffusion-v1-5",
text_encoder=text_encoder, text_encoder=text_encoder,
tokenizer=tokenizer, tokenizer=tokenizer,
custom_pipeline="gluegen" custom_pipeline="gluegen"
...@@ -1744,7 +1744,7 @@ from diffusers.utils import load_image ...@@ -1744,7 +1744,7 @@ from diffusers.utils import load_image
input_image = load_image("https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png") input_image = load_image("https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png")
pipe = StableDiffusionReferencePipeline.from_pretrained( pipe = StableDiffusionReferencePipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "stable-diffusion-v1-5/stable-diffusion-v1-5",
safety_checker=None, safety_checker=None,
torch_dtype=torch.float16 torch_dtype=torch.float16
).to('cuda:0') ).to('cuda:0')
...@@ -1801,7 +1801,7 @@ canny_image = Image.fromarray(image) ...@@ -1801,7 +1801,7 @@ canny_image = Image.fromarray(image)
controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16) controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
pipe = StableDiffusionControlNetReferencePipeline.from_pretrained( pipe = StableDiffusionControlNetReferencePipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "stable-diffusion-v1-5/stable-diffusion-v1-5",
controlnet=controlnet, controlnet=controlnet,
safety_checker=None, safety_checker=None,
torch_dtype=torch.float16 torch_dtype=torch.float16
...@@ -1857,7 +1857,7 @@ python -m pip install intel_extension_for_pytorch==<version_name> -f https://dev ...@@ -1857,7 +1857,7 @@ python -m pip install intel_extension_for_pytorch==<version_name> -f https://dev
**Note:** The setting of generated image height/width for `prepare_for_ipex()` should be same as the setting of pipeline inference. **Note:** The setting of generated image height/width for `prepare_for_ipex()` should be same as the setting of pipeline inference.
```python ```python
pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", custom_pipeline="stable_diffusion_ipex") pipe = DiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5", custom_pipeline="stable_diffusion_ipex")
# For Float32 # For Float32
pipe.prepare_for_ipex(prompt, dtype=torch.float32, height=512, width=512) # value of image height/width should be consistent with the pipeline inference pipe.prepare_for_ipex(prompt, dtype=torch.float32, height=512, width=512) # value of image height/width should be consistent with the pipeline inference
# For BFloat16 # For BFloat16
...@@ -1883,7 +1883,7 @@ from diffusers import StableDiffusionPipeline ...@@ -1883,7 +1883,7 @@ from diffusers import StableDiffusionPipeline
import time import time
prompt = "sailing ship in storm by Rembrandt" prompt = "sailing ship in storm by Rembrandt"
model_id = "runwayml/stable-diffusion-v1-5" model_id = "stable-diffusion-v1-5/stable-diffusion-v1-5"
# Helper function for time evaluation # Helper function for time evaluation
def elapsed_time(pipeline, nb_pass=3, num_inference_steps=20): def elapsed_time(pipeline, nb_pass=3, num_inference_steps=20):
# warmup # warmup
...@@ -2507,7 +2507,7 @@ from diffusers import DiffusionPipeline ...@@ -2507,7 +2507,7 @@ from diffusers import DiffusionPipeline
# load the pipeline # load the pipeline
# make sure you're logged in with `huggingface-cli login` # make sure you're logged in with `huggingface-cli login`
model_id_or_path = "runwayml/stable-diffusion-v1-5" model_id_or_path = "stable-diffusion-v1-5/stable-diffusion-v1-5"
# can also be used with dreamlike-art/dreamlike-photoreal-2.0 # can also be used with dreamlike-art/dreamlike-photoreal-2.0
pipe = DiffusionPipeline.from_pretrained(model_id_or_path, torch_dtype=torch.float16, custom_pipeline="pipeline_fabric").to("cuda") pipe = DiffusionPipeline.from_pretrained(model_id_or_path, torch_dtype=torch.float16, custom_pipeline="pipeline_fabric").to("cuda")
...@@ -2548,7 +2548,7 @@ image.save("black_to_blue.png") ...@@ -2548,7 +2548,7 @@ image.save("black_to_blue.png")
*With enough feedbacks you can create very similar high quality images.* *With enough feedbacks you can create very similar high quality images.*
The original codebase can be found at [sd-fabric/fabric](https://github.com/sd-fabric/fabric), and available checkpoints are [dreamlike-art/dreamlike-photoreal-2.0](https://huggingface.co/dreamlike-art/dreamlike-photoreal-2.0), [runwayml/stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5), and [stabilityai/stable-diffusion-2-1](https://huggingface.co/stabilityai/stable-diffusion-2-1) (may give unexpected results). The original codebase can be found at [sd-fabric/fabric](https://github.com/sd-fabric/fabric), and available checkpoints are [dreamlike-art/dreamlike-photoreal-2.0](https://huggingface.co/dreamlike-art/dreamlike-photoreal-2.0), [stable-diffusion-v1-5/stable-diffusion-v1-5](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5), and [stabilityai/stable-diffusion-2-1](https://huggingface.co/stabilityai/stable-diffusion-2-1) (may give unexpected results).
Let's have a look at the images (_512X512_) Let's have a look at the images (_512X512_)
...@@ -3579,7 +3579,7 @@ import torch ...@@ -3579,7 +3579,7 @@ import torch
from diffusers import DDIMScheduler, DiffusionPipeline from diffusers import DDIMScheduler, DiffusionPipeline
# Load the pipeline # Load the pipeline
model_path = "runwayml/stable-diffusion-v1-5" model_path = "stable-diffusion-v1-5/stable-diffusion-v1-5"
scheduler = DDIMScheduler.from_pretrained(model_path, subfolder="scheduler") scheduler = DDIMScheduler.from_pretrained(model_path, subfolder="scheduler")
pipe = DiffusionPipeline.from_pretrained(model_path, scheduler=scheduler, custom_pipeline="sde_drag") pipe = DiffusionPipeline.from_pretrained(model_path, scheduler=scheduler, custom_pipeline="sde_drag")
pipe.to('cuda') pipe.to('cuda')
...@@ -3674,7 +3674,7 @@ prompt = "A lying cat" ...@@ -3674,7 +3674,7 @@ prompt = "A lying cat"
prompt = "A lying dog" prompt = "A lying dog"
# Float32 is essential to a well optimization # Float32 is essential to a well optimization
model_path = "runwayml/stable-diffusion-v1-5" model_path = "stable-diffusion-v1-5/stable-diffusion-v1-5"
scheduler = DDIMScheduler(num_train_timesteps=1000, beta_start=0.00085, beta_end=0.0120, beta_schedule="scaled_linear") scheduler = DDIMScheduler(num_train_timesteps=1000, beta_start=0.00085, beta_end=0.0120, beta_schedule="scaled_linear")
pipeline = NullTextPipeline.from_pretrained(model_path, scheduler=scheduler, torch_dtype=torch.float32).to(device) pipeline = NullTextPipeline.from_pretrained(model_path, scheduler=scheduler, torch_dtype=torch.float32).to(device)
...@@ -3738,7 +3738,7 @@ controlnet = ControlNetModel.from_pretrained( ...@@ -3738,7 +3738,7 @@ controlnet = ControlNetModel.from_pretrained(
# You can use any finetuned SD here # You can use any finetuned SD here
pipe = DiffusionPipeline.from_pretrained( pipe = DiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", controlnet=controlnet, custom_pipeline='rerender_a_video').to('cuda') "stable-diffusion-v1-5/stable-diffusion-v1-5", controlnet=controlnet, custom_pipeline='rerender_a_video').to('cuda')
# Optional: you can download vae-ft-mse-840000-ema-pruned.ckpt to enhance the results # Optional: you can download vae-ft-mse-840000-ema-pruned.ckpt to enhance the results
# pipe.vae = AutoencoderKL.from_single_file( # pipe.vae = AutoencoderKL.from_single_file(
...@@ -4274,7 +4274,7 @@ from diffusers.utils import load_image, make_image_grid ...@@ -4274,7 +4274,7 @@ from diffusers.utils import load_image, make_image_grid
from diffusers.utils.torch_utils import randn_tensor from diffusers.utils.torch_utils import randn_tensor
pipe = StableDiffusionPipeline.from_pretrained( pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "stable-diffusion-v1-5/stable-diffusion-v1-5",
custom_pipeline="hyoungwoncho/sd_perturbed_attention_guidance", custom_pipeline="hyoungwoncho/sd_perturbed_attention_guidance",
torch_dtype=torch.float16 torch_dtype=torch.float16
) )
......
...@@ -209,7 +209,7 @@ def seamless_tiling(pipeline, x_axis, y_axis): ...@@ -209,7 +209,7 @@ def seamless_tiling(pipeline, x_axis, y_axis):
layer._conv_forward = asymmetric_conv2d_convforward.__get__(layer, torch.nn.Conv2d) layer._conv_forward = asymmetric_conv2d_convforward.__get__(layer, torch.nn.Conv2d)
return pipeline return pipeline
pipeline = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16, use_safetensors=True) pipeline = StableDiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16, use_safetensors=True)
pipeline.enable_model_cpu_offload() pipeline.enable_model_cpu_offload()
prompt = ["texture of a red brick wall"] prompt = ["texture of a red brick wall"]
seed = 123456 seed = 123456
......
...@@ -51,7 +51,7 @@ When running `accelerate config`, if we specify torch compile mode to True there ...@@ -51,7 +51,7 @@ When running `accelerate config`, if we specify torch compile mode to True there
The following uses the [Conceptual Captions 12M (CC12M) dataset](https://github.com/google-research-datasets/conceptual-12m) as an example, and for illustrative purposes only. For best results you may consider large and high-quality text-image datasets such as [LAION](https://laion.ai/blog/laion-400-open-dataset/). You may also need to search the hyperparameter space according to the dataset you use. The following uses the [Conceptual Captions 12M (CC12M) dataset](https://github.com/google-research-datasets/conceptual-12m) as an example, and for illustrative purposes only. For best results you may consider large and high-quality text-image datasets such as [LAION](https://laion.ai/blog/laion-400-open-dataset/). You may also need to search the hyperparameter space according to the dataset you use.
```bash ```bash
export MODEL_NAME="runwayml/stable-diffusion-v1-5" export MODEL_NAME="stable-diffusion-v1-5/stable-diffusion-v1-5"
export OUTPUT_DIR="path/to/saved/model" export OUTPUT_DIR="path/to/saved/model"
accelerate launch train_lcm_distill_sd_wds.py \ accelerate launch train_lcm_distill_sd_wds.py \
...@@ -85,7 +85,7 @@ Instead of fine-tuning the full model, we can also just train a LoRA that can be ...@@ -85,7 +85,7 @@ Instead of fine-tuning the full model, we can also just train a LoRA that can be
The following uses the [Conceptual Captions 12M (CC12M) dataset](https://github.com/google-research-datasets/conceptual-12m) as an example. For best results you may consider large and high-quality text-image datasets such as [LAION](https://laion.ai/blog/laion-400-open-dataset/). The following uses the [Conceptual Captions 12M (CC12M) dataset](https://github.com/google-research-datasets/conceptual-12m) as an example. For best results you may consider large and high-quality text-image datasets such as [LAION](https://laion.ai/blog/laion-400-open-dataset/).
```bash ```bash
export MODEL_NAME="runwayml/stable-diffusion-v1-5" export MODEL_NAME="stable-diffusion-v1-5/stable-diffusion-v1-5"
export OUTPUT_DIR="path/to/saved/model" export OUTPUT_DIR="path/to/saved/model"
accelerate launch train_lcm_distill_lora_sd_wds.py \ accelerate launch train_lcm_distill_lora_sd_wds.py \
......
...@@ -45,7 +45,7 @@ write_basic_config() ...@@ -45,7 +45,7 @@ write_basic_config()
The original dataset is hosted in the [ControlNet repo](https://huggingface.co/lllyasviel/ControlNet/blob/main/training/fill50k.zip). We re-uploaded it to be compatible with `datasets` [here](https://huggingface.co/datasets/fusing/fill50k). Note that `datasets` handles dataloading within the training script. The original dataset is hosted in the [ControlNet repo](https://huggingface.co/lllyasviel/ControlNet/blob/main/training/fill50k.zip). We re-uploaded it to be compatible with `datasets` [here](https://huggingface.co/datasets/fusing/fill50k). Note that `datasets` handles dataloading within the training script.
Our training examples use [Stable Diffusion 1.5](https://huggingface.co/runwayml/stable-diffusion-v1-5) as the original set of ControlNet models were trained from it. However, ControlNet can be trained to augment any Stable Diffusion compatible model (such as [CompVis/stable-diffusion-v1-4](https://huggingface.co/CompVis/stable-diffusion-v1-4)) or [stabilityai/stable-diffusion-2-1](https://huggingface.co/stabilityai/stable-diffusion-2-1). Our training examples use [Stable Diffusion 1.5](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5) as the original set of ControlNet models were trained from it. However, ControlNet can be trained to augment any Stable Diffusion compatible model (such as [CompVis/stable-diffusion-v1-4](https://huggingface.co/CompVis/stable-diffusion-v1-4)) or [stabilityai/stable-diffusion-2-1](https://huggingface.co/stabilityai/stable-diffusion-2-1).
## Training ## Training
...@@ -59,7 +59,7 @@ wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/ma ...@@ -59,7 +59,7 @@ wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/ma
```bash ```bash
export MODEL_DIR="runwayml/stable-diffusion-v1-5" export MODEL_DIR="stable-diffusion-v1-5/stable-diffusion-v1-5"
export OUTPUT_DIR="path to save model" export OUTPUT_DIR="path to save model"
accelerate launch train_controlnet.py \ accelerate launch train_controlnet.py \
...@@ -81,7 +81,7 @@ biases. ...@@ -81,7 +81,7 @@ biases.
Gradient accumulation with a smaller batch size can be used to reduce training requirements to ~20 GB VRAM. Gradient accumulation with a smaller batch size can be used to reduce training requirements to ~20 GB VRAM.
```bash ```bash
export MODEL_DIR="runwayml/stable-diffusion-v1-5" export MODEL_DIR="stable-diffusion-v1-5/stable-diffusion-v1-5"
export OUTPUT_DIR="path to save model" export OUTPUT_DIR="path to save model"
accelerate launch train_controlnet.py \ accelerate launch train_controlnet.py \
...@@ -102,7 +102,7 @@ accelerate launch train_controlnet.py \ ...@@ -102,7 +102,7 @@ accelerate launch train_controlnet.py \
for running distributed training with `accelerate`. Here is an example command: for running distributed training with `accelerate`. Here is an example command:
```bash ```bash
export MODEL_DIR="runwayml/stable-diffusion-v1-5" export MODEL_DIR="stable-diffusion-v1-5/stable-diffusion-v1-5"
export OUTPUT_DIR="path to save model" export OUTPUT_DIR="path to save model"
accelerate launch --mixed_precision="fp16" --multi_gpu train_controlnet.py \ accelerate launch --mixed_precision="fp16" --multi_gpu train_controlnet.py \
...@@ -149,7 +149,7 @@ Optimizations: ...@@ -149,7 +149,7 @@ Optimizations:
[bitandbytes install instructions](https://github.com/TimDettmers/bitsandbytes#requirements--installation). [bitandbytes install instructions](https://github.com/TimDettmers/bitsandbytes#requirements--installation).
```bash ```bash
export MODEL_DIR="runwayml/stable-diffusion-v1-5" export MODEL_DIR="stable-diffusion-v1-5/stable-diffusion-v1-5"
export OUTPUT_DIR="path to save model" export OUTPUT_DIR="path to save model"
accelerate launch train_controlnet.py \ accelerate launch train_controlnet.py \
...@@ -175,7 +175,7 @@ Optimizations: ...@@ -175,7 +175,7 @@ Optimizations:
- set grads to none - set grads to none
```bash ```bash
export MODEL_DIR="runwayml/stable-diffusion-v1-5" export MODEL_DIR="stable-diffusion-v1-5/stable-diffusion-v1-5"
export OUTPUT_DIR="path to save model" export OUTPUT_DIR="path to save model"
accelerate launch train_controlnet.py \ accelerate launch train_controlnet.py \
...@@ -235,7 +235,7 @@ it requires CUDA toolchain with the same version as pytorch. 8-bit optimizer ...@@ -235,7 +235,7 @@ it requires CUDA toolchain with the same version as pytorch. 8-bit optimizer
does not seem to be compatible with DeepSpeed at the moment. does not seem to be compatible with DeepSpeed at the moment.
```bash ```bash
export MODEL_DIR="runwayml/stable-diffusion-v1-5" export MODEL_DIR="stable-diffusion-v1-5/stable-diffusion-v1-5"
export OUTPUT_DIR="path to save model" export OUTPUT_DIR="path to save model"
accelerate launch train_controlnet.py \ accelerate launch train_controlnet.py \
...@@ -365,7 +365,7 @@ huggingface-cli login ...@@ -365,7 +365,7 @@ huggingface-cli login
Make sure you have the `MODEL_DIR`,`OUTPUT_DIR` and `HUB_MODEL_ID` environment variables set. The `OUTPUT_DIR` and `HUB_MODEL_ID` variables specify where to save the model to on the Hub: Make sure you have the `MODEL_DIR`,`OUTPUT_DIR` and `HUB_MODEL_ID` environment variables set. The `OUTPUT_DIR` and `HUB_MODEL_ID` variables specify where to save the model to on the Hub:
```bash ```bash
export MODEL_DIR="runwayml/stable-diffusion-v1-5" export MODEL_DIR="stable-diffusion-v1-5/stable-diffusion-v1-5"
export OUTPUT_DIR="runs/fill-circle-{timestamp}" export OUTPUT_DIR="runs/fill-circle-{timestamp}"
export HUB_MODEL_ID="controlnet-fill-circle" export HUB_MODEL_ID="controlnet-fill-circle"
``` ```
...@@ -397,7 +397,7 @@ Since we passed the `--push_to_hub` flag, it will automatically create a model r ...@@ -397,7 +397,7 @@ Since we passed the `--push_to_hub` flag, it will automatically create a model r
Our training script also provides limited support for streaming large datasets from the Hugging Face Hub. In order to enable streaming, one must also set `--max_train_samples`. Here is an example command (from [this blog article](https://huggingface.co/blog/train-your-controlnet)): Our training script also provides limited support for streaming large datasets from the Hugging Face Hub. In order to enable streaming, one must also set `--max_train_samples`. Here is an example command (from [this blog article](https://huggingface.co/blog/train-your-controlnet)):
```bash ```bash
export MODEL_DIR="runwayml/stable-diffusion-v1-5" export MODEL_DIR="stable-diffusion-v1-5/stable-diffusion-v1-5"
export OUTPUT_DIR="runs/uncanny-faces-{timestamp}" export OUTPUT_DIR="runs/uncanny-faces-{timestamp}"
export HUB_MODEL_ID="controlnet-uncanny-faces" export HUB_MODEL_ID="controlnet-uncanny-faces"
......
...@@ -313,7 +313,7 @@ Let's get started with a simple example. We will re-use the dog example of the [ ...@@ -313,7 +313,7 @@ Let's get started with a simple example. We will re-use the dog example of the [
First, you need to set-up your dreambooth training example as is explained in the [installation section](#Installing-the-dependencies). First, you need to set-up your dreambooth training example as is explained in the [installation section](#Installing-the-dependencies).
Next, let's download the dog dataset. Download images from [here](https://drive.google.com/drive/folders/1BO_dyz-p65qhBRRMRA4TbZ8qW4rB99JZ) and save them in a directory. Make sure to set `INSTANCE_DIR` to the name of your directory further below. This will be our training data. Next, let's download the dog dataset. Download images from [here](https://drive.google.com/drive/folders/1BO_dyz-p65qhBRRMRA4TbZ8qW4rB99JZ) and save them in a directory. Make sure to set `INSTANCE_DIR` to the name of your directory further below. This will be our training data.
Now, you can launch the training. Here we will use [Stable Diffusion 1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5). Now, you can launch the training. Here we will use [Stable Diffusion 1-5](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5).
**___Note: Change the `resolution` to 768 if you are using the [stable-diffusion-2](https://huggingface.co/stabilityai/stable-diffusion-2) 768x768 model.___** **___Note: Change the `resolution` to 768 if you are using the [stable-diffusion-2](https://huggingface.co/stabilityai/stable-diffusion-2) 768x768 model.___**
...@@ -321,7 +321,7 @@ Now, you can launch the training. Here we will use [Stable Diffusion 1-5](https: ...@@ -321,7 +321,7 @@ Now, you can launch the training. Here we will use [Stable Diffusion 1-5](https:
```bash ```bash
export MODEL_NAME="runwayml/stable-diffusion-v1-5" export MODEL_NAME="stable-diffusion-v1-5/stable-diffusion-v1-5"
export INSTANCE_DIR="dog" export INSTANCE_DIR="dog"
export OUTPUT_DIR="path-to-save-model" export OUTPUT_DIR="path-to-save-model"
``` ```
......
...@@ -65,7 +65,7 @@ Configure environment variables such as the dataset identifier and the Stable Di ...@@ -65,7 +65,7 @@ Configure environment variables such as the dataset identifier and the Stable Di
checkpoint: checkpoint:
```bash ```bash
export MODEL_NAME="runwayml/stable-diffusion-v1-5" export MODEL_NAME="stable-diffusion-v1-5/stable-diffusion-v1-5"
export DATASET_ID="fusing/instructpix2pix-1000-samples" export DATASET_ID="fusing/instructpix2pix-1000-samples"
``` ```
...@@ -122,7 +122,7 @@ for running distributed training with `accelerate`. Here is an example command: ...@@ -122,7 +122,7 @@ for running distributed training with `accelerate`. Here is an example command:
```bash ```bash
accelerate launch --mixed_precision="fp16" --multi_gpu train_instruct_pix2pix.py \ accelerate launch --mixed_precision="fp16" --multi_gpu train_instruct_pix2pix.py \
--pretrained_model_name_or_path=runwayml/stable-diffusion-v1-5 \ --pretrained_model_name_or_path=stable-diffusion-v1-5/stable-diffusion-v1-5 \
--dataset_name=sayakpaul/instructpix2pix-1000-samples \ --dataset_name=sayakpaul/instructpix2pix-1000-samples \
--use_ema \ --use_ema \
--enable_xformers_memory_efficient_attention \ --enable_xformers_memory_efficient_attention \
......
...@@ -155,7 +155,7 @@ If you're looking for some interesting ways to use the InstructPix2Pix training ...@@ -155,7 +155,7 @@ If you're looking for some interesting ways to use the InstructPix2Pix training
We aim to understand the differences resulting from the use of SD-1.5 and SDXL-0.9 as pretrained models. To achieve this, we trained on the [small toy dataset](https://huggingface.co/datasets/fusing/instructpix2pix-1000-samples) using both of these pretrained models. The training script is as follows: We aim to understand the differences resulting from the use of SD-1.5 and SDXL-0.9 as pretrained models. To achieve this, we trained on the [small toy dataset](https://huggingface.co/datasets/fusing/instructpix2pix-1000-samples) using both of these pretrained models. The training script is as follows:
```bash ```bash
export MODEL_NAME="runwayml/stable-diffusion-v1-5" or "stabilityai/stable-diffusion-xl-base-0.9" export MODEL_NAME="stable-diffusion-v1-5/stable-diffusion-v1-5" or "stabilityai/stable-diffusion-xl-base-0.9"
export DATASET_ID="fusing/instructpix2pix-1000-samples" export DATASET_ID="fusing/instructpix2pix-1000-samples"
accelerate launch train_instruct_pix2pix.py \ accelerate launch train_instruct_pix2pix.py \
......
...@@ -13,7 +13,7 @@ We provide implementations for both Stable Diffusion (SD) and Stable Diffusion X ...@@ -13,7 +13,7 @@ We provide implementations for both Stable Diffusion (SD) and Stable Diffusion X
```bash ```bash
accelerate launch train_diffusion_dpo.py \ accelerate launch train_diffusion_dpo.py \
--pretrained_model_name_or_path=runwayml/stable-diffusion-v1-5 \ --pretrained_model_name_or_path=stable-diffusion-v1-5/stable-diffusion-v1-5 \
--output_dir="diffusion-dpo" \ --output_dir="diffusion-dpo" \
--mixed_precision="fp16" \ --mixed_precision="fp16" \
--dataset_name=kashif/pickascore \ --dataset_name=kashif/pickascore \
......
...@@ -53,7 +53,7 @@ accelerate config ...@@ -53,7 +53,7 @@ accelerate config
### Cat toy example ### Cat toy example
You need to accept the model license before downloading or using the weights. In this example we'll use model version `v1-5`, so you'll need to visit [its card](https://huggingface.co/runwayml/stable-diffusion-v1-5), read the license and tick the checkbox if you agree. You need to accept the model license before downloading or using the weights. In this example we'll use model version `v1-5`, so you'll need to visit [its card](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5), read the license and tick the checkbox if you agree.
You have to be a registered user in 🤗 Hugging Face Hub, and you'll also need to use an access token for the code to work. For more information on access tokens, please refer to [this section of the documentation](https://huggingface.co/docs/hub/security-tokens). You have to be a registered user in 🤗 Hugging Face Hub, and you'll also need to use an access token for the code to work. For more information on access tokens, please refer to [this section of the documentation](https://huggingface.co/docs/hub/security-tokens).
...@@ -74,7 +74,7 @@ And launch the training using ...@@ -74,7 +74,7 @@ And launch the training using
**___Note: Change the `resolution` to 768 if you are using the [stable-diffusion-2](https://huggingface.co/stabilityai/stable-diffusion-2) 768x768 model.___** **___Note: Change the `resolution` to 768 if you are using the [stable-diffusion-2](https://huggingface.co/stabilityai/stable-diffusion-2) 768x768 model.___**
```bash ```bash
export MODEL_NAME="runwayml/stable-diffusion-v1-5" export MODEL_NAME="stable-diffusion-v1-5/stable-diffusion-v1-5"
export DATA_DIR="path-to-dir-containing-images" export DATA_DIR="path-to-dir-containing-images"
accelerate launch textual_inversion.py \ accelerate launch textual_inversion.py \
......
...@@ -39,7 +39,7 @@ accelerate config ...@@ -39,7 +39,7 @@ accelerate config
### Cat toy example ### Cat toy example
You need to accept the model license before downloading or using the weights. In this example we'll use model version `v1-5`, so you'll need to visit [its card](https://huggingface.co/runwayml/stable-diffusion-v1-5), read the license and tick the checkbox if you agree. You need to accept the model license before downloading or using the weights. In this example we'll use model version `v1-5`, so you'll need to visit [its card](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5), read the license and tick the checkbox if you agree.
You have to be a registered user in 🤗 Hugging Face Hub, and you'll also need to use an access token for the code to work. For more information on access tokens, please refer to [this section of the documentation](https://huggingface.co/docs/hub/security-tokens). You have to be a registered user in 🤗 Hugging Face Hub, and you'll also need to use an access token for the code to work. For more information on access tokens, please refer to [this section of the documentation](https://huggingface.co/docs/hub/security-tokens).
...@@ -73,7 +73,7 @@ In order to leverage onnxruntime to accelerate training, please use textual_inve ...@@ -73,7 +73,7 @@ In order to leverage onnxruntime to accelerate training, please use textual_inve
The command to train on custom data with onnxruntime: The command to train on custom data with onnxruntime:
```bash ```bash
export MODEL_NAME="runwayml/stable-diffusion-v1-5" export MODEL_NAME="stable-diffusion-v1-5/stable-diffusion-v1-5"
export DATA_DIR="path-to-dir-containing-images" export DATA_DIR="path-to-dir-containing-images"
accelerate launch textual_inversion.py \ accelerate launch textual_inversion.py \
......
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
``` ```
cd examples/research_projects/vae cd examples/research_projects/vae
python vae_roundtrip.py \ python vae_roundtrip.py \
--pretrained_model_name_or_path="runwayml/stable-diffusion-v1-5" \ --pretrained_model_name_or_path="stable-diffusion-v1-5/stable-diffusion-v1-5" \
--subfolder="vae" \ --subfolder="vae" \
--input_image="/path/to/your/input.png" --input_image="/path/to/your/input.png"
``` ```
...@@ -63,7 +63,7 @@ Now we can launch the training using: ...@@ -63,7 +63,7 @@ Now we can launch the training using:
**___Note: Please follow the [README_sdxl.md](./README_sdxl.md) if you are using the [stable-diffusion-xl](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0).___** **___Note: Please follow the [README_sdxl.md](./README_sdxl.md) if you are using the [stable-diffusion-xl](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0).___**
```bash ```bash
export MODEL_NAME="runwayml/stable-diffusion-v1-5" export MODEL_NAME="stable-diffusion-v1-5/stable-diffusion-v1-5"
export DATA_DIR="./cat" export DATA_DIR="./cat"
accelerate launch textual_inversion.py \ accelerate launch textual_inversion.py \
......
...@@ -56,8 +56,8 @@ Diffusion models often consist of multiple independently-trained models or other ...@@ -56,8 +56,8 @@ Diffusion models often consist of multiple independently-trained models or other
Each model has been trained independently on a different task and the scheduler can easily be swapped out and replaced with a different one. Each model has been trained independently on a different task and the scheduler can easily be swapped out and replaced with a different one.
During inference, we however want to be able to easily load all components and use them in inference - even if one component, *e.g.* CLIP's text encoder, originates from a different library, such as [Transformers](https://github.com/huggingface/transformers). To that end, all pipelines provide the following functionality: During inference, we however want to be able to easily load all components and use them in inference - even if one component, *e.g.* CLIP's text encoder, originates from a different library, such as [Transformers](https://github.com/huggingface/transformers). To that end, all pipelines provide the following functionality:
- [`from_pretrained` method](https://github.com/huggingface/diffusers/blob/5cbed8e0d157f65d3ddc2420dfd09f2df630e978/src/diffusers/pipeline_utils.py#L139) that accepts a Hugging Face Hub repository id, *e.g.* [runwayml/stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5) or a path to a local directory, *e.g.* - [`from_pretrained` method](https://github.com/huggingface/diffusers/blob/5cbed8e0d157f65d3ddc2420dfd09f2df630e978/src/diffusers/pipeline_utils.py#L139) that accepts a Hugging Face Hub repository id, *e.g.* [stable-diffusion-v1-5/stable-diffusion-v1-5](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5) or a path to a local directory, *e.g.*
"./stable-diffusion". To correctly retrieve which models and components should be loaded, one has to provide a `model_index.json` file, *e.g.* [runwayml/stable-diffusion-v1-5/model_index.json](https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/model_index.json), which defines all components that should be "./stable-diffusion". To correctly retrieve which models and components should be loaded, one has to provide a `model_index.json` file, *e.g.* [stable-diffusion-v1-5/stable-diffusion-v1-5/model_index.json](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5/blob/main/model_index.json), which defines all components that should be
loaded into the pipelines. More specifically, for each model/component one needs to define the format `<name>: ["<library>", "<class name>"]`. `<name>` is the attribute name given to the loaded instance of `<class name>` which can be found in the library or pipeline folder called `"<library>"`. loaded into the pipelines. More specifically, for each model/component one needs to define the format `<name>: ["<library>", "<class name>"]`. `<name>` is the attribute name given to the loaded instance of `<class name>` which can be found in the library or pipeline folder called `"<library>"`.
- [`save_pretrained`](https://github.com/huggingface/diffusers/blob/5cbed8e0d157f65d3ddc2420dfd09f2df630e978/src/diffusers/pipeline_utils.py#L90) that accepts a local path, *e.g.* `./stable-diffusion` under which all models/components of the pipeline will be saved. For each component/model a folder is created inside the local path that is named after the given attribute name, *e.g.* `./stable_diffusion/unet`. - [`save_pretrained`](https://github.com/huggingface/diffusers/blob/5cbed8e0d157f65d3ddc2420dfd09f2df630e978/src/diffusers/pipeline_utils.py#L90) that accepts a local path, *e.g.* `./stable-diffusion` under which all models/components of the pipeline will be saved. For each component/model a folder is created inside the local path that is named after the given attribute name, *e.g.* `./stable_diffusion/unet`.
In addition, a `model_index.json` file is created at the root of the local path, *e.g.* `./stable_diffusion/model_index.json` so that the complete pipeline can again be instantiated In addition, a `model_index.json` file is created at the root of the local path, *e.g.* `./stable_diffusion/model_index.json` so that the complete pipeline can again be instantiated
...@@ -89,7 +89,7 @@ logic including pre-processing, an unrolled diffusion loop, and post-processing ...@@ -89,7 +89,7 @@ logic including pre-processing, an unrolled diffusion loop, and post-processing
# make sure you're logged in with `huggingface-cli login` # make sure you're logged in with `huggingface-cli login`
from diffusers import StableDiffusionPipeline, LMSDiscreteScheduler from diffusers import StableDiffusionPipeline, LMSDiscreteScheduler
pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") pipe = StableDiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5")
pipe = pipe.to("cuda") pipe = pipe.to("cuda")
prompt = "a photo of an astronaut riding a horse on mars" prompt = "a photo of an astronaut riding a horse on mars"
...@@ -112,7 +112,7 @@ from diffusers import StableDiffusionImg2ImgPipeline ...@@ -112,7 +112,7 @@ from diffusers import StableDiffusionImg2ImgPipeline
# load the pipeline # load the pipeline
device = "cuda" device = "cuda"
pipe = StableDiffusionImg2ImgPipeline.from_pretrained( pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "stable-diffusion-v1-5/stable-diffusion-v1-5",
torch_dtype=torch.float16, torch_dtype=torch.float16,
).to(device) ).to(device)
......
...@@ -13,7 +13,7 @@ The summary of the model is the following: ...@@ -13,7 +13,7 @@ The summary of the model is the following:
- Stable Diffusion has the same architecture as [Latent Diffusion](https://arxiv.org/abs/2112.10752) but uses a frozen CLIP Text Encoder instead of training the text encoder jointly with the diffusion model. - Stable Diffusion has the same architecture as [Latent Diffusion](https://arxiv.org/abs/2112.10752) but uses a frozen CLIP Text Encoder instead of training the text encoder jointly with the diffusion model.
- An in-detail explanation of the Stable Diffusion model can be found under [Stable Diffusion with 🧨 Diffusers](https://huggingface.co/blog/stable_diffusion). - An in-detail explanation of the Stable Diffusion model can be found under [Stable Diffusion with 🧨 Diffusers](https://huggingface.co/blog/stable_diffusion).
- If you don't want to rely on the Hugging Face Hub and having to pass a authentication token, you can - If you don't want to rely on the Hugging Face Hub and having to pass a authentication token, you can
download the weights with `git lfs install; git clone https://huggingface.co/runwayml/stable-diffusion-v1-5` and instead pass the local path to the cloned folder to `from_pretrained` as shown below. download the weights with `git lfs install; git clone https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5` and instead pass the local path to the cloned folder to `from_pretrained` as shown below.
- Stable Diffusion can work with a variety of different samplers as is shown below. - Stable Diffusion can work with a variety of different samplers as is shown below.
## Available Pipelines: ## Available Pipelines:
...@@ -33,14 +33,14 @@ If you want to download the model weights using a single Python line, you need t ...@@ -33,14 +33,14 @@ If you want to download the model weights using a single Python line, you need t
```python ```python
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
pipeline = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") pipeline = DiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5")
``` ```
This however can make it difficult to build applications on top of `diffusers` as you will always have to pass the token around. A potential way to solve this issue is by downloading the weights to a local path `"./stable-diffusion-v1-5"`: This however can make it difficult to build applications on top of `diffusers` as you will always have to pass the token around. A potential way to solve this issue is by downloading the weights to a local path `"./stable-diffusion-v1-5"`:
``` ```
git lfs install git lfs install
git clone https://huggingface.co/runwayml/stable-diffusion-v1-5 git clone https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5
``` ```
and simply passing the local path to `from_pretrained`: and simply passing the local path to `from_pretrained`:
...@@ -57,7 +57,7 @@ pipe = StableDiffusionPipeline.from_pretrained("./stable-diffusion-v1-5") ...@@ -57,7 +57,7 @@ pipe = StableDiffusionPipeline.from_pretrained("./stable-diffusion-v1-5")
# make sure you're logged in with `huggingface-cli login` # make sure you're logged in with `huggingface-cli login`
from diffusers import StableDiffusionPipeline from diffusers import StableDiffusionPipeline
pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") pipe = StableDiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5")
pipe = pipe.to("cuda") pipe = pipe.to("cuda")
prompt = "a photo of an astronaut riding a horse on mars" prompt = "a photo of an astronaut riding a horse on mars"
...@@ -75,7 +75,7 @@ from diffusers import StableDiffusionPipeline, DDIMScheduler ...@@ -75,7 +75,7 @@ from diffusers import StableDiffusionPipeline, DDIMScheduler
scheduler = DDIMScheduler.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="scheduler") scheduler = DDIMScheduler.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="scheduler")
pipe = StableDiffusionPipeline.from_pretrained( pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "stable-diffusion-v1-5/stable-diffusion-v1-5",
scheduler=scheduler, scheduler=scheduler,
).to("cuda") ).to("cuda")
...@@ -94,7 +94,7 @@ from diffusers import StableDiffusionPipeline, LMSDiscreteScheduler ...@@ -94,7 +94,7 @@ from diffusers import StableDiffusionPipeline, LMSDiscreteScheduler
lms = LMSDiscreteScheduler.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="scheduler") lms = LMSDiscreteScheduler.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="scheduler")
pipe = StableDiffusionPipeline.from_pretrained( pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", "stable-diffusion-v1-5/stable-diffusion-v1-5",
scheduler=lms, scheduler=lms,
).to("cuda") ).to("cuda")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment