Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "61697cab912fc2ed40a80ba4fa5054791acfaff1"
Unverified Commit bfd9aa3a authored by Muyang Li's avatar Muyang Li Committed by GitHub
Browse files

Merge pull request #97 from Aprilhuu/flux_tool_comfyui 

[Major] Add Comfy UI workflow for Flux.1 tools
parents 51650a36 6ccdd388
Hide whitespace changes
Inline Side-by-side
Showing with 2350 additions and 24 deletions
comfyui/README.md
View file @ bfd9aa3a
......@@ -4,13 +4,21 @@
## Installation
1. Install `nunchaku` following [README.md](https://github.com/mit-han-lab/nunchaku?tab=readme-ov-file#installation).
2. Set up the dependencies for [ComfyUI](https://github.com/comfyanonymous/ComfyUI/tree/master) with the following commands:
2. Install dependencies needed to run custom ComfyUI nodes:
```shell
pip install git+https://github.com/asomoza/image_gen_aux.git
```
3. Set up the dependencies for [ComfyUI](https://github.com/comfyanonymous/ComfyUI/tree/master) with the following commands:
```shell
git clone https://github.com/comfyanonymous/ComfyUI.git
cd ComfyUI
pip install -r requirements.txt
```
4. Install [ComfyUI-Manager](https://github.com/ltdrdata/ComfyUI-Manager) with the following commands then restart ComfyUI:
```shell
cd ComfyUI/custom_nodes
git clone https://github.com/ltdrdata/ComfyUI-Manager comfyui-manager
```
## Usage
......@@ -34,6 +42,7 @@ pip install -r requirements.txt
cd custom_nodes
ln -s ../../nunchaku/comfyui svdquant
```
* Install missing nodes (e.g., comfyui-inpainteasy) following [this tutorial](https://github.com/ltdrdata/ComfyUI-Manager?tab=readme-ov-file#support-of-missing-nodes-installation).
2. **Download Required Models**: Follow [this tutorial](https://comfyanonymous.github.io/ComfyUI_examples/flux/) and download the required models into the appropriate directories using the commands below:
......@@ -50,13 +59,13 @@ pip install -r requirements.txt
python main.py
```
4. **Select the SVDQuant Workflow**: Choose one of the SVDQuant workflows (`flux.1-dev-svdquant.json` or `flux.1-schnell-svdquant.json`) to get started.
4. **Select the SVDQuant Workflow**: Choose one of the SVDQuant workflows (`flux.1-dev-svdquant.json`, `flux.1-schnell-svdquant.json`, `flux.1-depth-svdquant.json`, `flux.1-canny-svdquant.json` or `flux.1-fill-svdquant.json`) to get started. For the flux.1 fill workflow, you can use the built-in MaskEditor tool to add mask on top of an image.
## SVDQuant Nodes
* **SVDQuant Flux DiT Loader**: A node for loading the FLUX diffusion model.
* `model_path`: Specifies the model location. If set to `mit-han-lab/svdq-int4-flux.1-schnell` or `mit-han-lab/svdq-int4-flux.1-dev`, the model will be automatically downloaded from our Hugging Face repository. Alternatively, you can manually download the model directory by running the following command:
* `model_path`: Specifies the model location. If set to `mit-han-lab/svdq-int4-flux.1-schnell`, `mit-han-lab/svdq-int4-flux.1-dev`, `mit-han-lab/svdq-int4-flux.1-canny-dev`, `mit-han-lab/svdq-int4-flux.1-fill-dev` or `mit-han-lab/svdq-int4-flux.1-depth-dev`, the model will be automatically downloaded from our Hugging Face repository. Alternatively, you can manually download the model directory by running the following command example:
```shell
huggingface-cli download mit-han-lab/svdq-int4-flux.1-dev --local-dir models/diffusion_models/svdq-int4-flux.1-dev
......
comfyui/nodes.py
View file @ bfd9aa3a
......@@ -7,16 +7,17 @@ import comfy.sd
import folder_paths
import GPUtil
import torch
import numpy as np
from comfy.ldm.common_dit import pad_to_patch_size
from comfy.supported_models import Flux, FluxSchnell
from diffusers import FluxTransformer2DModel
from einops import rearrange, repeat
from torch import nn
from transformers import T5EncoderModel
from image_gen_aux import DepthPreprocessor
from nunchaku.models.transformer_flux import NunchakuFluxTransformer2dModel
class ComfyUIFluxForwardWrapper(nn.Module):
def __init__(self, model: NunchakuFluxTransformer2dModel, config):
super(ComfyUIFluxForwardWrapper, self).__init__()
......@@ -24,13 +25,25 @@ class ComfyUIFluxForwardWrapper(nn.Module):
self.dtype = next(model.parameters()).dtype
self.config = config
def forward(self, x, timestep, context, y, guidance, control=None, transformer_options={}, **kwargs):
def forward(
self,
x,
timestep,
context,
y,
guidance,
control=None,
transformer_options={},
**kwargs,
):
assert control is None # for now
bs, c, h, w = x.shape
patch_size = self.config["patch_size"]
x = pad_to_patch_size(x, (patch_size, patch_size))
img = rearrange(x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch_size, pw=patch_size)
img = rearrange(
x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch_size, pw=patch_size
)
h_len = (h + (patch_size // 2)) // patch_size
w_len = (w + (patch_size // 2)) // patch_size
......@@ -54,21 +67,30 @@ class ComfyUIFluxForwardWrapper(nn.Module):
guidance=guidance if self.config["guidance_embed"] else None,
).sample
out = rearrange(out, "b (h w) (c ph pw) -> b c (h ph) (w pw)", h=h_len, w=w_len, ph=2, pw=2)[:, :, :h, :w]
out = rearrange(
out, "b (h w) (c ph pw) -> b c (h ph) (w pw)", h=h_len, w=w_len, ph=2, pw=2
)[:, :, :h, :w]
return out
class SVDQuantFluxDiTLoader:
@classmethod
def INPUT_TYPES(s):
model_paths = ["mit-han-lab/svdq-int4-flux.1-schnell", "mit-han-lab/svdq-int4-flux.1-dev"]
model_paths = [
"mit-han-lab/svdq-int4-flux.1-schnell",
"mit-han-lab/svdq-int4-flux.1-dev",
"mit-han-lab/svdq-int4-flux.1-canny-dev",
"mit-han-lab/svdq-int4-flux.1-depth-dev",
"mit-han-lab/svdq-int4-flux.1-fill-dev",
]
prefix = "models/diffusion_models"
local_folders = os.listdir(prefix)
local_folders = sorted(
[
folder
for folder in local_folders
if not folder.startswith(".") and os.path.isdir(os.path.join(prefix, folder))
if not folder.startswith(".")
and os.path.isdir(os.path.join(prefix, folder))
]
)
model_paths.extend(local_folders)
......@@ -78,7 +100,14 @@ class SVDQuantFluxDiTLoader:
"model_path": (model_paths,),
"device_id": (
"INT",
{"default": 0, "min": 0, "max": ngpus, "step": 1, "display": "number", "lazy": True},
{
"default": 0,
"min": 0,
"max": ngpus,
"step": 1,
"display": "number",
"lazy": True,
},
),
}
}
......@@ -88,17 +117,20 @@ class SVDQuantFluxDiTLoader:
CATEGORY = "SVDQuant"
TITLE = "SVDQuant Flux DiT Loader"
def load_model(self, model_path: str, device_id: int, **kwargs) -> tuple[FluxTransformer2DModel]:
def load_model(
self, model_path: str, device_id: int, **kwargs
) -> tuple[FluxTransformer2DModel]:
device = f"cuda:{device_id}"
prefix = "models/diffusion_models"
if os.path.exists(os.path.join(prefix, model_path)):
model_path = os.path.join(prefix, model_path)
else:
model_path = model_path
transformer = NunchakuFluxTransformer2dModel.from_pretrained(model_path).to(device)
transformer = NunchakuFluxTransformer2dModel.from_pretrained(model_path).to(
device
)
dit_config = {
"image_model": "flux",
"in_channels": 16,
"patch_size": 2,
"out_channels": 16,
"vec_in_dim": 768,
......@@ -111,21 +143,34 @@ class SVDQuantFluxDiTLoader:
"axes_dim": [16, 56, 56],
"theta": 10000,
"qkv_bias": True,
"guidance_embed": True,
"disable_unet_model_creation": True,
}
if "schnell" in model_path:
dit_config["guidance_embed"] = False
dit_config["in_channels"] = 16
model_config = FluxSchnell(dit_config)
elif "canny" in model_path or "depth" in model_path:
dit_config["in_channels"] = 32
model_config = Flux(dit_config)
elif "fill" in model_path:
dit_config["in_channels"] = 64
model_config = Flux(dit_config)
else:
assert "dev" in model_path
dit_config["guidance_embed"] = True
assert (
model_path == "mit-han-lab/svdq-int4-flux.1-dev"
), f"model {model_path} not supported"
dit_config["in_channels"] = 16
model_config = Flux(dit_config)
model_config.set_inference_dtype(torch.bfloat16, None)
model_config.custom_operations = None
model = model_config.get_model({})
model.diffusion_model = ComfyUIFluxForwardWrapper(transformer, config=dit_config)
model.diffusion_model = ComfyUIFluxForwardWrapper(
transformer, config=dit_config
)
model = comfy.model_patcher.ModelPatcher(model, device, device_id)
return (model,)
......@@ -157,7 +202,8 @@ class SVDQuantTextEncoderLoader:
[
folder
for folder in local_folders
if not folder.startswith(".") and os.path.isdir(os.path.join(prefix, folder))
if not folder.startswith(".")
and os.path.isdir(os.path.join(prefix, folder))
]
)
model_paths.extend(local_folders)
......@@ -168,7 +214,14 @@ class SVDQuantTextEncoderLoader:
"text_encoder2": (folder_paths.get_filename_list("text_encoders"),),
"t5_min_length": (
"INT",
{"default": 512, "min": 256, "max": 1024, "step": 128, "display": "number", "lazy": True},
{
"default": 512,
"min": 256,
"max": 1024,
"step": 128,
"display": "number",
"lazy": True,
},
),
"t5_precision": (["BF16", "INT4"],),
"int4_model": (model_paths, {"tooltip": "The name of the INT4 model."}),
......@@ -191,8 +244,12 @@ class SVDQuantTextEncoderLoader:
t5_precision: str,
int4_model: str,
):
text_encoder_path1 = folder_paths.get_full_path_or_raise("text_encoders", text_encoder1)
text_encoder_path2 = folder_paths.get_full_path_or_raise("text_encoders", text_encoder2)
text_encoder_path1 = folder_paths.get_full_path_or_raise(
"text_encoders", text_encoder1
)
text_encoder_path2 = folder_paths.get_full_path_or_raise(
"text_encoders", text_encoder2
)
if model_type == "flux":
clip_type = comfy.sd.CLIPType.FLUX
else:
......@@ -223,7 +280,9 @@ class SVDQuantTextEncoderLoader:
transformer = NunchakuT5EncoderModel.from_pretrained(model_path)
transformer.forward = types.MethodType(svdquant_t5_forward, transformer)
clip.cond_stage_model.t5xxl.transformer = (
transformer.to(device=device, dtype=dtype) if device.type == "cuda" else transformer
transformer.to(device=device, dtype=dtype)
if device.type == "cuda"
else transformer
)
return (clip,)
......@@ -239,11 +298,17 @@ class SVDQuantLoraLoader:
lora_name_list = [
"None",
*folder_paths.get_filename_list("loras"),
*[f"mit-han-lab/svdquant-models/svdq-flux.1-dev-lora-{n}.safetensors" for n in hf_lora_names],
*[
f"mit-han-lab/svdquant-models/svdq-flux.1-dev-lora-{n}.safetensors"
for n in hf_lora_names
],
]
return {
"required": {
"model": ("MODEL", {"tooltip": "The diffusion model the LoRA will be applied to."}),
"model": (
"MODEL",
{"tooltip": "The diffusion model the LoRA will be applied to."},
),
"lora_name": (lora_name_list, {"tooltip": "The name of the LoRA."}),
"lora_strength": (
"FLOAT",
......@@ -292,8 +357,50 @@ class SVDQuantLoraLoader:
return (model,)
class DepthPreprocesser:
@classmethod
def INPUT_TYPES(s):
model_paths = ["LiheYoung/depth-anything-large-hf"]
prefix = "models/checkpoints"
local_folders = os.listdir(prefix)
local_folders = sorted(
[
folder
for folder in local_folders
if not folder.startswith(".")
and os.path.isdir(os.path.join(prefix, folder))
]
)
model_paths.extend(local_folders)
return {
"required": {
"image": ("IMAGE", {}),
"model_path": (
model_paths,
{"tooltip": "Name of the depth preprocesser model."},
),
}
}
RETURN_TYPES = ("IMAGE",)
FUNCTION = "depth_preprocess"
CATEGORY = "Flux.1"
TITLE = "Flux.1 Depth Preprocessor"
def depth_preprocess(self, image, model_path):
prefix = "models/checkpoints"
if os.path.exists(os.path.join(prefix, model_path)):
model_path = os.path.join(prefix, model_path)
processor = DepthPreprocessor.from_pretrained(model_path)
np_image = np.asarray(image)
np_result = np.array(processor(np_image)[0].convert("RGB"))
out_tensor = torch.from_numpy(np_result.astype(np.float32) / 255.0).unsqueeze(0)
return (out_tensor,)
NODE_CLASS_MAPPINGS = {
"SVDQuantFluxDiTLoader": SVDQuantFluxDiTLoader,
"SVDQuantTextEncoderLoader": SVDQuantTextEncoderLoader,
"SVDQuantLoRALoader": SVDQuantLoraLoader,
"DepthPreprocesser": DepthPreprocesser
}
comfyui/workflows/flux.1-canny-svdquant.json 0 → 100644
View file @ bfd9aa3a
{
"last_node_id": 38,
"last_link_id": 76,
"nodes": [
{
"id": 3,
"type": "KSampler",
"pos": [
1290,
40
],
"size": [
315,
262
],
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 71,
"label": "model"
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 64,
"label": "positive"
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 65,
"label": "negative"
},
{
"name": "latent_image",
"type": "LATENT",
"link": 66,
"label": "latent_image"
}
],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
7
],
"slot_index": 0,
"label": "LATENT"
}
],
"properties": {
"Node name for S&R": "KSampler"
},
"widgets_values": [
646386750200194,
"randomize",
20,
1,
"euler",
"normal",
1
]
},
{
"id": 35,
"type": "InstructPixToPixConditioning",
"pos": [
1040,
50
],
"size": [
235.1999969482422,
86
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "positive",
"type": "CONDITIONING",
"link": 67,
"label": "positive"
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 68,
"label": "negative"
},
{
"name": "vae",
"type": "VAE",
"link": 69,
"label": "vae"
},
{
"name": "pixels",
"type": "IMAGE",
"link": 70,
"label": "pixels"
}
],
"outputs": [
{
"name": "positive",
"type": "CONDITIONING",
"links": [
64
],
"slot_index": 0,
"label": "positive"
},
{
"name": "negative",
"type": "CONDITIONING",
"links": [
65
],
"slot_index": 1,
"label": "negative"
},
{
"name": "latent",
"type": "LATENT",
"links": [
66
],
"slot_index": 2,
"label": "latent"
}
],
"properties": {
"Node name for S&R": "InstructPixToPixConditioning"
},
"widgets_values": []
},
{
"id": 8,
"type": "VAEDecode",
"pos": [
1620,
40
],
"size": [
210,
46
],
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 7,
"label": "samples"
},
{
"name": "vae",
"type": "VAE",
"link": 60,
"label": "vae"
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
9
],
"slot_index": 0,
"label": "IMAGE"
}
],
"properties": {
"Node name for S&R": "VAEDecode"
},
"widgets_values": []
},
{
"id": 9,
"type": "SaveImage",
"pos": [
1850,
40
],
"size": [
828.9535522460938,
893.8475341796875
],
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 9,
"label": "images"
}
],
"outputs": [],
"properties": {},
"widgets_values": [
"ComfyUI"
]
},
{
"id": 32,
"type": "VAELoader",
"pos": [
1290,
350
],
"size": [
315,
58
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "VAE",
"type": "VAE",
"links": [
60,
69
],
"slot_index": 0,
"label": "VAE"
}
],
"properties": {
"Node name for S&R": "VAELoader"
},
"widgets_values": [
"ae.safetensors"
]
},
{
"id": 26,
"type": "FluxGuidance",
"pos": [
700,
50
],
"size": [
317.4000244140625,
58
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "conditioning",
"type": "CONDITIONING",
"link": 41,
"label": "conditioning"
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
67
],
"slot_index": 0,
"shape": 3,
"label": "CONDITIONING"
}
],
"properties": {
"Node name for S&R": "FluxGuidance"
},
"widgets_values": [
30
]
},
{
"id": 34,
"type": "DualCLIPLoader",
"pos": [
-80,
110
],
"size": [
315,
106
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"links": [
62,
63
],
"label": "CLIP"
}
],
"properties": {
"Node name for S&R": "DualCLIPLoader"
},
"widgets_values": [
"clip_l.safetensors",
"t5xxl_fp16.safetensors",
"flux",
"default"
]
},
{
"id": 7,
"type": "CLIPTextEncode",
"pos": [
307,
282
],
"size": [
425.27801513671875,
180.6060791015625
],
"flags": {
"collapsed": true
},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 63,
"label": "clip"
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
68
],
"slot_index": 0,
"label": "CONDITIONING"
}
],
"title": "CLIP Text Encode (Negative Prompt)",
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
""
],
"color": "#322",
"bgcolor": "#533"
},
{
"id": 23,
"type": "CLIPTextEncode",
"pos": [
260,
50
],
"size": [
422.84503173828125,
164.31304931640625
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 62,
"label": "clip"
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
41
],
"slot_index": 0,
"label": "CONDITIONING"
}
],
"title": "CLIP Text Encode (Positive Prompt)",
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"A robot made of exotic candies and chocolates of different kinds. The background is filled with confetti and celebratory gifts."
],
"color": "#232",
"bgcolor": "#353"
},
{
"id": 19,
"type": "PreviewImage",
"pos": [
1127.9403076171875,
554.3356323242188
],
"size": [
571.5869140625,
625.5296020507812
],
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 26,
"label": "images"
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 18,
"type": "Canny",
"pos": [
744.2684936523438,
566.853515625
],
"size": [
315,
82
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 76,
"label": "image"
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
26,
70
],
"slot_index": 0,
"shape": 3,
"label": "IMAGE"
}
],
"properties": {
"Node name for S&R": "Canny"
},
"widgets_values": [
0.15,
0.3
]
},
{
"id": 36,
"type": "SVDQuantFluxDiTLoader",
"pos": [
865.4989624023438,
-95.86973571777344
],
"size": [
315,
82
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
71
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "SVDQuantFluxDiTLoader"
},
"widgets_values": [
"mit-han-lab/svdq-int4-flux.1-canny-dev",
0
]
},
{
"id": 17,
"type": "LoadImage",
"pos": [
6.694743633270264,
562.3865966796875
],
"size": [
315,
314.0000305175781
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
75
],
"slot_index": 0,
"shape": 3,
"label": "IMAGE"
},
{
"name": "MASK",
"type": "MASK",
"links": null,
"shape": 3,
"label": "MASK"
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"robot.png",
"image"
]
},
{
"id": 38,
"type": "ImageScale",
"pos": [
379.69903564453125,
565.2651977539062
],
"size": [
315,
130
],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 75
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
76
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "ImageScale"
},
"widgets_values": [
"nearest-exact",
1024,
1024,
"center"
]
}
],
"links": [
[
7,
3,
0,
8,
0,
"LATENT"
],
[
9,
8,
0,
9,
0,
"IMAGE"
],
[
26,
18,
0,
19,
0,
"IMAGE"
],
[
41,
23,
0,
26,
0,
"CONDITIONING"
],
[
60,
32,
0,
8,
1,
"VAE"
],
[
62,
34,
0,
23,
0,
"CLIP"
],
[
63,
34,
0,
7,
0,
"CLIP"
],
[
64,
35,
0,
3,
1,
"CONDITIONING"
],
[
65,
35,
1,
3,
2,
"CONDITIONING"
],
[
66,
35,
2,
3,
3,
"LATENT"
],
[
67,
26,
0,
35,
0,
"CONDITIONING"
],
[
68,
7,
0,
35,
1,
"CONDITIONING"
],
[
69,
32,
0,
35,
2,
"VAE"
],
[
70,
18,
0,
35,
3,
"IMAGE"
],
[
71,
36,
0,
3,
0,
"MODEL"
],
[
75,
17,
0,
38,
0,
"IMAGE"
],
[
76,
38,
0,
18,
0,
"IMAGE"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.895430243255241,
"offset": [
203.26101803057463,
215.36536277004458
]
}
},
"version": 0.4
}
\ No newline at end of file
comfyui/workflows/flux.1-depth-svdquant.json 0 → 100644
View file @ bfd9aa3a
{
"last_node_id": 44,
"last_link_id": 85,
"nodes": [
{
"id": 7,
"type": "CLIPTextEncode",
"pos": [
307,
282
],
"size": [
425.27801513671875,
180.6060791015625
],
"flags": {
"collapsed": true
},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 63,
"label": "clip"
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
68
],
"slot_index": 0,
"label": "CONDITIONING"
}
],
"title": "CLIP Text Encode (Negative Prompt)",
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
""
],
"color": "#322",
"bgcolor": "#533"
},
{
"id": 34,
"type": "DualCLIPLoader",
"pos": [
-238,
112
],
"size": [
315,
106
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"links": [
62,
63
],
"label": "CLIP"
}
],
"properties": {
"Node name for S&R": "DualCLIPLoader"
},
"widgets_values": [
"clip_l.safetensors",
"t5xxl_fp16.safetensors",
"flux",
"default"
]
},
{
"id": 26,
"type": "FluxGuidance",
"pos": [
621,
8
],
"size": [
317.4000244140625,
58
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "conditioning",
"type": "CONDITIONING",
"link": 41,
"label": "conditioning"
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
67
],
"slot_index": 0,
"shape": 3,
"label": "CONDITIONING"
}
],
"properties": {
"Node name for S&R": "FluxGuidance"
},
"widgets_values": [
10
]
},
{
"id": 32,
"type": "VAELoader",
"pos": [
656,
165
],
"size": [
315,
58
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "VAE",
"type": "VAE",
"links": [
60,
69
],
"slot_index": 0,
"label": "VAE"
}
],
"properties": {
"Node name for S&R": "VAELoader"
},
"widgets_values": [
"ae.safetensors"
]
},
{
"id": 3,
"type": "KSampler",
"pos": [
1280,
100
],
"size": [
315,
262
],
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 78,
"label": "model"
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 64,
"label": "positive"
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 65,
"label": "negative"
},
{
"name": "latent_image",
"type": "LATENT",
"link": 73,
"label": "latent_image"
}
],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
7
],
"slot_index": 0,
"label": "LATENT"
}
],
"properties": {
"Node name for S&R": "KSampler"
},
"widgets_values": [
718322679777603,
"randomize",
20,
1,
"euler",
"normal",
1
]
},
{
"id": 35,
"type": "InstructPixToPixConditioning",
"pos": [
1008,
118
],
"size": [
235.1999969482422,
86
],
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "positive",
"type": "CONDITIONING",
"link": 67,
"label": "positive"
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 68,
"label": "negative"
},
{
"name": "vae",
"type": "VAE",
"link": 69,
"label": "vae"
},
{
"name": "pixels",
"type": "IMAGE",
"link": 80,
"label": "pixels"
}
],
"outputs": [
{
"name": "positive",
"type": "CONDITIONING",
"links": [
64
],
"slot_index": 0,
"label": "positive"
},
{
"name": "negative",
"type": "CONDITIONING",
"links": [
65
],
"slot_index": 1,
"label": "negative"
},
{
"name": "latent",
"type": "LATENT",
"links": [
73
],
"slot_index": 2,
"label": "latent"
}
],
"properties": {
"Node name for S&R": "InstructPixToPixConditioning"
},
"widgets_values": []
},
{
"id": 39,
"type": "SVDQuantFluxDiTLoader",
"pos": [
707.80908203125,
-172.0343017578125
],
"size": [
315,
82
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
78
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "SVDQuantFluxDiTLoader"
},
"widgets_values": [
"mit-han-lab/svdq-int4-flux.1-depth-dev",
0
]
},
{
"id": 42,
"type": "ImageScale",
"pos": [
378.3890686035156,
472.7001953125
],
"size": [
315,
130
],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 82
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
83
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "ImageScale"
},
"widgets_values": [
"nearest-exact",
1024,
1024,
"center"
]
},
{
"id": 17,
"type": "LoadImage",
"pos": [
30.604948043823242,
419.3930358886719
],
"size": [
315,
314.0000305175781
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
82
],
"slot_index": 0,
"shape": 3,
"label": "IMAGE"
},
{
"name": "MASK",
"type": "MASK",
"links": null,
"shape": 3,
"label": "MASK"
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"robot.png",
"image"
]
},
{
"id": 23,
"type": "CLIPTextEncode",
"pos": [
115,
-17
],
"size": [
422.84503173828125,
164.31304931640625
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 62,
"label": "clip"
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
41
],
"slot_index": 0,
"label": "CONDITIONING"
}
],
"title": "CLIP Text Encode (Positive Prompt)",
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"A robot made of exotic candies and chocolates of different kinds. The background is filled with confetti and celebratory gifts."
],
"color": "#232",
"bgcolor": "#353"
},
{
"id": 43,
"type": "PreviewImage",
"pos": [
1001.3873291015625,
432.09039306640625
],
"size": [
571.5869140625,
625.5296020507812
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 84,
"label": "images"
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 40,
"type": "DepthPreprocesser",
"pos": [
639.0159301757812,
350.06134033203125
],
"size": [
315,
58
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 83
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
80,
84
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "DepthPreprocesser"
},
"widgets_values": [
"LiheYoung/depth-anything-large-hf"
]
},
{
"id": 8,
"type": "VAEDecode",
"pos": [
1620,
98
],
"size": [
210,
46
],
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 7,
"label": "samples"
},
{
"name": "vae",
"type": "VAE",
"link": 60,
"label": "vae"
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
85
],
"slot_index": 0,
"label": "IMAGE"
}
],
"properties": {
"Node name for S&R": "VAEDecode"
},
"widgets_values": []
},
{
"id": 44,
"type": "SaveImage",
"pos": [
1912.7984619140625,
109.0069580078125
],
"size": [
828.9535522460938,
893.8475341796875
],
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 85,
"label": "images"
}
],
"outputs": [],
"properties": {},
"widgets_values": [
"ComfyUI"
]
}
],
"links": [
[
7,
3,
0,
8,
0,
"LATENT"
],
[
41,
23,
0,
26,
0,
"CONDITIONING"
],
[
60,
32,
0,
8,
1,
"VAE"
],
[
62,
34,
0,
23,
0,
"CLIP"
],
[
63,
34,
0,
7,
0,
"CLIP"
],
[
64,
35,
0,
3,
1,
"CONDITIONING"
],
[
65,
35,
1,
3,
2,
"CONDITIONING"
],
[
67,
26,
0,
35,
0,
"CONDITIONING"
],
[
68,
7,
0,
35,
1,
"CONDITIONING"
],
[
69,
32,
0,
35,
2,
"VAE"
],
[
73,
35,
2,
3,
3,
"LATENT"
],
[
78,
39,
0,
3,
0,
"MODEL"
],
[
80,
40,
0,
35,
3,
"IMAGE"
],
[
82,
17,
0,
42,
0,
"IMAGE"
],
[
83,
42,
0,
40,
0,
"IMAGE"
],
[
84,
40,
0,
43,
0,
"IMAGE"
],
[
85,
8,
0,
44,
0,
"IMAGE"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.6115909044841502,
"offset": [
724.4911189218763,
518.3043483917891
]
}
},
"version": 0.4
}
\ No newline at end of file
comfyui/workflows/flux.1-fill-svdquant.json 0 → 100644
View file @ bfd9aa3a
{
"last_node_id": 58,
"last_link_id": 108,
"nodes": [
{
"id": 8,
"type": "VAEDecode",
"pos": [
1620,
98
],
"size": [
210,
46
],
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 7
},
{
"name": "vae",
"type": "VAE",
"link": 60
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
95
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAEDecode"
},
"widgets_values": []
},
{
"id": 38,
"type": "InpaintModelConditioning",
"pos": [
952,
78
],
"size": [
302.4000244140625,
138
],
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "positive",
"type": "CONDITIONING",
"link": 80
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 81
},
{
"name": "vae",
"type": "VAE",
"link": 82
},
{
"name": "pixels",
"type": "IMAGE",
"link": 107
},
{
"name": "mask",
"type": "MASK",
"link": 108
}
],
"outputs": [
{
"name": "positive",
"type": "CONDITIONING",
"links": [
77
],
"slot_index": 0
},
{
"name": "negative",
"type": "CONDITIONING",
"links": [
78
],
"slot_index": 1
},
{
"name": "latent",
"type": "LATENT",
"links": [
88
],
"slot_index": 2
}
],
"properties": {
"Node name for S&R": "InpaintModelConditioning"
},
"widgets_values": [
false
]
},
{
"id": 3,
"type": "KSampler",
"pos": [
1280,
100
],
"size": [
315,
262
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 102
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 77
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 78
},
{
"name": "latent_image",
"type": "LATENT",
"link": 88
}
],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
7
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "KSampler"
},
"widgets_values": [
1078304127779394,
"randomize",
20,
1,
"euler",
"normal",
1
]
},
{
"id": 9,
"type": "SaveImage",
"pos": [
1879,
90
],
"size": [
828.9535522460938,
893.8475341796875
],
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 95
}
],
"outputs": [],
"properties": {
"Node name for S&R": "SaveImage"
},
"widgets_values": [
"ComfyUI"
]
},
{
"id": 26,
"type": "FluxGuidance",
"pos": [
596,
48
],
"size": [
317.4000244140625,
58
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "conditioning",
"type": "CONDITIONING",
"link": 41
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
80
],
"slot_index": 0,
"shape": 3
}
],
"properties": {
"Node name for S&R": "FluxGuidance"
},
"widgets_values": [
30
]
},
{
"id": 7,
"type": "CLIPTextEncode",
"pos": [
165,
267
],
"size": [
425.27801513671875,
180.6060791015625
],
"flags": {
"collapsed": true
},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 63
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
81
],
"slot_index": 0
}
],
"title": "CLIP Text Encode (Negative Prompt)",
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
""
],
"color": "#322",
"bgcolor": "#533"
},
{
"id": 34,
"type": "DualCLIPLoader",
"pos": [
-237,
76
],
"size": [
315,
106
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"links": [
62,
63
]
}
],
"properties": {
"Node name for S&R": "DualCLIPLoader"
},
"widgets_values": [
"clip_l.safetensors",
"t5xxl_fp16.safetensors",
"flux",
"default"
]
},
{
"id": 32,
"type": "VAELoader",
"pos": [
1303,
424
],
"size": [
315,
58
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "VAE",
"type": "VAE",
"links": [
60,
82
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAELoader"
},
"widgets_values": [
"ae.safetensors"
]
},
{
"id": 45,
"type": "SVDQuantFluxDiTLoader",
"pos": [
936.3029174804688,
-113.06819915771484
],
"size": [
315,
82
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
102
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "SVDQuantFluxDiTLoader"
},
"widgets_values": [
"mit-han-lab/svdq-int4-flux.1-fill-dev",
0
]
},
{
"id": 48,
"type": "Note",
"pos": [
466.9884033203125,
643.9080810546875
],
"size": [
314.99755859375,
117.98363494873047
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [],
"properties": {
"text": ""
},
"widgets_values": [
"To add mask for fill inpainting, right click on the uploaded image and select \"Open in MaskEditor\". Use the brush tool to add masking and click save to continue."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 17,
"type": "LoadImage",
"pos": [
126.66505432128906,
460.53631591796875
],
"size": [
315,
314.0000305175781
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
105
],
"slot_index": 0,
"shape": 3
},
{
"name": "MASK",
"type": "MASK",
"links": [
106
],
"slot_index": 1,
"shape": 3
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"clipspace/clipspace-mask-123191.png [input]",
"image"
]
},
{
"id": 58,
"type": "ImageAndMaskResizeNode",
"pos": [
536.786865234375,
328.54388427734375
],
"size": [
315,
174
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 105
},
{
"name": "mask",
"type": "MASK",
"link": 106
}
],
"outputs": [
{
"name": "image",
"type": "IMAGE",
"links": [
107
],
"slot_index": 0
},
{
"name": "mask",
"type": "MASK",
"links": [
108
],
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "ImageAndMaskResizeNode"
},
"widgets_values": [
1024,
1024,
"nearest-exact",
"center",
10
]
},
{
"id": 23,
"type": "CLIPTextEncode",
"pos": [
144,
-7
],
"size": [
422.84503173828125,
164.31304931640625
],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 62
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
41
],
"slot_index": 0
}
],
"title": "CLIP Text Encode (Positive Prompt)",
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"A robot with a closed eye pink face is giving a presentation"
],
"color": "#232",
"bgcolor": "#353"
}
],
"links": [
[
7,
3,
0,
8,
0,
"LATENT"
],
[
41,
23,
0,
26,
0,
"CONDITIONING"
],
[
60,
32,
0,
8,
1,
"VAE"
],
[
62,
34,
0,
23,
0,
"CLIP"
],
[
63,
34,
0,
7,
0,
"CLIP"
],
[
77,
38,
0,
3,
1,
"CONDITIONING"
],
[
78,
38,
1,
3,
2,
"CONDITIONING"
],
[
80,
26,
0,
38,
0,
"CONDITIONING"
],
[
81,
7,
0,
38,
1,
"CONDITIONING"
],
[
82,
32,
0,
38,
2,
"VAE"
],
[
88,
38,
2,
3,
3,
"LATENT"
],
[
95,
8,
0,
9,
0,
"IMAGE"
],
[
102,
45,
0,
3,
0,
"MODEL"
],
[
105,
17,
0,
58,
0,
"IMAGE"
],
[
106,
17,
1,
58,
1,
"MASK"
],
[
107,
58,
0,
38,
3,
"IMAGE"
],
[
108,
58,
1,
38,
4,
"MASK"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.8390545288824038,
"offset": [
361.8437326514503,
242.3368651567008
]
},
"node_versions": {
"comfy-core": "0.3.14",
"comfyui-inpainteasy": "1.0.2"
}
},
"version": 0.4
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment