Skip to content

GitLab

Unverified Commit bfd9aa3a authored by Muyang Li's avatar Muyang Li Committed by GitHub
Browse files

Merge pull request #97 from Aprilhuu/flux_tool_comfyui 

[Major] Add Comfy UI workflow for Flux.1 tools
parents 51650a36 6ccdd388
Hide whitespace changes
Inline Side-by-side
Showing with 2350 additions and 24 deletions
comfyui/README.md
View file @ bfd9aa3a
...@@ -4,13 +4,21 @@ ...@@ -4,13 +4,21 @@
## Installation ## Installation
1. Install `nunchaku` following [README.md](https://github.com/mit-han-lab/nunchaku?tab=readme-ov-file#installation). 1. Install `nunchaku` following [README.md](https://github.com/mit-han-lab/nunchaku?tab=readme-ov-file#installation).
2. Set up the dependencies for [ComfyUI](https://github.com/comfyanonymous/ComfyUI/tree/master) with the following commands: 2. Install dependencies needed to run custom ComfyUI nodes:
```shell
pip install git+https://github.com/asomoza/image_gen_aux.git
```
3. Set up the dependencies for [ComfyUI](https://github.com/comfyanonymous/ComfyUI/tree/master) with the following commands:
```shell ```shell
git clone https://github.com/comfyanonymous/ComfyUI.git git clone https://github.com/comfyanonymous/ComfyUI.git
cd ComfyUI cd ComfyUI
pip install -r requirements.txt pip install -r requirements.txt
``` ```
4. Install [ComfyUI-Manager](https://github.com/ltdrdata/ComfyUI-Manager) with the following commands then restart ComfyUI:
```shell
cd ComfyUI/custom_nodes
git clone https://github.com/ltdrdata/ComfyUI-Manager comfyui-manager
```
## Usage ## Usage
...@@ -34,6 +42,7 @@ pip install -r requirements.txt ...@@ -34,6 +42,7 @@ pip install -r requirements.txt
cd custom_nodes cd custom_nodes
ln -s ../../nunchaku/comfyui svdquant ln -s ../../nunchaku/comfyui svdquant
``` ```
* Install missing nodes (e.g., comfyui-inpainteasy) following [this tutorial](https://github.com/ltdrdata/ComfyUI-Manager?tab=readme-ov-file#support-of-missing-nodes-installation).
2. **Download Required Models**: Follow [this tutorial](https://comfyanonymous.github.io/ComfyUI_examples/flux/) and download the required models into the appropriate directories using the commands below: 2. **Download Required Models**: Follow [this tutorial](https://comfyanonymous.github.io/ComfyUI_examples/flux/) and download the required models into the appropriate directories using the commands below:
...@@ -50,13 +59,13 @@ pip install -r requirements.txt ...@@ -50,13 +59,13 @@ pip install -r requirements.txt
python main.py python main.py
``` ```
4. **Select the SVDQuant Workflow**: Choose one of the SVDQuant workflows (`flux.1-dev-svdquant.json` or `flux.1-schnell-svdquant.json`) to get started. 4. **Select the SVDQuant Workflow**: Choose one of the SVDQuant workflows (`flux.1-dev-svdquant.json`, `flux.1-schnell-svdquant.json`, `flux.1-depth-svdquant.json`, `flux.1-canny-svdquant.json` or `flux.1-fill-svdquant.json`) to get started. For the flux.1 fill workflow, you can use the built-in MaskEditor tool to add mask on top of an image.
## SVDQuant Nodes ## SVDQuant Nodes
* **SVDQuant Flux DiT Loader**: A node for loading the FLUX diffusion model. * **SVDQuant Flux DiT Loader**: A node for loading the FLUX diffusion model.
* `model_path`: Specifies the model location. If set to `mit-han-lab/svdq-int4-flux.1-schnell` or `mit-han-lab/svdq-int4-flux.1-dev`, the model will be automatically downloaded from our Hugging Face repository. Alternatively, you can manually download the model directory by running the following command: * `model_path`: Specifies the model location. If set to `mit-han-lab/svdq-int4-flux.1-schnell`, `mit-han-lab/svdq-int4-flux.1-dev`, `mit-han-lab/svdq-int4-flux.1-canny-dev`, `mit-han-lab/svdq-int4-flux.1-fill-dev` or `mit-han-lab/svdq-int4-flux.1-depth-dev`, the model will be automatically downloaded from our Hugging Face repository. Alternatively, you can manually download the model directory by running the following command example:
```shell ```shell
huggingface-cli download mit-han-lab/svdq-int4-flux.1-dev --local-dir models/diffusion_models/svdq-int4-flux.1-dev huggingface-cli download mit-han-lab/svdq-int4-flux.1-dev --local-dir models/diffusion_models/svdq-int4-flux.1-dev
......
comfyui/nodes.py
View file @ bfd9aa3a
...@@ -7,16 +7,17 @@ import comfy.sd ...@@ -7,16 +7,17 @@ import comfy.sd
import folder_paths import folder_paths
import GPUtil import GPUtil
import torch import torch
import numpy as np
from comfy.ldm.common_dit import pad_to_patch_size from comfy.ldm.common_dit import pad_to_patch_size
from comfy.supported_models import Flux, FluxSchnell from comfy.supported_models import Flux, FluxSchnell
from diffusers import FluxTransformer2DModel from diffusers import FluxTransformer2DModel
from einops import rearrange, repeat from einops import rearrange, repeat
from torch import nn from torch import nn
from transformers import T5EncoderModel from transformers import T5EncoderModel
from image_gen_aux import DepthPreprocessor
from nunchaku.models.transformer_flux import NunchakuFluxTransformer2dModel from nunchaku.models.transformer_flux import NunchakuFluxTransformer2dModel
class ComfyUIFluxForwardWrapper(nn.Module): class ComfyUIFluxForwardWrapper(nn.Module):
def __init__(self, model: NunchakuFluxTransformer2dModel, config): def __init__(self, model: NunchakuFluxTransformer2dModel, config):
super(ComfyUIFluxForwardWrapper, self).__init__() super(ComfyUIFluxForwardWrapper, self).__init__()
...@@ -24,13 +25,25 @@ class ComfyUIFluxForwardWrapper(nn.Module): ...@@ -24,13 +25,25 @@ class ComfyUIFluxForwardWrapper(nn.Module):
self.dtype = next(model.parameters()).dtype self.dtype = next(model.parameters()).dtype
self.config = config self.config = config
def forward(self, x, timestep, context, y, guidance, control=None, transformer_options={}, **kwargs): def forward(
self,
x,
timestep,
context,
y,
guidance,
control=None,
transformer_options={},
**kwargs,
):
assert control is None # for now assert control is None # for now
bs, c, h, w = x.shape bs, c, h, w = x.shape
patch_size = self.config["patch_size"] patch_size = self.config["patch_size"]
x = pad_to_patch_size(x, (patch_size, patch_size)) x = pad_to_patch_size(x, (patch_size, patch_size))
img = rearrange(x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch_size, pw=patch_size) img = rearrange(
x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch_size, pw=patch_size
)
h_len = (h + (patch_size // 2)) // patch_size h_len = (h + (patch_size // 2)) // patch_size
w_len = (w + (patch_size // 2)) // patch_size w_len = (w + (patch_size // 2)) // patch_size
...@@ -54,21 +67,30 @@ class ComfyUIFluxForwardWrapper(nn.Module): ...@@ -54,21 +67,30 @@ class ComfyUIFluxForwardWrapper(nn.Module):
guidance=guidance if self.config["guidance_embed"] else None, guidance=guidance if self.config["guidance_embed"] else None,
).sample ).sample
out = rearrange(out, "b (h w) (c ph pw) -> b c (h ph) (w pw)", h=h_len, w=w_len, ph=2, pw=2)[:, :, :h, :w] out = rearrange(
out, "b (h w) (c ph pw) -> b c (h ph) (w pw)", h=h_len, w=w_len, ph=2, pw=2
)[:, :, :h, :w]
return out return out
class SVDQuantFluxDiTLoader: class SVDQuantFluxDiTLoader:
@classmethod @classmethod
def INPUT_TYPES(s): def INPUT_TYPES(s):
model_paths = ["mit-han-lab/svdq-int4-flux.1-schnell", "mit-han-lab/svdq-int4-flux.1-dev"] model_paths = [
"mit-han-lab/svdq-int4-flux.1-schnell",
"mit-han-lab/svdq-int4-flux.1-dev",
"mit-han-lab/svdq-int4-flux.1-canny-dev",
"mit-han-lab/svdq-int4-flux.1-depth-dev",
"mit-han-lab/svdq-int4-flux.1-fill-dev",
]
prefix = "models/diffusion_models" prefix = "models/diffusion_models"
local_folders = os.listdir(prefix) local_folders = os.listdir(prefix)
local_folders = sorted( local_folders = sorted(
[ [
folder folder
for folder in local_folders for folder in local_folders
if not folder.startswith(".") and os.path.isdir(os.path.join(prefix, folder)) if not folder.startswith(".")
and os.path.isdir(os.path.join(prefix, folder))
] ]
) )
model_paths.extend(local_folders) model_paths.extend(local_folders)
...@@ -78,7 +100,14 @@ class SVDQuantFluxDiTLoader: ...@@ -78,7 +100,14 @@ class SVDQuantFluxDiTLoader:
"model_path": (model_paths,), "model_path": (model_paths,),
"device_id": ( "device_id": (
"INT", "INT",
{"default": 0, "min": 0, "max": ngpus, "step": 1, "display": "number", "lazy": True}, {
"default": 0,
"min": 0,
"max": ngpus,
"step": 1,
"display": "number",
"lazy": True,
},
), ),
} }
} }
...@@ -88,17 +117,20 @@ class SVDQuantFluxDiTLoader: ...@@ -88,17 +117,20 @@ class SVDQuantFluxDiTLoader:
CATEGORY = "SVDQuant" CATEGORY = "SVDQuant"
TITLE = "SVDQuant Flux DiT Loader" TITLE = "SVDQuant Flux DiT Loader"
def load_model(self, model_path: str, device_id: int, **kwargs) -> tuple[FluxTransformer2DModel]: def load_model(
self, model_path: str, device_id: int, **kwargs
) -> tuple[FluxTransformer2DModel]:
device = f"cuda:{device_id}" device = f"cuda:{device_id}"
prefix = "models/diffusion_models" prefix = "models/diffusion_models"
if os.path.exists(os.path.join(prefix, model_path)): if os.path.exists(os.path.join(prefix, model_path)):
model_path = os.path.join(prefix, model_path) model_path = os.path.join(prefix, model_path)
else: else:
model_path = model_path model_path = model_path
transformer = NunchakuFluxTransformer2dModel.from_pretrained(model_path).to(device) transformer = NunchakuFluxTransformer2dModel.from_pretrained(model_path).to(
device
)
dit_config = { dit_config = {
"image_model": "flux", "image_model": "flux",
"in_channels": 16,
"patch_size": 2, "patch_size": 2,
"out_channels": 16, "out_channels": 16,
"vec_in_dim": 768, "vec_in_dim": 768,
...@@ -111,21 +143,34 @@ class SVDQuantFluxDiTLoader: ...@@ -111,21 +143,34 @@ class SVDQuantFluxDiTLoader:
"axes_dim": [16, 56, 56], "axes_dim": [16, 56, 56],
"theta": 10000, "theta": 10000,
"qkv_bias": True, "qkv_bias": True,
"guidance_embed": True,
"disable_unet_model_creation": True, "disable_unet_model_creation": True,
} }
if "schnell" in model_path: if "schnell" in model_path:
dit_config["guidance_embed"] = False dit_config["guidance_embed"] = False
dit_config["in_channels"] = 16
model_config = FluxSchnell(dit_config) model_config = FluxSchnell(dit_config)
elif "canny" in model_path or "depth" in model_path:
dit_config["in_channels"] = 32
model_config = Flux(dit_config)
elif "fill" in model_path:
dit_config["in_channels"] = 64
model_config = Flux(dit_config)
else: else:
assert "dev" in model_path assert (
dit_config["guidance_embed"] = True model_path == "mit-han-lab/svdq-int4-flux.1-dev"
), f"model {model_path} not supported"
dit_config["in_channels"] = 16
model_config = Flux(dit_config) model_config = Flux(dit_config)
model_config.set_inference_dtype(torch.bfloat16, None) model_config.set_inference_dtype(torch.bfloat16, None)
model_config.custom_operations = None model_config.custom_operations = None
model = model_config.get_model({}) model = model_config.get_model({})
model.diffusion_model = ComfyUIFluxForwardWrapper(transformer, config=dit_config) model.diffusion_model = ComfyUIFluxForwardWrapper(
transformer, config=dit_config
)
model = comfy.model_patcher.ModelPatcher(model, device, device_id) model = comfy.model_patcher.ModelPatcher(model, device, device_id)
return (model,) return (model,)
...@@ -157,7 +202,8 @@ class SVDQuantTextEncoderLoader: ...@@ -157,7 +202,8 @@ class SVDQuantTextEncoderLoader:
[ [
folder folder
for folder in local_folders for folder in local_folders
if not folder.startswith(".") and os.path.isdir(os.path.join(prefix, folder)) if not folder.startswith(".")
and os.path.isdir(os.path.join(prefix, folder))
] ]
) )
model_paths.extend(local_folders) model_paths.extend(local_folders)
...@@ -168,7 +214,14 @@ class SVDQuantTextEncoderLoader: ...@@ -168,7 +214,14 @@ class SVDQuantTextEncoderLoader:
"text_encoder2": (folder_paths.get_filename_list("text_encoders"),), "text_encoder2": (folder_paths.get_filename_list("text_encoders"),),
"t5_min_length": ( "t5_min_length": (
"INT", "INT",
{"default": 512, "min": 256, "max": 1024, "step": 128, "display": "number", "lazy": True}, {
"default": 512,
"min": 256,
"max": 1024,
"step": 128,
"display": "number",
"lazy": True,
},
), ),
"t5_precision": (["BF16", "INT4"],), "t5_precision": (["BF16", "INT4"],),
"int4_model": (model_paths, {"tooltip": "The name of the INT4 model."}), "int4_model": (model_paths, {"tooltip": "The name of the INT4 model."}),
...@@ -191,8 +244,12 @@ class SVDQuantTextEncoderLoader: ...@@ -191,8 +244,12 @@ class SVDQuantTextEncoderLoader:
t5_precision: str, t5_precision: str,
int4_model: str, int4_model: str,
): ):
text_encoder_path1 = folder_paths.get_full_path_or_raise("text_encoders", text_encoder1) text_encoder_path1 = folder_paths.get_full_path_or_raise(
text_encoder_path2 = folder_paths.get_full_path_or_raise("text_encoders", text_encoder2) "text_encoders", text_encoder1
)
text_encoder_path2 = folder_paths.get_full_path_or_raise(
"text_encoders", text_encoder2
)
if model_type == "flux": if model_type == "flux":
clip_type = comfy.sd.CLIPType.FLUX clip_type = comfy.sd.CLIPType.FLUX
else: else:
...@@ -223,7 +280,9 @@ class SVDQuantTextEncoderLoader: ...@@ -223,7 +280,9 @@ class SVDQuantTextEncoderLoader:
transformer = NunchakuT5EncoderModel.from_pretrained(model_path) transformer = NunchakuT5EncoderModel.from_pretrained(model_path)
transformer.forward = types.MethodType(svdquant_t5_forward, transformer) transformer.forward = types.MethodType(svdquant_t5_forward, transformer)
clip.cond_stage_model.t5xxl.transformer = ( clip.cond_stage_model.t5xxl.transformer = (
transformer.to(device=device, dtype=dtype) if device.type == "cuda" else transformer transformer.to(device=device, dtype=dtype)
if device.type == "cuda"
else transformer
) )
return (clip,) return (clip,)
...@@ -239,11 +298,17 @@ class SVDQuantLoraLoader: ...@@ -239,11 +298,17 @@ class SVDQuantLoraLoader:
lora_name_list = [ lora_name_list = [
"None", "None",
*folder_paths.get_filename_list("loras"), *folder_paths.get_filename_list("loras"),
*[f"mit-han-lab/svdquant-models/svdq-flux.1-dev-lora-{n}.safetensors" for n in hf_lora_names], *[
f"mit-han-lab/svdquant-models/svdq-flux.1-dev-lora-{n}.safetensors"
for n in hf_lora_names
],
] ]
return { return {
"required": { "required": {
"model": ("MODEL", {"tooltip": "The diffusion model the LoRA will be applied to."}), "model": (
"MODEL",
{"tooltip": "The diffusion model the LoRA will be applied to."},
),
"lora_name": (lora_name_list, {"tooltip": "The name of the LoRA."}), "lora_name": (lora_name_list, {"tooltip": "The name of the LoRA."}),
"lora_strength": ( "lora_strength": (
"FLOAT", "FLOAT",
...@@ -292,8 +357,50 @@ class SVDQuantLoraLoader: ...@@ -292,8 +357,50 @@ class SVDQuantLoraLoader:
return (model,) return (model,)
class DepthPreprocesser:
@classmethod
def INPUT_TYPES(s):
model_paths = ["LiheYoung/depth-anything-large-hf"]
prefix = "models/checkpoints"
local_folders = os.listdir(prefix)
local_folders = sorted(
[
folder
for folder in local_folders
if not folder.startswith(".")
and os.path.isdir(os.path.join(prefix, folder))
]
)
model_paths.extend(local_folders)
return {
"required": {
"image": ("IMAGE", {}),
"model_path": (
model_paths,
{"tooltip": "Name of the depth preprocesser model."},
),
}
}
RETURN_TYPES = ("IMAGE",)
FUNCTION = "depth_preprocess"
CATEGORY = "Flux.1"
TITLE = "Flux.1 Depth Preprocessor"
def depth_preprocess(self, image, model_path):
prefix = "models/checkpoints"
if os.path.exists(os.path.join(prefix, model_path)):
model_path = os.path.join(prefix, model_path)
processor = DepthPreprocessor.from_pretrained(model_path)
np_image = np.asarray(image)
np_result = np.array(processor(np_image)[0].convert("RGB"))
out_tensor = torch.from_numpy(np_result.astype(np.float32) / 255.0).unsqueeze(0)
return (out_tensor,)
NODE_CLASS_MAPPINGS = { NODE_CLASS_MAPPINGS = {
"SVDQuantFluxDiTLoader": SVDQuantFluxDiTLoader, "SVDQuantFluxDiTLoader": SVDQuantFluxDiTLoader,
"SVDQuantTextEncoderLoader": SVDQuantTextEncoderLoader, "SVDQuantTextEncoderLoader": SVDQuantTextEncoderLoader,
"SVDQuantLoRALoader": SVDQuantLoraLoader, "SVDQuantLoRALoader": SVDQuantLoraLoader,
"DepthPreprocesser": DepthPreprocesser
} }
comfyui/workflows/flux.1-canny-svdquant.json 0 → 100644
View file @ bfd9aa3a
{
"last_node_id": 38,
"last_link_id": 76,
"nodes": [
{
"id": 3,
"type": "KSampler",
"pos": [
1290,
40
],
"size": [
315,
262
],
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 71,
"label": "model"
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 64,
"label": "positive"
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 65,
"label": "negative"
},
{
"name": "latent_image",
"type": "LATENT",
"link": 66,
"label": "latent_image"
}
],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
7
],
"slot_index": 0,
"label": "LATENT"
}
],
"properties": {
"Node name for S&R": "KSampler"
},
"widgets_values": [
646386750200194,
"randomize",
20,
1,
"euler",
"normal",
1
]
},
{
"id": 35,
"type": "InstructPixToPixConditioning",
"pos": [
1040,
50
],
"size": [
235.1999969482422,
86
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "positive",
"type": "CONDITIONING",
"link": 67,
"label": "positive"
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 68,
"label": "negative"
},
{
"name": "vae",
"type": "VAE",
"link": 69,
"label": "vae"
},
{
"name": "pixels",
"type": "IMAGE",
"link": 70,
"label": "pixels"
}
],
"outputs": [
{
"name": "positive",
"type": "CONDITIONING",
"links": [
64
],
"slot_index": 0,
"label": "positive"
},
{
"name": "negative",
"type": "CONDITIONING",
"links": [
65
],
"slot_index": 1,
"label": "negative"
},
{
"name": "latent",
"type": "LATENT",
"links": [
66
],
"slot_index": 2,
"label": "latent"
}
],
"properties": {
"Node name for S&R": "InstructPixToPixConditioning"
},
"widgets_values": []
},
{
"id": 8,
"type": "VAEDecode",
"pos": [
1620,
40
],
"size": [
210,
46
],
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 7,
"label": "samples"
},
{
"name": "vae",
"type": "VAE",
"link": 60,
"label": "vae"
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
9
],
"slot_index": 0,
"label": "IMAGE"
}
],
"properties": {
"Node name for S&R": "VAEDecode"
},
"widgets_values": []
},
{
"id": 9,
"type": "SaveImage",
"pos": [
1850,
40
],
"size": [
828.9535522460938,
893.8475341796875
],
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 9,
"label": "images"
}
],
"outputs": [],
"properties": {},
"widgets_values": [
"ComfyUI"
]
},
{
"id": 32,
"type": "VAELoader",
"pos": [
1290,
350
],
"size": [
315,
58
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "VAE",
"type": "VAE",
"links": [
60,
69
],
"slot_index": 0,
"label": "VAE"
}
],
"properties": {
"Node name for S&R": "VAELoader"
},
"widgets_values": [
"ae.safetensors"
]
},
{
"id": 26,
"type": "FluxGuidance",
"pos": [
700,
50
],
"size": [
317.4000244140625,
58
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "conditioning",
"type": "CONDITIONING",
"link": 41,
"label": "conditioning"
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
67
],
"slot_index": 0,
"shape": 3,
"label": "CONDITIONING"
}
],
"properties": {
"Node name for S&R": "FluxGuidance"
},
"widgets_values": [
30
]
},
{
"id": 34,
"type": "DualCLIPLoader",
"pos": [
-80,
110
],
"size": [
315,
106
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"links": [
62,
63
],
"label": "CLIP"
}
],
"properties": {
"Node name for S&R": "DualCLIPLoader"
},
"widgets_values": [
"clip_l.safetensors",
"t5xxl_fp16.safetensors",
"flux",
"default"
]
},
{
"id": 7,
"type": "CLIPTextEncode",
"pos": [
307,
282
],
"size": [
425.27801513671875,
180.6060791015625
],
"flags": {
"collapsed": true
},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 63,
"label": "clip"
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
68
],
"slot_index": 0,
"label": "CONDITIONING"
}
],
"title": "CLIP Text Encode (Negative Prompt)",
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
""
],
"color": "#322",
"bgcolor": "#533"
},
{
"id": 23,
"type": "CLIPTextEncode",
"pos": [
260,
50
],
"size": [
422.84503173828125,
164.31304931640625
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 62,
"label": "clip"
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
41
],
"slot_index": 0,
"label": "CONDITIONING"
}
],
"title": "CLIP Text Encode (Positive Prompt)",
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"A robot made of exotic candies and chocolates of different kinds. The background is filled with confetti and celebratory gifts."
],
"color": "#232",
"bgcolor": "#353"
},
{
"id": 19,
"type": "PreviewImage",
"pos": [
1127.9403076171875,
554.3356323242188
],
"size": [
571.5869140625,
625.5296020507812
],
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 26,
"label": "images"
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 18,
"type": "Canny",
"pos": [
744.2684936523438,
566.853515625
],
"size": [
315,
82
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 76,
"label": "image"
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
26,
70
],
"slot_index": 0,
"shape": 3,
"label": "IMAGE"
}
],
"properties": {
"Node name for S&R": "Canny"
},
"widgets_values": [
0.15,
0.3
]
},
{
"id": 36,
"type": "SVDQuantFluxDiTLoader",
"pos": [
865.4989624023438,
-95.86973571777344
],
"size": [
315,
82
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
71
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "SVDQuantFluxDiTLoader"
},
"widgets_values": [
"mit-han-lab/svdq-int4-flux.1-canny-dev",
0
]
},
{
"id": 17,
"type": "LoadImage",
"pos": [
6.694743633270264,
562.3865966796875
],
"size": [
315,
314.0000305175781
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
75
],
"slot_index": 0,
"shape": 3,
"label": "IMAGE"
},
{
"name": "MASK",
"type": "MASK",
"links": null,
"shape": 3,
"label": "MASK"
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"robot.png",
"image"
]
},
{
"id": 38,
"type": "ImageScale",
"pos": [
379.69903564453125,
565.2651977539062
],
"size": [
315,
130
],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 75
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
76
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "ImageScale"
},
"widgets_values": [
"nearest-exact",
1024,
1024,
"center"
]
}
],
"links": [
[
7,
3,
0,
8,
0,
"LATENT"
],
[
9,
8,
0,
9,
0,
"IMAGE"
],
[
26,
18,
0,
19,
0,
"IMAGE"
],
[
41,
23,
0,
26,
0,
"CONDITIONING"
],
[
60,
32,
0,
8,
1,
"VAE"
],
[
62,
34,
0,
23,
0,
"CLIP"
],
[
63,
34,
0,
7,
0,
"CLIP"
],
[
64,
35,
0,
3,
1,
"CONDITIONING"
],
[
65,
35,
1,
3,
2,
"CONDITIONING"
],
[
66,
35,
2,
3,
3,
"LATENT"
],
[
67,
26,
0,
35,
0,
"CONDITIONING"
],
[
68,
7,
0,
35,
1,
"CONDITIONING"
],
[
69,
32,
0,
35,
2,
"VAE"
],
[
70,
18,
0,
35,
3,
"IMAGE"
],
[
71,
36,
0,
3,
0,
"MODEL"
],
[
75,
17,
0,
38,
0,
"IMAGE"
],
[
76,
38,
0,
18,
0,
"IMAGE"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.895430243255241,
"offset": [
203.26101803057463,
215.36536277004458
]
}
},
"version": 0.4
}
\ No newline at end of file
comfyui/workflows/flux.1-depth-svdquant.json 0 → 100644
View file @ bfd9aa3a
{
"last_node_id": 44,
"last_link_id": 85,
"nodes": [
{
"id": 7,
"type": "CLIPTextEncode",
"pos": [
307,
282
],
"size": [
425.27801513671875,
180.6060791015625
],
"flags": {
"collapsed": true
},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 63,
"label": "clip"
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
68
],
"slot_index": 0,
"label": "CONDITIONING"
}
],
"title": "CLIP Text Encode (Negative Prompt)",
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
""
],
"color": "#322",
"bgcolor": "#533"
},
{
"id": 34,
"type": "DualCLIPLoader",
"pos": [
-238,
112
],
"size": [
315,
106
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"links": [
62,
63
],
"label": "CLIP"
}
],
"properties": {
"Node name for S&R": "DualCLIPLoader"
},
"widgets_values": [
"clip_l.safetensors",
"t5xxl_fp16.safetensors",
"flux",
"default"
]
},
{
"id": 26,
"type": "FluxGuidance",
"pos": [
621,
8
],
"size": [
317.4000244140625,
58
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "conditioning",
"type": "CONDITIONING",
"link": 41,
"label": "conditioning"
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
67
],
"slot_index": 0,
"shape": 3,
"label": "CONDITIONING"
}
],
"properties": {
"Node name for S&R": "FluxGuidance"
},
"widgets_values": [
10
]
},
{
"id": 32,
"type": "VAELoader",
"pos": [
656,
165
],
"size": [
315,
58
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "VAE",
"type": "VAE",
"links": [
60,
69
],
"slot_index": 0,
"label": "VAE"
}
],
"properties": {
"Node name for S&R": "VAELoader"
},
"widgets_values": [
"ae.safetensors"
]
},
{
"id": 3,
"type": "KSampler",
"pos": [
1280,
100
],
"size": [
315,
262
],
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 78,
"label": "model"
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 64,
"label": "positive"
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 65,
"label": "negative"
},
{
"name": "latent_image",
"type": "LATENT",
"link": 73,
"label": "latent_image"
}
],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
7
],
"slot_index": 0,
"label": "LATENT"
}
],
"properties": {
"Node name for S&R": "KSampler"
},
"widgets_values": [
718322679777603,
"randomize",
20,
1,
"euler",
"normal",
1
]
},
{
"id": 35,
"type": "InstructPixToPixConditioning",
"pos": [
1008,
118
],
"size": [
235.1999969482422,
86
],
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "positive",
"type": "CONDITIONING",
"link": 67,
"label": "positive"
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 68,
"label": "negative"
},
{
"name": "vae",
"type": "VAE",
"link": 69,
"label": "vae"
},
{
"name": "pixels",
"type": "IMAGE",
"link": 80,
"label": "pixels"
}
],
"outputs": [
{
"name": "positive",
"type": "CONDITIONING",
"links": [
64
],
"slot_index": 0,
"label": "positive"
},
{
"name": "negative",
"type": "CONDITIONING",
"links": [
65
],
"slot_index": 1,
"label": "negative"
},
{
"name": "latent",
"type": "LATENT",
"links": [
73
],
"slot_index": 2,
"label": "latent"
}
],
"properties": {
"Node name for S&R": "InstructPixToPixConditioning"
},
"widgets_values": []
},
{
"id": 39,
"type": "SVDQuantFluxDiTLoader",
"pos": [
707.80908203125,
-172.0343017578125
],
"size": [
315,
82
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
78
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "SVDQuantFluxDiTLoader"
},
"widgets_values": [
"mit-han-lab/svdq-int4-flux.1-depth-dev",
0
]
},
{
"id": 42,
"type": "ImageScale",
"pos": [
378.3890686035156,
472.7001953125
],
"size": [
315,
130
],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 82
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
83
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "ImageScale"
},
"widgets_values": [
"nearest-exact",
1024,
1024,
"center"
]
},
{
"id": 17,
"type": "LoadImage",
"pos": [
30.604948043823242,
419.3930358886719
],
"size": [
315,
314.0000305175781
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
82
],
"slot_index": 0,
"shape": 3,
"label": "IMAGE"
},
{
"name": "MASK",
"type": "MASK",
"links": null,
"shape": 3,
"label": "MASK"
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"robot.png",
"image"
]
},
{
"id": 23,
"type": "CLIPTextEncode",
"pos": [
115,
-17
],
"size": [
422.84503173828125,
164.31304931640625
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 62,
"label": "clip"
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
41
],
"slot_index": 0,
"label": "CONDITIONING"
}
],
"title": "CLIP Text Encode (Positive Prompt)",
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"A robot made of exotic candies and chocolates of different kinds. The background is filled with confetti and celebratory gifts."
],
"color": "#232",
"bgcolor": "#353"
},
{
"id": 43,
"type": "PreviewImage",
"pos": [
1001.3873291015625,
432.09039306640625
],
"size": [
571.5869140625,
625.5296020507812
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 84,
"label": "images"
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 40,
"type": "DepthPreprocesser",
"pos": [
639.0159301757812,
350.06134033203125
],
"size": [
315,
58
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 83
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
80,
84
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "DepthPreprocesser"
},
"widgets_values": [
"LiheYoung/depth-anything-large-hf"
]
},
{
"id": 8,
"type": "VAEDecode",
"pos": [
1620,
98
],
"size": [
210,
46
],
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 7,
"label": "samples"
},
{
"name": "vae",
"type": "VAE",
"link": 60,
"label": "vae"
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
85
],
"slot_index": 0,
"label": "IMAGE"
}
],
"properties": {
"Node name for S&R": "VAEDecode"
},
"widgets_values": []
},
{
"id": 44,
"type": "SaveImage",
"pos": [
1912.7984619140625,
109.0069580078125
],
"size": [
828.9535522460938,
893.8475341796875
],
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 85,
"label": "images"
}
],
"outputs": [],
"properties": {},
"widgets_values": [
"ComfyUI"
]
}
],
"links": [
[
7,
3,
0,
8,
0,
"LATENT"
],
[
41,
23,
0,
26,
0,
"CONDITIONING"
],
[
60,
32,
0,
8,
1,
"VAE"
],
[
62,
34,
0,
23,
0,
"CLIP"
],
[
63,
34,
0,
7,
0,
"CLIP"
],
[
64,
35,
0,
3,
1,
"CONDITIONING"
],
[
65,
35,
1,
3,
2,
"CONDITIONING"
],
[
67,
26,
0,
35,
0,
"CONDITIONING"
],
[
68,
7,
0,
35,
1,
"CONDITIONING"
],
[
69,
32,
0,
35,
2,
"VAE"
],
[
73,
35,
2,
3,
3,
"LATENT"
],
[
78,
39,
0,
3,
0,
"MODEL"
],
[
80,
40,
0,
35,
3,
"IMAGE"
],
[
82,
17,
0,
42,
0,
"IMAGE"
],
[
83,
42,
0,
40,
0,
"IMAGE"
],
[
84,
40,
0,
43,
0,
"IMAGE"
],
[
85,
8,
0,
44,
0,
"IMAGE"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.6115909044841502,
"offset": [
724.4911189218763,
518.3043483917891
]
}
},
"version": 0.4
}
\ No newline at end of file
comfyui/workflows/flux.1-fill-svdquant.json 0 → 100644
View file @ bfd9aa3a
{
"last_node_id": 58,
"last_link_id": 108,
"nodes": [
{
"id": 8,
"type": "VAEDecode",
"pos": [
1620,
98
],
"size": [
210,
46
],
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 7
},
{
"name": "vae",
"type": "VAE",
"link": 60
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
95
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAEDecode"
},
"widgets_values": []
},
{
"id": 38,
"type": "InpaintModelConditioning",
"pos": [
952,
78
],
"size": [
302.4000244140625,
138
],
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "positive",
"type": "CONDITIONING",
"link": 80
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 81
},
{
"name": "vae",
"type": "VAE",
"link": 82
},
{
"name": "pixels",
"type": "IMAGE",
"link": 107
},
{
"name": "mask",
"type": "MASK",
"link": 108
}
],
"outputs": [
{
"name": "positive",
"type": "CONDITIONING",
"links": [
77
],
"slot_index": 0
},
{
"name": "negative",
"type": "CONDITIONING",
"links": [
78
],
"slot_index": 1
},
{
"name": "latent",
"type": "LATENT",
"links": [
88
],
"slot_index": 2
}
],
"properties": {
"Node name for S&R": "InpaintModelConditioning"
},
"widgets_values": [
false
]
},
{
"id": 3,
"type": "KSampler",
"pos": [
1280,
100
],
"size": [
315,
262
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 102
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 77
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 78
},
{
"name": "latent_image",
"type": "LATENT",
"link": 88
}
],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
7
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "KSampler"
},
"widgets_values": [
1078304127779394,
"randomize",
20,
1,
"euler",
"normal",
1
]
},
{
"id": 9,
"type": "SaveImage",
"pos": [
1879,
90
],
"size": [
828.9535522460938,
893.8475341796875
],
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 95
}
],
"outputs": [],
"properties": {
"Node name for S&R": "SaveImage"
},
"widgets_values": [
"ComfyUI"
]
},
{
"id": 26,
"type": "FluxGuidance",
"pos": [
596,
48
],
"size": [
317.4000244140625,
58
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "conditioning",
"type": "CONDITIONING",
"link": 41
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
80
],
"slot_index": 0,
"shape": 3
}
],
"properties": {
"Node name for S&R": "FluxGuidance"
},
"widgets_values": [
30
]
},
{
"id": 7,
"type": "CLIPTextEncode",
"pos": [
165,
267
],
"size": [
425.27801513671875,
180.6060791015625
],
"flags": {
"collapsed": true
},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 63
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
81
],
"slot_index": 0
}
],
"title": "CLIP Text Encode (Negative Prompt)",
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
""
],
"color": "#322",
"bgcolor": "#533"
},
{
"id": 34,
"type": "DualCLIPLoader",
"pos": [
-237,
76
],
"size": [
315,
106
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"links": [
62,
63
]
}
],
"properties": {
"Node name for S&R": "DualCLIPLoader"
},
"widgets_values": [
"clip_l.safetensors",
"t5xxl_fp16.safetensors",
"flux",
"default"
]
},
{
"id": 32,
"type": "VAELoader",
"pos": [
1303,
424
],
"size": [
315,
58
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "VAE",
"type": "VAE",
"links": [
60,
82
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAELoader"
},
"widgets_values": [
"ae.safetensors"
]
},
{
"id": 45,
"type": "SVDQuantFluxDiTLoader",
"pos": [
936.3029174804688,
-113.06819915771484
],
"size": [
315,
82
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
102
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "SVDQuantFluxDiTLoader"
},
"widgets_values": [
"mit-han-lab/svdq-int4-flux.1-fill-dev",
0
]
},
{
"id": 48,
"type": "Note",
"pos": [
466.9884033203125,
643.9080810546875
],
"size": [
314.99755859375,
117.98363494873047
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [],
"properties": {
"text": ""
},
"widgets_values": [
"To add mask for fill inpainting, right click on the uploaded image and select \"Open in MaskEditor\". Use the brush tool to add masking and click save to continue."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 17,
"type": "LoadImage",
"pos": [
126.66505432128906,
460.53631591796875
],
"size": [
315,
314.0000305175781
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
105
],
"slot_index": 0,
"shape": 3
},
{
"name": "MASK",
"type": "MASK",
"links": [
106
],
"slot_index": 1,
"shape": 3
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"clipspace/clipspace-mask-123191.png [input]",
"image"
]
},
{
"id": 58,
"type": "ImageAndMaskResizeNode",
"pos": [
536.786865234375,
328.54388427734375
],
"size": [
315,
174
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 105
},
{
"name": "mask",
"type": "MASK",
"link": 106
}
],
"outputs": [
{
"name": "image",
"type": "IMAGE",
"links": [
107
],
"slot_index": 0
},
{
"name": "mask",
"type": "MASK",
"links": [
108
],
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "ImageAndMaskResizeNode"
},
"widgets_values": [
1024,
1024,
"nearest-exact",
"center",
10
]
},
{
"id": 23,
"type": "CLIPTextEncode",
"pos": [
144,
-7
],
"size": [
422.84503173828125,
164.31304931640625
],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 62
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
41
],
"slot_index": 0
}
],
"title": "CLIP Text Encode (Positive Prompt)",
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"A robot with a closed eye pink face is giving a presentation"
],
"color": "#232",
"bgcolor": "#353"
}
],
"links": [
[
7,
3,
0,
8,
0,
"LATENT"
],
[
41,
23,
0,
26,
0,
"CONDITIONING"
],
[
60,
32,
0,
8,
1,
"VAE"
],
[
62,
34,
0,
23,
0,
"CLIP"
],
[
63,
34,
0,
7,
0,
"CLIP"
],
[
77,
38,
0,
3,
1,
"CONDITIONING"
],
[
78,
38,
1,
3,
2,
"CONDITIONING"
],
[
80,
26,
0,
38,
0,
"CONDITIONING"
],
[
81,
7,
0,
38,
1,
"CONDITIONING"
],
[
82,
32,
0,
38,
2,
"VAE"
],
[
88,
38,
2,
3,
3,
"LATENT"
],
[
95,
8,
0,
9,
0,
"IMAGE"
],
[
102,
45,
0,
3,
0,
"MODEL"
],
[
105,
17,
0,
58,
0,
"IMAGE"
],
[
106,
17,
1,
58,
1,
"MASK"
],
[
107,
58,
0,
38,
3,
"IMAGE"
],
[
108,
58,
1,
38,
4,
"MASK"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.8390545288824038,
"offset": [
361.8437326514503,
242.3368651567008
]
},
"node_versions": {
"comfy-core": "0.3.14",
"comfyui-inpainteasy": "1.0.2"
}
},
"version": 0.4
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment