Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
test_flux_tools.py 3.7 KB
Newer Older
muyangli's avatar
v0.1.4 ready to release  
muyangli committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92 
import torch
from controlnet_aux import CannyDetector
from diffusers import FluxControlPipeline, FluxFillPipeline, FluxPipeline, FluxPriorReduxPipeline
from diffusers.utils import load_image
from image_gen_aux import DepthPreprocessor

from nunchaku import NunchakuFluxTransformer2dModel


def test_flux_dev_canny():
    transformer = NunchakuFluxTransformer2dModel.from_pretrained("mit-han-lab/svdq-int4-flux.1-canny-dev")
    pipe = FluxControlPipeline.from_pretrained(
        "black-forest-labs/FLUX.1-Canny-dev", transformer=transformer, torch_dtype=torch.bfloat16
    ).to("cuda")

    prompt = "A robot made of exotic candies and chocolates of different kinds. The background is filled with confetti and celebratory gifts."
    control_image = load_image(
        "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/robot.png"
    )

    processor = CannyDetector()
    control_image = processor(
        control_image, low_threshold=50, high_threshold=200, detect_resolution=1024, image_resolution=1024
    )

    image = pipe(
        prompt=prompt, control_image=control_image, height=1024, width=1024, num_inference_steps=50, guidance_scale=30.0
    ).images[0]
    image.save("flux.1-canny-dev.png")


def test_flux_dev_depth():
    transformer = NunchakuFluxTransformer2dModel.from_pretrained("mit-han-lab/svdq-int4-flux.1-depth-dev")

    pipe = FluxControlPipeline.from_pretrained(
        "black-forest-labs/FLUX.1-Depth-dev",
        transformer=transformer,
        torch_dtype=torch.bfloat16,
    ).to("cuda")

    prompt = "A robot made of exotic candies and chocolates of different kinds. The background is filled with confetti and celebratory gifts."
    control_image = load_image(
        "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/robot.png"
    )

    processor = DepthPreprocessor.from_pretrained("LiheYoung/depth-anything-large-hf")
    control_image = processor(control_image)[0].convert("RGB")

    image = pipe(
        prompt=prompt, control_image=control_image, height=1024, width=1024, num_inference_steps=30, guidance_scale=10.0
    ).images[0]
    image.save("flux.1-depth-dev.png")


def test_flux_dev_fill():
    image = load_image("https://huggingface.co/mit-han-lab/svdq-int4-flux.1-fill-dev/resolve/main/example.png")
    mask = load_image("https://huggingface.co/mit-han-lab/svdq-int4-flux.1-fill-dev/resolve/main/mask.png")

    transformer = NunchakuFluxTransformer2dModel.from_pretrained("mit-han-lab/svdq-int4-flux.1-fill-dev")
    pipe = FluxFillPipeline.from_pretrained(
        "black-forest-labs/FLUX.1-Fill-dev", transformer=transformer, torch_dtype=torch.bfloat16
    ).to("cuda")
    image = pipe(
        prompt="A wooden basket of a cat.",
        image=image,
        mask_image=mask,
        height=1024,
        width=1024,
        guidance_scale=30,
        num_inference_steps=50,
        max_sequence_length=512,
    ).images[0]
    image.save("flux.1-fill-dev.png")


def test_flux_dev_redux():
    pipe_prior_redux = FluxPriorReduxPipeline.from_pretrained(
        "black-forest-labs/FLUX.1-Redux-dev", torch_dtype=torch.bfloat16
    ).to("cuda")
    transformer = NunchakuFluxTransformer2dModel.from_pretrained("mit-han-lab/svdq-int4-flux.1-dev")
    pipe = FluxPipeline.from_pretrained(
        "black-forest-labs/FLUX.1-dev",
        text_encoder=None,
        text_encoder_2=None,
        transformer=transformer,
        torch_dtype=torch.bfloat16,
    ).to("cuda")

    image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/robot.png")
    pipe_prior_output = pipe_prior_redux(image)
    images = pipe(guidance_scale=2.5, num_inference_steps=50, **pipe_prior_output).images
    images[0].save("flux.1-redux-dev.png")