samplers.py 30.2 KB
Newer Older
1
from .k_diffusion import sampling as k_diffusion_sampling
2
from .extra_samplers import uni_pc
comfyanonymous's avatar
comfyanonymous committed
3
import torch
4
import torch.nn.functional as F
5
import enum
6
from comfy import model_management
comfyanonymous's avatar
comfyanonymous committed
7
import math
8
from comfy import model_base
9
import comfy.utils
10
import comfy.conds
11
12


comfyanonymous's avatar
comfyanonymous committed
13
#The main sampling function shared by all the samplers
comfyanonymous's avatar
comfyanonymous committed
14
#Returns denoised
15
def sampling_function(model, x, timestep, uncond, cond, cond_scale, model_options={}, seed=None):
16
        def get_area_and_mult(conds, x_in, timestep_in):
17
18
            area = (x_in.shape[2], x_in.shape[3], 0, 0)
            strength = 1.0
19
20
21

            if 'timestep_start' in conds:
                timestep_start = conds['timestep_start']
22
                if timestep_in[0] > timestep_start:
23
                    return None
24
25
            if 'timestep_end' in conds:
                timestep_end = conds['timestep_end']
26
                if timestep_in[0] < timestep_end:
27
                    return None
28
29
30
31
            if 'area' in conds:
                area = conds['area']
            if 'strength' in conds:
                strength = conds['strength']
32

33
            input_x = x_in[:,:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]]
34
            if 'mask' in conds:
Jacob Segal's avatar
Jacob Segal committed
35
36
                # Scale the mask to the size of the input
                # The mask should have been resized as we began the sampling process
37
                mask_strength = 1.0
38
39
40
                if "mask_strength" in conds:
                    mask_strength = conds["mask_strength"]
                mask = conds['mask']
Jacob Segal's avatar
Jacob Segal committed
41
42
                assert(mask.shape[1] == x_in.shape[2])
                assert(mask.shape[2] == x_in.shape[3])
43
                mask = mask[:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]] * mask_strength
Jacob Segal's avatar
Jacob Segal committed
44
                mask = mask.unsqueeze(1).repeat(input_x.shape[0] // mask.shape[0], input_x.shape[1], 1, 1)
Jacob Segal's avatar
Jacob Segal committed
45
46
47
48
            else:
                mask = torch.ones_like(input_x)
            mult = mask * strength

49
            if 'mask' not in conds:
Jacob Segal's avatar
Jacob Segal committed
50
51
52
53
54
55
56
57
58
59
60
61
62
63
                rr = 8
                if area[2] != 0:
                    for t in range(rr):
                        mult[:,:,t:1+t,:] *= ((1.0/rr) * (t + 1))
                if (area[0] + area[2]) < x_in.shape[2]:
                    for t in range(rr):
                        mult[:,:,area[0] - 1 - t:area[0] - t,:] *= ((1.0/rr) * (t + 1))
                if area[3] != 0:
                    for t in range(rr):
                        mult[:,:,:,t:1+t] *= ((1.0/rr) * (t + 1))
                if (area[1] + area[3]) < x_in.shape[3]:
                    for t in range(rr):
                        mult[:,:,:,area[1] - 1 - t:area[1] - t] *= ((1.0/rr) * (t + 1))

64
            conditioning = {}
65
66
            model_conds = conds["model_conds"]
            for c in model_conds:
67
                conditioning[c] = model_conds[c].process_cond(batch_size=x_in.shape[0], device=x_in.device, area=area)
68

comfyanonymous's avatar
comfyanonymous committed
69
            control = None
70
71
            if 'control' in conds:
                control = conds['control']
72
73

            patches = None
74
75
            if 'gligen' in conds:
                gligen = conds['gligen']
76
77
78
79
                patches = {}
                gligen_type = gligen[0]
                gligen_model = gligen[1]
                if gligen_type == "position":
comfyanonymous's avatar
comfyanonymous committed
80
                    gligen_patch = gligen_model.model.set_position(input_x.shape, gligen[2], input_x.device)
81
                else:
comfyanonymous's avatar
comfyanonymous committed
82
                    gligen_patch = gligen_model.model.set_empty(input_x.shape, input_x.device)
83
84
85

                patches['middle_patch'] = [gligen_patch]

86
            return (input_x, mult, conditioning, area, control, patches)
comfyanonymous's avatar
comfyanonymous committed
87
88

        def cond_equal_size(c1, c2):
comfyanonymous's avatar
comfyanonymous committed
89
90
            if c1 is c2:
                return True
comfyanonymous's avatar
comfyanonymous committed
91
92
            if c1.keys() != c2.keys():
                return False
93
94
            for k in c1:
                if not c1[k].can_concat(c2[k]):
95
                    return False
comfyanonymous's avatar
comfyanonymous committed
96
97
            return True

comfyanonymous's avatar
comfyanonymous committed
98
99
100
        def can_concat_cond(c1, c2):
            if c1[0].shape != c2[0].shape:
                return False
101
102

            #control
comfyanonymous's avatar
comfyanonymous committed
103
104
105
106
107
108
            if (c1[4] is None) != (c2[4] is None):
                return False
            if c1[4] is not None:
                if c1[4] is not c2[4]:
                    return False

109
110
111
112
113
114
115
            #patches
            if (c1[5] is None) != (c2[5] is None):
                return False
            if (c1[5] is not None):
                if c1[5] is not c2[5]:
                    return False

comfyanonymous's avatar
comfyanonymous committed
116
117
            return cond_equal_size(c1[2], c2[2])

comfyanonymous's avatar
comfyanonymous committed
118
119
120
        def cond_cat(c_list):
            c_crossattn = []
            c_concat = []
121
            c_adm = []
comfyanonymous's avatar
comfyanonymous committed
122
            crossattn_max_len = 0
123
124

            temp = {}
comfyanonymous's avatar
comfyanonymous committed
125
            for x in c_list:
126
127
128
129
130
                for k in x:
                    cur = temp.get(k, [])
                    cur.append(x[k])
                    temp[k] = cur

comfyanonymous's avatar
comfyanonymous committed
131
            out = {}
132
133
134
135
            for k in temp:
                conds = temp[k]
                out[k] = conds[0].concat(conds[1:])

comfyanonymous's avatar
comfyanonymous committed
136
137
            return out

138
        def calc_cond_uncond_batch(model, cond, uncond, x_in, timestep, model_options):
comfyanonymous's avatar
comfyanonymous committed
139
            out_cond = torch.zeros_like(x_in)
comfyanonymous's avatar
comfyanonymous committed
140
            out_count = torch.ones_like(x_in) * 1e-37
141
142

            out_uncond = torch.zeros_like(x_in)
comfyanonymous's avatar
comfyanonymous committed
143
            out_uncond_count = torch.ones_like(x_in) * 1e-37
144
145
146

            COND = 0
            UNCOND = 1
comfyanonymous's avatar
comfyanonymous committed
147

148
            to_run = []
comfyanonymous's avatar
comfyanonymous committed
149
            for x in cond:
150
                p = get_area_and_mult(x, x_in, timestep)
151
                if p is None:
comfyanonymous's avatar
comfyanonymous committed
152
                    continue
153
154

                to_run += [(p, COND)]
155
156
            if uncond is not None:
                for x in uncond:
157
                    p = get_area_and_mult(x, x_in, timestep)
158
159
                    if p is None:
                        continue
160

161
                    to_run += [(p, UNCOND)]
162
163
164
165

            while len(to_run) > 0:
                first = to_run[0]
                first_shape = first[0][0].shape
166
                to_batch_temp = []
167
                for x in range(len(to_run)):
comfyanonymous's avatar
comfyanonymous committed
168
169
                    if can_concat_cond(to_run[x][0], first[0]):
                        to_batch_temp += [x]
170
171
172
173

                to_batch_temp.reverse()
                to_batch = to_batch_temp[:1]

174
                free_memory = model_management.get_free_memory(x_in.device)
175
176
                for i in range(1, len(to_batch_temp) + 1):
                    batch_amount = to_batch_temp[:len(to_batch_temp)//i]
177
178
                    input_shape = [len(batch_amount) * first_shape[0]] + list(first_shape)[1:]
                    if model.memory_required(input_shape) < free_memory:
179
180
                        to_batch = batch_amount
                        break
181
182
183
184
185
186

                input_x = []
                mult = []
                c = []
                cond_or_uncond = []
                area = []
comfyanonymous's avatar
comfyanonymous committed
187
                control = None
188
                patches = None
189
190
191
192
193
194
195
196
                for x in to_batch:
                    o = to_run.pop(x)
                    p = o[0]
                    input_x += [p[0]]
                    mult += [p[1]]
                    c += [p[2]]
                    area += [p[3]]
                    cond_or_uncond += [o[1]]
comfyanonymous's avatar
comfyanonymous committed
197
                    control = p[4]
198
                    patches = p[5]
199
200
201

                batch_chunks = len(cond_or_uncond)
                input_x = torch.cat(input_x)
comfyanonymous's avatar
comfyanonymous committed
202
                c = cond_cat(c)
comfyanonymous's avatar
comfyanonymous committed
203
                timestep_ = torch.cat([timestep] * batch_chunks)
204

comfyanonymous's avatar
comfyanonymous committed
205
                if control is not None:
206
                    c['control'] = control.get_control(input_x, timestep_, c, len(cond_or_uncond))
comfyanonymous's avatar
comfyanonymous committed
207

208
                transformer_options = {}
209
                if 'transformer_options' in model_options:
210
211
212
                    transformer_options = model_options['transformer_options'].copy()

                if patches is not None:
213
214
215
216
217
218
219
220
221
                    if "patches" in transformer_options:
                        cur_patches = transformer_options["patches"].copy()
                        for p in patches:
                            if p in cur_patches:
                                cur_patches[p] = cur_patches[p] + patches[p]
                            else:
                                cur_patches[p] = patches[p]
                    else:
                        transformer_options["patches"] = patches
222

223
                transformer_options["cond_or_uncond"] = cond_or_uncond[:]
224
225
                transformer_options["sigmas"] = timestep

226
                c['transformer_options'] = transformer_options
227

228
                if 'model_function_wrapper' in model_options:
229
                    output = model_options['model_function_wrapper'](model.apply_model, {"input": input_x, "timestep": timestep_, "c": c, "cond_or_uncond": cond_or_uncond}).chunk(batch_chunks)
230
                else:
231
                    output = model.apply_model(input_x, timestep_, **c).chunk(batch_chunks)
comfyanonymous's avatar
comfyanonymous committed
232
                del input_x
233
234
235
236
237
238
239
240

                for o in range(batch_chunks):
                    if cond_or_uncond[o] == COND:
                        out_cond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
                        out_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
                    else:
                        out_uncond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
                        out_uncond_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
comfyanonymous's avatar
comfyanonymous committed
241
242
243
244
                del mult

            out_cond /= out_count
            del out_count
245
246
247
            out_uncond /= out_uncond_count
            del out_uncond_count
            return out_cond, out_uncond
comfyanonymous's avatar
comfyanonymous committed
248
249


250
251
        # if we're doing SAG, we still need to do uncond guidance, even though the cond and uncond will cancel out.
        if math.isclose(cond_scale, 1.0) and "sag" not in model_options:
252
253
            uncond = None

254
255
        cond_pred, uncond_pred = calc_cond_uncond_batch(model, cond, uncond, x, timestep, model_options)
        cfg_result = uncond_pred + (cond_pred - uncond_pred) * cond_scale
256
        if "sampler_cfg_function" in model_options:
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
            args = {"cond": x - cond_pred, "uncond": x - uncond_pred, "cond_scale": cond_scale, "timestep": timestep, "input": x, "sigma": timestep}
            cfg_result = x - model_options["sampler_cfg_function"](args)

        if "sag" in model_options:
            assert uncond is not None, "SAG requires uncond guidance"
            sag_scale = model_options["sag_scale"]
            sag_sigma = model_options["sag_sigma"]
            sag_threshold = model_options.get("sag_threshold", 1.0)

            # these methods are added by the sag patcher
            uncond_attn = model.get_attn_scores()
            mid_shape = model.get_mid_block_shape()
            # create the adversarially blurred image
            degraded = create_blur_map(uncond_pred, uncond_attn, mid_shape, sag_sigma, sag_threshold)
            degraded_noised = degraded + x - uncond_pred
            # call into the UNet
            (sag, _) = calc_cond_uncond_batch(model, uncond, None, degraded_noised, timestep, model_options)
            cfg_result += (degraded - sag) * sag_scale
        return cfg_result

def create_blur_map(x0, attn, mid_shape, sigma=3.0, threshold=1.0):
    # reshape and GAP the attention map
    _, hw1, hw2 = attn.shape
    b, _, lh, lw = x0.shape
    attn = attn.reshape(b, -1, hw1, hw2)
    # Global Average Pool
    mask = attn.mean(1, keepdim=False).sum(1, keepdim=False) > threshold
    # Reshape
    mask = (
        mask.reshape(b, *mid_shape)
        .unsqueeze(1)
        .type(attn.dtype)
    )
    # Upsample
    mask = F.interpolate(mask, (lh, lw))

    blurred = gaussian_blur_2d(x0, kernel_size=9, sigma=sigma)
    blurred = blurred * mask + x0 * (1 - mask)
    return blurred

def gaussian_blur_2d(img, kernel_size, sigma):
    ksize_half = (kernel_size - 1) * 0.5

    x = torch.linspace(-ksize_half, ksize_half, steps=kernel_size)

    pdf = torch.exp(-0.5 * (x / sigma).pow(2))

    x_kernel = pdf / pdf.sum()
    x_kernel = x_kernel.to(device=img.device, dtype=img.dtype)

    kernel2d = torch.mm(x_kernel[:, None], x_kernel[None, :])
    kernel2d = kernel2d.expand(img.shape[-3], 1, kernel2d.shape[0], kernel2d.shape[1])

    padding = [kernel_size // 2, kernel_size // 2, kernel_size // 2, kernel_size // 2]

    img = F.pad(img, padding, mode="reflect")
    img = F.conv2d(img, kernel2d, groups=img.shape[-3])
    return img
comfyanonymous's avatar
comfyanonymous committed
315

comfyanonymous's avatar
comfyanonymous committed
316
317
318
319
class CFGNoisePredictor(torch.nn.Module):
    def __init__(self, model):
        super().__init__()
        self.inner_model = model
320
    def apply_model(self, x, timestep, cond, uncond, cond_scale, model_options={}, seed=None):
321
        out = sampling_function(self.inner_model, x, timestep, uncond, cond, cond_scale, model_options=model_options, seed=seed)
comfyanonymous's avatar
comfyanonymous committed
322
        return out
comfyanonymous's avatar
comfyanonymous committed
323
324
    def forward(self, *args, **kwargs):
        return self.apply_model(*args, **kwargs)
comfyanonymous's avatar
comfyanonymous committed
325
326

class KSamplerX0Inpaint(torch.nn.Module):
327
328
329
    def __init__(self, model):
        super().__init__()
        self.inner_model = model
330
    def forward(self, x, sigma, uncond, cond, cond_scale, denoise_mask, model_options={}, seed=None):
331
332
        if denoise_mask is not None:
            latent_mask = 1. - denoise_mask
333
            x = x * denoise_mask + (self.latent_image + self.noise * sigma.reshape([sigma.shape[0]] + [1] * (len(self.noise.shape) - 1))) * latent_mask
334
        out = self.inner_model(x, sigma, cond=cond, uncond=uncond, cond_scale=cond_scale, model_options=model_options, seed=seed)
335
        if denoise_mask is not None:
comfyanonymous's avatar
comfyanonymous committed
336
            out = out * denoise_mask + self.latent_image * latent_mask
337
        return out
338

comfyanonymous's avatar
comfyanonymous committed
339
def simple_scheduler(model, steps):
comfyanonymous's avatar
comfyanonymous committed
340
    s = model.model_sampling
comfyanonymous's avatar
comfyanonymous committed
341
    sigs = []
comfyanonymous's avatar
comfyanonymous committed
342
    ss = len(s.sigmas) / steps
comfyanonymous's avatar
comfyanonymous committed
343
    for x in range(steps):
comfyanonymous's avatar
comfyanonymous committed
344
        sigs += [float(s.sigmas[-(1 + int(x * ss))])]
comfyanonymous's avatar
comfyanonymous committed
345
346
347
    sigs += [0.0]
    return torch.FloatTensor(sigs)

comfyanonymous's avatar
comfyanonymous committed
348
def ddim_scheduler(model, steps):
comfyanonymous's avatar
comfyanonymous committed
349
    s = model.model_sampling
comfyanonymous's avatar
comfyanonymous committed
350
    sigs = []
comfyanonymous's avatar
comfyanonymous committed
351
352
353
354
355
356
    ss = len(s.sigmas) // steps
    x = 1
    while x < len(s.sigmas):
        sigs += [float(s.sigmas[x])]
        x += ss
    sigs = sigs[::-1]
comfyanonymous's avatar
comfyanonymous committed
357
358
359
    sigs += [0.0]
    return torch.FloatTensor(sigs)

comfyanonymous's avatar
comfyanonymous committed
360
361
362
363
364
365
366
367
368
369
def normal_scheduler(model, steps, sgm=False, floor=False):
    s = model.model_sampling
    start = s.timestep(s.sigma_max)
    end = s.timestep(s.sigma_min)

    if sgm:
        timesteps = torch.linspace(start, end, steps + 1)[:-1]
    else:
        timesteps = torch.linspace(start, end, steps)

370
371
372
    sigs = []
    for x in range(len(timesteps)):
        ts = timesteps[x]
comfyanonymous's avatar
comfyanonymous committed
373
        sigs.append(s.sigma(ts))
374
375
376
    sigs += [0.0]
    return torch.FloatTensor(sigs)

Jacob Segal's avatar
Jacob Segal committed
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
def get_mask_aabb(masks):
    if masks.numel() == 0:
        return torch.zeros((0, 4), device=masks.device, dtype=torch.int)

    b = masks.shape[0]

    bounding_boxes = torch.zeros((b, 4), device=masks.device, dtype=torch.int)
    is_empty = torch.zeros((b), device=masks.device, dtype=torch.bool)
    for i in range(b):
        mask = masks[i]
        if mask.numel() == 0:
            continue
        if torch.max(mask != 0) == False:
            is_empty[i] = True
            continue
        y, x = torch.where(mask)
        bounding_boxes[i, 0] = torch.min(x)
        bounding_boxes[i, 1] = torch.min(y)
        bounding_boxes[i, 2] = torch.max(x)
        bounding_boxes[i, 3] = torch.max(y)

    return bounding_boxes, is_empty

400
def resolve_areas_and_cond_masks(conditions, h, w, device):
Jacob Segal's avatar
Jacob Segal committed
401
402
403
404
    # We need to decide on an area outside the sampling loop in order to properly generate opposite areas of equal sizes.
    # While we're doing this, we can also resolve the mask device and scaling for performance reasons
    for i in range(len(conditions)):
        c = conditions[i]
405
406
        if 'area' in c:
            area = c['area']
407
            if area[0] == "percentage":
408
                modified = c.copy()
409
410
                area = (max(1, round(area[1] * h)), max(1, round(area[2] * w)), round(area[3] * h), round(area[4] * w))
                modified['area'] = area
411
                c = modified
412
413
                conditions[i] = c

414
415
        if 'mask' in c:
            mask = c['mask']
Jacob Segal's avatar
Jacob Segal committed
416
            mask = mask.to(device=device)
417
            modified = c.copy()
Jacob Segal's avatar
Jacob Segal committed
418
419
            if len(mask.shape) == 2:
                mask = mask.unsqueeze(0)
mara's avatar
mara committed
420
            if mask.shape[1] != h or mask.shape[2] != w:
Jacob Segal's avatar
Jacob Segal committed
421
422
                mask = torch.nn.functional.interpolate(mask.unsqueeze(1), size=(h, w), mode='bilinear', align_corners=False).squeeze(1)

Jacob Segal's avatar
Jacob Segal committed
423
            if modified.get("set_area_to_bounds", False):
Jacob Segal's avatar
Jacob Segal committed
424
                bounds = torch.max(torch.abs(mask),dim=0).values.unsqueeze(0)
Jacob Segal's avatar
Jacob Segal committed
425
426
427
428
                boxes, is_empty = get_mask_aabb(bounds)
                if is_empty[0]:
                    # Use the minimum possible size for efficiency reasons. (Since the mask is all-0, this becomes a noop anyway)
                    modified['area'] = (8, 8, 0, 0)
Jacob Segal's avatar
Jacob Segal committed
429
                else:
Jacob Segal's avatar
Jacob Segal committed
430
                    box = boxes[0]
Jacob Segal's avatar
Jacob Segal committed
431
                    H, W, Y, X = (box[3] - box[1] + 1, box[2] - box[0] + 1, box[1], box[0])
432
433
                    H = max(8, H)
                    W = max(8, W)
Jacob Segal's avatar
Jacob Segal committed
434
435
                    area = (int(H), int(W), int(Y), int(X))
                    modified['area'] = area
Jacob Segal's avatar
Jacob Segal committed
436
437

            modified['mask'] = mask
438
            conditions[i] = modified
Jacob Segal's avatar
Jacob Segal committed
439

comfyanonymous's avatar
comfyanonymous committed
440
def create_cond_with_same_area_if_none(conds, c):
441
    if 'area' not in c:
comfyanonymous's avatar
comfyanonymous committed
442
443
        return

444
    c_area = c['area']
comfyanonymous's avatar
comfyanonymous committed
445
446
    smallest = None
    for x in conds:
447
448
        if 'area' in x:
            a = x['area']
comfyanonymous's avatar
comfyanonymous committed
449
450
451
452
453
            if c_area[2] >= a[2] and c_area[3] >= a[3]:
                if a[0] + a[2] >= c_area[0] + c_area[2]:
                    if a[1] + a[3] >= c_area[1] + c_area[3]:
                        if smallest is None:
                            smallest = x
454
                        elif 'area' not in smallest:
comfyanonymous's avatar
comfyanonymous committed
455
456
                            smallest = x
                        else:
457
                            if smallest['area'][0] * smallest['area'][1] > a[0] * a[1]:
comfyanonymous's avatar
comfyanonymous committed
458
459
460
461
462
463
                                smallest = x
        else:
            if smallest is None:
                smallest = x
    if smallest is None:
        return
464
465
    if 'area' in smallest:
        if smallest['area'] == c_area:
comfyanonymous's avatar
comfyanonymous committed
466
            return
467
468
469
470

    out = c.copy()
    out['model_conds'] = smallest['model_conds'].copy() #TODO: which fields should be copied?
    conds += [out]
comfyanonymous's avatar
comfyanonymous committed
471

472
def calculate_start_end_timesteps(model, conds):
473
    s = model.model_sampling
474
475
476
477
478
    for t in range(len(conds)):
        x = conds[t]

        timestep_start = None
        timestep_end = None
479
        if 'start_percent' in x:
480
            timestep_start = s.percent_to_sigma(x['start_percent'])
481
        if 'end_percent' in x:
482
            timestep_end = s.percent_to_sigma(x['end_percent'])
483
484

        if (timestep_start is not None) or (timestep_end is not None):
485
            n = x.copy()
486
487
488
489
            if (timestep_start is not None):
                n['timestep_start'] = timestep_start
            if (timestep_end is not None):
                n['timestep_end'] = timestep_end
490
            conds[t] = n
491

492
def pre_run_control(model, conds):
493
    s = model.model_sampling
494
495
496
497
498
    for t in range(len(conds)):
        x = conds[t]

        timestep_start = None
        timestep_end = None
499
        percent_to_timestep_function = lambda a: s.percent_to_sigma(a)
500
        if 'control' in x:
501
            x['control'].pre_run(model, percent_to_timestep_function)
502

503
def apply_empty_x_to_equal_area(conds, uncond, name, uncond_fill_func):
comfyanonymous's avatar
comfyanonymous committed
504
505
506
507
508
509
    cond_cnets = []
    cond_other = []
    uncond_cnets = []
    uncond_other = []
    for t in range(len(conds)):
        x = conds[t]
510
511
512
        if 'area' not in x:
            if name in x and x[name] is not None:
                cond_cnets.append(x[name])
comfyanonymous's avatar
comfyanonymous committed
513
514
515
516
            else:
                cond_other.append((x, t))
    for t in range(len(uncond)):
        x = uncond[t]
517
518
519
        if 'area' not in x:
            if name in x and x[name] is not None:
                uncond_cnets.append(x[name])
comfyanonymous's avatar
comfyanonymous committed
520
521
522
523
524
525
526
527
528
            else:
                uncond_other.append((x, t))

    if len(uncond_cnets) > 0:
        return

    for x in range(len(cond_cnets)):
        temp = uncond_other[x % len(uncond_other)]
        o = temp[0]
529
530
        if name in o and o[name] is not None:
            n = o.copy()
531
            n[name] = uncond_fill_func(cond_cnets, x)
532
            uncond += [n]
comfyanonymous's avatar
comfyanonymous committed
533
        else:
534
            n = o.copy()
535
            n[name] = uncond_fill_func(cond_cnets, x)
536
            uncond[temp[1]] = n
537

538
def encode_model_conds(model_function, conds, noise, device, prompt_type, **kwargs):
539
540
    for t in range(len(conds)):
        x = conds[t]
541
        params = x.copy()
542
        params["device"] = device
543
544
545
546
        params["noise"] = noise
        params["width"] = params.get("width", noise.shape[3] * 8)
        params["height"] = params.get("height", noise.shape[2] * 8)
        params["prompt_type"] = params.get("prompt_type", prompt_type)
547
548
549
550
551
        for k in kwargs:
            if k not in params:
                params[k] = kwargs[k]

        out = model_function(**params)
552
553
554
555
556
557
        x = x.copy()
        model_conds = x['model_conds'].copy()
        for k in out:
            model_conds[k] = out[k]
        x['model_conds'] = model_conds
        conds[t] = x
558
    return conds
559

comfyanonymous's avatar
comfyanonymous committed
560
561
562
563
564
class Sampler:
    def sample(self):
        pass

    def max_denoise(self, model_wrap, sigmas):
comfyanonymous's avatar
comfyanonymous committed
565
566
567
        max_sigma = float(model_wrap.inner_model.model_sampling.sigma_max)
        sigma = float(sigmas[0])
        return math.isclose(max_sigma, sigma, rel_tol=1e-05) or sigma > max_sigma
comfyanonymous's avatar
comfyanonymous committed
568
569
570

class UNIPC(Sampler):
    def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
571
        return uni_pc.sample_unipc(model_wrap, noise, latent_image, sigmas, max_denoise=self.max_denoise(model_wrap, sigmas), extra_args=extra_args, noise_mask=denoise_mask, callback=callback, disable=disable_pbar)
comfyanonymous's avatar
comfyanonymous committed
572
573
574

class UNIPCBH2(Sampler):
    def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
575
        return uni_pc.sample_unipc(model_wrap, noise, latent_image, sigmas, max_denoise=self.max_denoise(model_wrap, sigmas), extra_args=extra_args, noise_mask=denoise_mask, callback=callback, variant='bh2', disable=disable_pbar)
comfyanonymous's avatar
comfyanonymous committed
576

comfyanonymous's avatar
comfyanonymous committed
577
KSAMPLER_NAMES = ["euler", "euler_ancestral", "heun", "heunpp2","dpm_2", "dpm_2_ancestral",
comfyanonymous's avatar
comfyanonymous committed
578
                  "lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu",
comfyanonymous's avatar
comfyanonymous committed
579
                  "dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm"]
comfyanonymous's avatar
comfyanonymous committed
580

581
582
583
584
585
class KSAMPLER(Sampler):
    def __init__(self, sampler_function, extra_options={}, inpaint_options={}):
        self.sampler_function = sampler_function
        self.extra_options = extra_options
        self.inpaint_options = inpaint_options
comfyanonymous's avatar
comfyanonymous committed
586

587
588
589
590
591
592
593
594
595
    def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
        extra_args["denoise_mask"] = denoise_mask
        model_k = KSamplerX0Inpaint(model_wrap)
        model_k.latent_image = latent_image
        if self.inpaint_options.get("random", False): #TODO: Should this be the default?
            generator = torch.manual_seed(extra_args.get("seed", 41) + 1)
            model_k.noise = torch.randn(noise.shape, generator=generator, device="cpu").to(noise.dtype).to(noise.device)
        else:
            model_k.noise = noise
comfyanonymous's avatar
comfyanonymous committed
596

597
598
599
600
601
602
603
604
605
606
607
608
        if self.max_denoise(model_wrap, sigmas):
            noise = noise * torch.sqrt(1.0 + sigmas[0] ** 2.0)
        else:
            noise = noise * sigmas[0]

        k_callback = None
        total_steps = len(sigmas) - 1
        if callback is not None:
            k_callback = lambda x: callback(x["i"], x["denoised"], x["x"], total_steps)

        if latent_image is not None:
            noise += latent_image
comfyanonymous's avatar
comfyanonymous committed
609

610
611
612
613
614
615
616
        samples = self.sampler_function(model_k, noise, sigmas, extra_args=extra_args, callback=k_callback, disable=disable_pbar, **self.extra_options)
        return samples


def ksampler(sampler_name, extra_options={}, inpaint_options={}):
    if sampler_name == "dpm_fast":
        def dpm_fast_function(model, noise, sigmas, extra_args, callback, disable):
comfyanonymous's avatar
comfyanonymous committed
617
618
619
            sigma_min = sigmas[-1]
            if sigma_min == 0:
                sigma_min = sigmas[-2]
620
621
622
623
624
625
626
627
628
629
630
631
            total_steps = len(sigmas) - 1
            return k_diffusion_sampling.sample_dpm_fast(model, noise, sigma_min, sigmas[0], total_steps, extra_args=extra_args, callback=callback, disable=disable)
        sampler_function = dpm_fast_function
    elif sampler_name == "dpm_adaptive":
        def dpm_adaptive_function(model, noise, sigmas, extra_args, callback, disable):
            sigma_min = sigmas[-1]
            if sigma_min == 0:
                sigma_min = sigmas[-2]
            return k_diffusion_sampling.sample_dpm_adaptive(model, noise, sigma_min, sigmas[0], extra_args=extra_args, callback=callback, disable=disable)
        sampler_function = dpm_adaptive_function
    else:
        sampler_function = getattr(k_diffusion_sampling, "sample_{}".format(sampler_name))
comfyanonymous's avatar
comfyanonymous committed
632

633
    return KSAMPLER(sampler_function, extra_options, inpaint_options)
comfyanonymous's avatar
comfyanonymous committed
634

comfyanonymous's avatar
comfyanonymous committed
635
636
def wrap_model(model):
    model_denoise = CFGNoisePredictor(model)
comfyanonymous's avatar
comfyanonymous committed
637
    return model_denoise
comfyanonymous's avatar
comfyanonymous committed
638
639
640
641
642
643
644
645

def sample(model, noise, positive, negative, cfg, device, sampler, sigmas, model_options={}, latent_image=None, denoise_mask=None, callback=None, disable_pbar=False, seed=None):
    positive = positive[:]
    negative = negative[:]

    resolve_areas_and_cond_masks(positive, noise.shape[2], noise.shape[3], device)
    resolve_areas_and_cond_masks(negative, noise.shape[2], noise.shape[3], device)

comfyanonymous's avatar
comfyanonymous committed
646
    model_wrap = wrap_model(model)
comfyanonymous's avatar
comfyanonymous committed
647

648
649
    calculate_start_end_timesteps(model, negative)
    calculate_start_end_timesteps(model, positive)
comfyanonymous's avatar
comfyanonymous committed
650
651
652
653
654
655
656

    #make sure each cond area has an opposite one with the same area
    for c in positive:
        create_cond_with_same_area_if_none(negative, c)
    for c in negative:
        create_cond_with_same_area_if_none(positive, c)

657
    pre_run_control(model, negative + positive)
comfyanonymous's avatar
comfyanonymous committed
658

659
    apply_empty_x_to_equal_area(list(filter(lambda c: c.get('control_apply_to_uncond', False) == True, positive)), negative, 'control', lambda cond_cnets, x: cond_cnets[x])
comfyanonymous's avatar
comfyanonymous committed
660
661
    apply_empty_x_to_equal_area(positive, negative, 'gligen', lambda cond_cnets, x: cond_cnets[x])

662
663
664
    if latent_image is not None:
        latent_image = model.process_latent_in(latent_image)

665
666
667
    if hasattr(model, 'extra_conds'):
        positive = encode_model_conds(model.extra_conds, positive, noise, device, "positive", latent_image=latent_image, denoise_mask=denoise_mask)
        negative = encode_model_conds(model.extra_conds, negative, noise, device, "negative", latent_image=latent_image, denoise_mask=denoise_mask)
comfyanonymous's avatar
comfyanonymous committed
668
669
670
671
672
673

    extra_args = {"cond":positive, "uncond":negative, "cond_scale": cfg, "model_options": model_options, "seed":seed}

    samples = sampler.sample(model_wrap, sigmas, extra_args, callback, noise, latent_image, denoise_mask, disable_pbar)
    return model.process_latent_out(samples.to(torch.float32))

comfyanonymous's avatar
comfyanonymous committed
674
675
676
677
678
SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform"]
SAMPLER_NAMES = KSAMPLER_NAMES + ["ddim", "uni_pc", "uni_pc_bh2"]

def calculate_sigmas_scheduler(model, scheduler_name, steps):
    if scheduler_name == "karras":
comfyanonymous's avatar
comfyanonymous committed
679
        sigmas = k_diffusion_sampling.get_sigmas_karras(n=steps, sigma_min=float(model.model_sampling.sigma_min), sigma_max=float(model.model_sampling.sigma_max))
comfyanonymous's avatar
comfyanonymous committed
680
    elif scheduler_name == "exponential":
comfyanonymous's avatar
comfyanonymous committed
681
        sigmas = k_diffusion_sampling.get_sigmas_exponential(n=steps, sigma_min=float(model.model_sampling.sigma_min), sigma_max=float(model.model_sampling.sigma_max))
comfyanonymous's avatar
comfyanonymous committed
682
    elif scheduler_name == "normal":
comfyanonymous's avatar
comfyanonymous committed
683
        sigmas = normal_scheduler(model, steps)
comfyanonymous's avatar
comfyanonymous committed
684
    elif scheduler_name == "simple":
comfyanonymous's avatar
comfyanonymous committed
685
        sigmas = simple_scheduler(model, steps)
comfyanonymous's avatar
comfyanonymous committed
686
    elif scheduler_name == "ddim_uniform":
comfyanonymous's avatar
comfyanonymous committed
687
        sigmas = ddim_scheduler(model, steps)
comfyanonymous's avatar
comfyanonymous committed
688
    elif scheduler_name == "sgm_uniform":
comfyanonymous's avatar
comfyanonymous committed
689
        sigmas = normal_scheduler(model, steps, sgm=True)
comfyanonymous's avatar
comfyanonymous committed
690
691
692
693
    else:
        print("error invalid scheduler", self.scheduler)
    return sigmas

694
def sampler_object(name):
695
    if name == "uni_pc":
696
        sampler = UNIPC()
697
    elif name == "uni_pc_bh2":
698
        sampler = UNIPCBH2()
699
    elif name == "ddim":
700
        sampler = ksampler("euler", inpaint_options={"random": True})
701
702
703
704
    else:
        sampler = ksampler(name)
    return sampler

comfyanonymous's avatar
comfyanonymous committed
705
class KSampler:
comfyanonymous's avatar
comfyanonymous committed
706
707
    SCHEDULERS = SCHEDULER_NAMES
    SAMPLERS = SAMPLER_NAMES
comfyanonymous's avatar
comfyanonymous committed
708

709
    def __init__(self, model, steps, device, sampler=None, scheduler=None, denoise=None, model_options={}):
comfyanonymous's avatar
comfyanonymous committed
710
711
712
713
714
715
716
717
718
        self.model = model
        self.device = device
        if scheduler not in self.SCHEDULERS:
            scheduler = self.SCHEDULERS[0]
        if sampler not in self.SAMPLERS:
            sampler = self.SAMPLERS[0]
        self.scheduler = scheduler
        self.sampler = sampler
        self.set_steps(steps, denoise)
719
        self.denoise = denoise
720
        self.model_options = model_options
comfyanonymous's avatar
comfyanonymous committed
721

comfyanonymous's avatar
comfyanonymous committed
722
723
724
725
    def calculate_sigmas(self, steps):
        sigmas = None

        discard_penultimate_sigma = False
726
        if self.sampler in ['dpm_2', 'dpm_2_ancestral', 'uni_pc', 'uni_pc_bh2']:
comfyanonymous's avatar
comfyanonymous committed
727
728
729
            steps += 1
            discard_penultimate_sigma = True

comfyanonymous's avatar
comfyanonymous committed
730
        sigmas = calculate_sigmas_scheduler(self.model, self.scheduler, steps)
comfyanonymous's avatar
comfyanonymous committed
731
732
733
734
735

        if discard_penultimate_sigma:
            sigmas = torch.cat([sigmas[:-2], sigmas[-1:]])
        return sigmas

comfyanonymous's avatar
comfyanonymous committed
736
737
    def set_steps(self, steps, denoise=None):
        self.steps = steps
738
        if denoise is None or denoise > 0.9999:
comfyanonymous's avatar
comfyanonymous committed
739
            self.sigmas = self.calculate_sigmas(steps).to(self.device)
comfyanonymous's avatar
comfyanonymous committed
740
741
        else:
            new_steps = int(steps/denoise)
comfyanonymous's avatar
comfyanonymous committed
742
            sigmas = self.calculate_sigmas(new_steps).to(self.device)
comfyanonymous's avatar
comfyanonymous committed
743
744
            self.sigmas = sigmas[-(steps + 1):]

745
    def sample(self, noise, positive, negative, cfg, latent_image=None, start_step=None, last_step=None, force_full_denoise=False, denoise_mask=None, sigmas=None, callback=None, disable_pbar=False, seed=None):
746
747
        if sigmas is None:
            sigmas = self.sigmas
comfyanonymous's avatar
comfyanonymous committed
748

comfyanonymous's avatar
comfyanonymous committed
749
        if last_step is not None and last_step < (len(sigmas) - 1):
comfyanonymous's avatar
comfyanonymous committed
750
            sigmas = sigmas[:last_step + 1]
comfyanonymous's avatar
comfyanonymous committed
751
752
753
            if force_full_denoise:
                sigmas[-1] = 0

comfyanonymous's avatar
comfyanonymous committed
754
        if start_step is not None:
comfyanonymous's avatar
comfyanonymous committed
755
756
757
758
759
760
761
            if start_step < (len(sigmas) - 1):
                sigmas = sigmas[start_step:]
            else:
                if latent_image is not None:
                    return latent_image
                else:
                    return torch.zeros_like(noise)
comfyanonymous's avatar
comfyanonymous committed
762

763
        sampler = sampler_object(self.sampler)
764

765
        return sample(self.model, noise, positive, negative, cfg, self.device, sampler, sigmas, self.model_options, latent_image=latent_image, denoise_mask=denoise_mask, callback=callback, disable_pbar=disable_pbar, seed=seed)