attention.py 22.9 KB
Newer Older
Patrick von Platen's avatar
Patrick von Platen committed
1
# Copyright 2023 The HuggingFace Team. All rights reserved.
2
3
4
5
6
7
8
9
10
11
12
13
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
14
from typing import Any, Dict, Optional
15
16
17
18

import torch
from torch import nn

19
from ..utils import USE_PEFT_BACKEND
Dhruv Nair's avatar
Dhruv Nair committed
20
from ..utils.torch_utils import maybe_allow_in_graph
21
from .activations import GEGLU, GELU, ApproximateGELU
Patrick von Platen's avatar
Patrick von Platen committed
22
from .attention_processor import Attention
Dhruv Nair's avatar
Dhruv Nair committed
23
from .embeddings import SinusoidalPositionalEmbedding
24
from .lora import LoRACompatibleLinear
25
from .normalization import AdaLayerNorm, AdaLayerNormZero
26
27


Suraj Patil's avatar
Suraj Patil committed
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
def _chunked_feed_forward(
    ff: nn.Module, hidden_states: torch.Tensor, chunk_dim: int, chunk_size: int, lora_scale: Optional[float] = None
):
    # "feed_forward_chunk_size" can be used to save memory
    if hidden_states.shape[chunk_dim] % chunk_size != 0:
        raise ValueError(
            f"`hidden_states` dimension to be chunked: {hidden_states.shape[chunk_dim]} has to be divisible by chunk size: {chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."
        )

    num_chunks = hidden_states.shape[chunk_dim] // chunk_size
    if lora_scale is None:
        ff_output = torch.cat(
            [ff(hid_slice) for hid_slice in hidden_states.chunk(num_chunks, dim=chunk_dim)],
            dim=chunk_dim,
        )
    else:
        # TOOD(Patrick): LoRA scale can be removed once PEFT refactor is complete
        ff_output = torch.cat(
            [ff(hid_slice, scale=lora_scale) for hid_slice in hidden_states.chunk(num_chunks, dim=chunk_dim)],
            dim=chunk_dim,
        )

    return ff_output


53
54
@maybe_allow_in_graph
class GatedSelfAttentionDense(nn.Module):
55
56
57
58
59
60
61
62
63
64
65
    r"""
    A gated self-attention dense layer that combines visual features and object features.

    Parameters:
        query_dim (`int`): The number of channels in the query.
        context_dim (`int`): The number of channels in the context.
        n_heads (`int`): The number of heads to use for attention.
        d_head (`int`): The number of channels in each head.
    """

    def __init__(self, query_dim: int, context_dim: int, n_heads: int, d_head: int):
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
        super().__init__()

        # we need a linear projection since we need cat visual feature and obj feature
        self.linear = nn.Linear(context_dim, query_dim)

        self.attn = Attention(query_dim=query_dim, heads=n_heads, dim_head=d_head)
        self.ff = FeedForward(query_dim, activation_fn="geglu")

        self.norm1 = nn.LayerNorm(query_dim)
        self.norm2 = nn.LayerNorm(query_dim)

        self.register_parameter("alpha_attn", nn.Parameter(torch.tensor(0.0)))
        self.register_parameter("alpha_dense", nn.Parameter(torch.tensor(0.0)))

        self.enabled = True

82
    def forward(self, x: torch.Tensor, objs: torch.Tensor) -> torch.Tensor:
83
84
85
86
87
88
89
90
91
92
93
94
        if not self.enabled:
            return x

        n_visual = x.shape[1]
        objs = self.linear(objs)

        x = x + self.alpha_attn.tanh() * self.attn(self.norm1(torch.cat([x, objs], dim=1)))[:, :n_visual, :]
        x = x + self.alpha_dense.tanh() * self.ff(self.norm2(x))

        return x


95
@maybe_allow_in_graph
Patrick von Platen's avatar
Patrick von Platen committed
96
class BasicTransformerBlock(nn.Module):
Kashif Rasul's avatar
Kashif Rasul committed
97
98
99
100
    r"""
    A basic Transformer block.

    Parameters:
Will Berman's avatar
Will Berman committed
101
102
103
104
        dim (`int`): The number of channels in the input and output.
        num_attention_heads (`int`): The number of heads to use for multi-head attention.
        attention_head_dim (`int`): The number of channels in each head.
        dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
Will Berman's avatar
Will Berman committed
105
        cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention.
Will Berman's avatar
Will Berman committed
106
107
108
109
110
        activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.
        num_embeds_ada_norm (:
            obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`.
        attention_bias (:
            obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter.
111
112
113
114
115
116
117
118
119
120
121
122
123
124
        only_cross_attention (`bool`, *optional*):
            Whether to use only cross-attention layers. In this case two cross attention layers are used.
        double_self_attention (`bool`, *optional*):
            Whether to use two self-attention layers. In this case no cross attention layers are used.
        upcast_attention (`bool`, *optional*):
            Whether to upcast the attention computation to float32. This is useful for mixed precision training.
        norm_elementwise_affine (`bool`, *optional*, defaults to `True`):
            Whether to use learnable elementwise affine parameters for normalization.
        norm_type (`str`, *optional*, defaults to `"layer_norm"`):
            The normalization layer to use. Can be `"layer_norm"`, `"ada_norm"` or `"ada_norm_zero"`.
        final_dropout (`bool` *optional*, defaults to False):
            Whether to apply a final dropout after the last feed-forward layer.
        attention_type (`str`, *optional*, defaults to `"default"`):
            The type of attention to use. Can be `"default"` or `"gated"` or `"gated-text-image"`.
Dhruv Nair's avatar
Dhruv Nair committed
125
126
127
128
        positional_embeddings (`str`, *optional*, defaults to `None`):
            The type of positional embeddings to apply to.
        num_positional_embeddings (`int`, *optional*, defaults to `None`):
            The maximum number of positional embeddings to apply.
Kashif Rasul's avatar
Kashif Rasul committed
129
130
131
132
133
    """

    def __init__(
        self,
        dim: int,
Will Berman's avatar
Will Berman committed
134
135
        num_attention_heads: int,
        attention_head_dim: int,
Kashif Rasul's avatar
Kashif Rasul committed
136
        dropout=0.0,
Will Berman's avatar
Will Berman committed
137
138
139
140
        cross_attention_dim: Optional[int] = None,
        activation_fn: str = "geglu",
        num_embeds_ada_norm: Optional[int] = None,
        attention_bias: bool = False,
141
        only_cross_attention: bool = False,
142
        double_self_attention: bool = False,
143
        upcast_attention: bool = False,
Kashif Rasul's avatar
Kashif Rasul committed
144
        norm_elementwise_affine: bool = True,
Sayak Paul's avatar
Sayak Paul committed
145
146
        norm_type: str = "layer_norm",  # 'layer_norm', 'ada_norm', 'ada_norm_zero', 'ada_norm_single'
        norm_eps: float = 1e-5,
Kashif Rasul's avatar
Kashif Rasul committed
147
        final_dropout: bool = False,
148
        attention_type: str = "default",
Dhruv Nair's avatar
Dhruv Nair committed
149
150
        positional_embeddings: Optional[str] = None,
        num_positional_embeddings: Optional[int] = None,
Kashif Rasul's avatar
Kashif Rasul committed
151
    ):
Patrick von Platen's avatar
Patrick von Platen committed
152
        super().__init__()
153
        self.only_cross_attention = only_cross_attention
Kashif Rasul's avatar
Kashif Rasul committed
154
155
156

        self.use_ada_layer_norm_zero = (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
        self.use_ada_layer_norm = (num_embeds_ada_norm is not None) and norm_type == "ada_norm"
Sayak Paul's avatar
Sayak Paul committed
157
158
        self.use_ada_layer_norm_single = norm_type == "ada_norm_single"
        self.use_layer_norm = norm_type == "layer_norm"
Kashif Rasul's avatar
Kashif Rasul committed
159
160
161
162
163
164

        if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
            raise ValueError(
                f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
                f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."
            )
165

Dhruv Nair's avatar
Dhruv Nair committed
166
167
168
169
170
171
172
173
174
175
        if positional_embeddings and (num_positional_embeddings is None):
            raise ValueError(
                "If `positional_embedding` type is defined, `num_positition_embeddings` must also be defined."
            )

        if positional_embeddings == "sinusoidal":
            self.pos_embed = SinusoidalPositionalEmbedding(dim, max_seq_length=num_positional_embeddings)
        else:
            self.pos_embed = None

176
        # Define 3 blocks. Each block has its own normalization layer.
177
        # 1. Self-Attn
178
179
180
181
182
        if self.use_ada_layer_norm:
            self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm)
        elif self.use_ada_layer_norm_zero:
            self.norm1 = AdaLayerNormZero(dim, num_embeds_ada_norm)
        else:
Sayak Paul's avatar
Sayak Paul committed
183
184
            self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps)

Patrick von Platen's avatar
Patrick von Platen committed
185
        self.attn1 = Attention(
Will Berman's avatar
Will Berman committed
186
187
188
189
190
            query_dim=dim,
            heads=num_attention_heads,
            dim_head=attention_head_dim,
            dropout=dropout,
            bias=attention_bias,
191
            cross_attention_dim=cross_attention_dim if only_cross_attention else None,
192
            upcast_attention=upcast_attention,
193
194
        )

195
        # 2. Cross-Attn
196
        if cross_attention_dim is not None or double_self_attention:
197
198
199
200
201
202
            # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
            # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
            # the second cross attention block.
            self.norm2 = (
                AdaLayerNorm(dim, num_embeds_ada_norm)
                if self.use_ada_layer_norm
Sayak Paul's avatar
Sayak Paul committed
203
                else nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps)
204
            )
Patrick von Platen's avatar
Patrick von Platen committed
205
            self.attn2 = Attention(
206
                query_dim=dim,
207
                cross_attention_dim=cross_attention_dim if not double_self_attention else None,
208
209
210
211
                heads=num_attention_heads,
                dim_head=attention_head_dim,
                dropout=dropout,
                bias=attention_bias,
212
                upcast_attention=upcast_attention,
Will Berman's avatar
Will Berman committed
213
            )  # is self-attn if encoder_hidden_states is none
214
215
        else:
            self.norm2 = None
216
            self.attn2 = None
217
218

        # 3. Feed-forward
Sayak Paul's avatar
Sayak Paul committed
219
220
221
        if not self.use_ada_layer_norm_single:
            self.norm3 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps)

Suraj Patil's avatar
Suraj Patil committed
222
223
224
225
226
227
        self.ff = FeedForward(
            dim,
            dropout=dropout,
            activation_fn=activation_fn,
            final_dropout=final_dropout,
        )
Patrick von Platen's avatar
Patrick von Platen committed
228

229
        # 4. Fuser
230
        if attention_type == "gated" or attention_type == "gated-text-image":
231
232
            self.fuser = GatedSelfAttentionDense(dim, cross_attention_dim, num_attention_heads, attention_head_dim)

Sayak Paul's avatar
Sayak Paul committed
233
234
235
236
        # 5. Scale-shift for PixArt-Alpha.
        if self.use_ada_layer_norm_single:
            self.scale_shift_table = nn.Parameter(torch.randn(6, dim) / dim**0.5)

237
238
239
240
        # let chunk size default to None
        self._chunk_size = None
        self._chunk_dim = 0

Suraj Patil's avatar
Suraj Patil committed
241
    def set_chunk_feed_forward(self, chunk_size: Optional[int], dim: int = 0):
242
243
244
245
        # Sets chunk feed-forward
        self._chunk_size = chunk_size
        self._chunk_dim = dim

246
247
    def forward(
        self,
248
249
250
251
252
253
254
        hidden_states: torch.FloatTensor,
        attention_mask: Optional[torch.FloatTensor] = None,
        encoder_hidden_states: Optional[torch.FloatTensor] = None,
        encoder_attention_mask: Optional[torch.FloatTensor] = None,
        timestep: Optional[torch.LongTensor] = None,
        cross_attention_kwargs: Dict[str, Any] = None,
        class_labels: Optional[torch.LongTensor] = None,
255
    ) -> torch.FloatTensor:
256
        # Notice that normalization is always applied before the real computation in the following blocks.
257
        # 0. Self-Attention
Sayak Paul's avatar
Sayak Paul committed
258
259
        batch_size = hidden_states.shape[0]

Kashif Rasul's avatar
Kashif Rasul committed
260
261
262
263
264
265
        if self.use_ada_layer_norm:
            norm_hidden_states = self.norm1(hidden_states, timestep)
        elif self.use_ada_layer_norm_zero:
            norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(
                hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype
            )
Sayak Paul's avatar
Sayak Paul committed
266
        elif self.use_layer_norm:
Kashif Rasul's avatar
Kashif Rasul committed
267
            norm_hidden_states = self.norm1(hidden_states)
Sayak Paul's avatar
Sayak Paul committed
268
269
270
271
272
273
274
275
276
        elif self.use_ada_layer_norm_single:
            shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
                self.scale_shift_table[None] + timestep.reshape(batch_size, 6, -1)
            ).chunk(6, dim=1)
            norm_hidden_states = self.norm1(hidden_states)
            norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa
            norm_hidden_states = norm_hidden_states.squeeze(1)
        else:
            raise ValueError("Incorrect norm used")
Kashif Rasul's avatar
Kashif Rasul committed
277

Dhruv Nair's avatar
Dhruv Nair committed
278
279
280
        if self.pos_embed is not None:
            norm_hidden_states = self.pos_embed(norm_hidden_states)

281
282
283
284
        # 1. Retrieve lora scale.
        lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0

        # 2. Prepare GLIGEN inputs
285
286
        cross_attention_kwargs = cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {}
        gligen_kwargs = cross_attention_kwargs.pop("gligen", None)
287

288
289
290
291
292
293
        attn_output = self.attn1(
            norm_hidden_states,
            encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,
            attention_mask=attention_mask,
            **cross_attention_kwargs,
        )
Kashif Rasul's avatar
Kashif Rasul committed
294
295
        if self.use_ada_layer_norm_zero:
            attn_output = gate_msa.unsqueeze(1) * attn_output
Sayak Paul's avatar
Sayak Paul committed
296
297
298
        elif self.use_ada_layer_norm_single:
            attn_output = gate_msa * attn_output

299
        hidden_states = attn_output + hidden_states
Sayak Paul's avatar
Sayak Paul committed
300
301
        if hidden_states.ndim == 4:
            hidden_states = hidden_states.squeeze(1)
Will Berman's avatar
Will Berman committed
302

303
        # 2.5 GLIGEN Control
304
305
306
        if gligen_kwargs is not None:
            hidden_states = self.fuser(hidden_states, gligen_kwargs["objs"])

307
        # 3. Cross-Attention
308
        if self.attn2 is not None:
Sayak Paul's avatar
Sayak Paul committed
309
310
311
312
313
314
315
316
317
318
319
            if self.use_ada_layer_norm:
                norm_hidden_states = self.norm2(hidden_states, timestep)
            elif self.use_ada_layer_norm_zero or self.use_layer_norm:
                norm_hidden_states = self.norm2(hidden_states)
            elif self.use_ada_layer_norm_single:
                # For PixArt norm2 isn't applied here:
                # https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L70C1-L76C103
                norm_hidden_states = hidden_states
            else:
                raise ValueError("Incorrect norm")

320
            if self.pos_embed is not None and self.use_ada_layer_norm_single is False:
Dhruv Nair's avatar
Dhruv Nair committed
321
                norm_hidden_states = self.pos_embed(norm_hidden_states)
Kashif Rasul's avatar
Kashif Rasul committed
322

323
324
325
            attn_output = self.attn2(
                norm_hidden_states,
                encoder_hidden_states=encoder_hidden_states,
326
                attention_mask=encoder_attention_mask,
327
                **cross_attention_kwargs,
Will Berman's avatar
Will Berman committed
328
            )
329
            hidden_states = attn_output + hidden_states
Will Berman's avatar
Will Berman committed
330

331
        # 4. Feed-forward
Sayak Paul's avatar
Sayak Paul committed
332
333
        if not self.use_ada_layer_norm_single:
            norm_hidden_states = self.norm3(hidden_states)
Kashif Rasul's avatar
Kashif Rasul committed
334
335
336
337

        if self.use_ada_layer_norm_zero:
            norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

Sayak Paul's avatar
Sayak Paul committed
338
339
340
341
        if self.use_ada_layer_norm_single:
            norm_hidden_states = self.norm2(hidden_states)
            norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp

342
343
        if self._chunk_size is not None:
            # "feed_forward_chunk_size" can be used to save memory
Suraj Patil's avatar
Suraj Patil committed
344
345
            ff_output = _chunked_feed_forward(
                self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size, lora_scale=lora_scale
346
347
            )
        else:
348
            ff_output = self.ff(norm_hidden_states, scale=lora_scale)
Kashif Rasul's avatar
Kashif Rasul committed
349
350
351

        if self.use_ada_layer_norm_zero:
            ff_output = gate_mlp.unsqueeze(1) * ff_output
Sayak Paul's avatar
Sayak Paul committed
352
353
        elif self.use_ada_layer_norm_single:
            ff_output = gate_mlp * ff_output
Kashif Rasul's avatar
Kashif Rasul committed
354
355

        hidden_states = ff_output + hidden_states
Sayak Paul's avatar
Sayak Paul committed
356
357
        if hidden_states.ndim == 4:
            hidden_states = hidden_states.squeeze(1)
Will Berman's avatar
Will Berman committed
358

359
        return hidden_states
Patrick von Platen's avatar
Patrick von Platen committed
360
361


Suraj Patil's avatar
Suraj Patil committed
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
@maybe_allow_in_graph
class TemporalBasicTransformerBlock(nn.Module):
    r"""
    A basic Transformer block for video like data.

    Parameters:
        dim (`int`): The number of channels in the input and output.
        time_mix_inner_dim (`int`): The number of channels for temporal attention.
        num_attention_heads (`int`): The number of heads to use for multi-head attention.
        attention_head_dim (`int`): The number of channels in each head.
        cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention.
    """

    def __init__(
        self,
        dim: int,
        time_mix_inner_dim: int,
        num_attention_heads: int,
        attention_head_dim: int,
        cross_attention_dim: Optional[int] = None,
    ):
        super().__init__()
        self.is_res = dim == time_mix_inner_dim

        self.norm_in = nn.LayerNorm(dim)

        # Define 3 blocks. Each block has its own normalization layer.
        # 1. Self-Attn
        self.norm_in = nn.LayerNorm(dim)
        self.ff_in = FeedForward(
            dim,
            dim_out=time_mix_inner_dim,
            activation_fn="geglu",
        )

        self.norm1 = nn.LayerNorm(time_mix_inner_dim)
        self.attn1 = Attention(
            query_dim=time_mix_inner_dim,
            heads=num_attention_heads,
            dim_head=attention_head_dim,
            cross_attention_dim=None,
        )

        # 2. Cross-Attn
        if cross_attention_dim is not None:
            # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
            # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
            # the second cross attention block.
            self.norm2 = nn.LayerNorm(time_mix_inner_dim)
            self.attn2 = Attention(
                query_dim=time_mix_inner_dim,
                cross_attention_dim=cross_attention_dim,
                heads=num_attention_heads,
                dim_head=attention_head_dim,
            )  # is self-attn if encoder_hidden_states is none
        else:
            self.norm2 = None
            self.attn2 = None

        # 3. Feed-forward
        self.norm3 = nn.LayerNorm(time_mix_inner_dim)
        self.ff = FeedForward(time_mix_inner_dim, activation_fn="geglu")

        # let chunk size default to None
        self._chunk_size = None
        self._chunk_dim = None

    def set_chunk_feed_forward(self, chunk_size: Optional[int], **kwargs):
        # Sets chunk feed-forward
        self._chunk_size = chunk_size
        # chunk dim should be hardcoded to 1 to have better speed vs. memory trade-off
        self._chunk_dim = 1

    def forward(
        self,
        hidden_states: torch.FloatTensor,
        num_frames: int,
        encoder_hidden_states: Optional[torch.FloatTensor] = None,
    ) -> torch.FloatTensor:
        # Notice that normalization is always applied before the real computation in the following blocks.
        # 0. Self-Attention
        batch_size = hidden_states.shape[0]

        batch_frames, seq_length, channels = hidden_states.shape
        batch_size = batch_frames // num_frames

        hidden_states = hidden_states[None, :].reshape(batch_size, num_frames, seq_length, channels)
        hidden_states = hidden_states.permute(0, 2, 1, 3)
        hidden_states = hidden_states.reshape(batch_size * seq_length, num_frames, channels)

        residual = hidden_states
        hidden_states = self.norm_in(hidden_states)

        if self._chunk_size is not None:
            hidden_states = _chunked_feed_forward(self.ff, hidden_states, self._chunk_dim, self._chunk_size)
        else:
            hidden_states = self.ff_in(hidden_states)

        if self.is_res:
            hidden_states = hidden_states + residual

        norm_hidden_states = self.norm1(hidden_states)
        attn_output = self.attn1(norm_hidden_states, encoder_hidden_states=None)
        hidden_states = attn_output + hidden_states

        # 3. Cross-Attention
        if self.attn2 is not None:
            norm_hidden_states = self.norm2(hidden_states)
            attn_output = self.attn2(norm_hidden_states, encoder_hidden_states=encoder_hidden_states)
            hidden_states = attn_output + hidden_states

        # 4. Feed-forward
        norm_hidden_states = self.norm3(hidden_states)

        if self._chunk_size is not None:
            ff_output = _chunked_feed_forward(self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size)
        else:
            ff_output = self.ff(norm_hidden_states)

        if self.is_res:
            hidden_states = ff_output + hidden_states
        else:
            hidden_states = ff_output

        hidden_states = hidden_states[None, :].reshape(batch_size, seq_length, num_frames, channels)
        hidden_states = hidden_states.permute(0, 2, 1, 3)
        hidden_states = hidden_states.reshape(batch_size * num_frames, seq_length, channels)

        return hidden_states


Patrick von Platen's avatar
Patrick von Platen committed
493
class FeedForward(nn.Module):
Kashif Rasul's avatar
Kashif Rasul committed
494
495
496
497
    r"""
    A feed-forward layer.

    Parameters:
Will Berman's avatar
Will Berman committed
498
499
500
501
502
        dim (`int`): The number of channels in the input.
        dim_out (`int`, *optional*): The number of channels in the output. If not given, defaults to `dim`.
        mult (`int`, *optional*, defaults to 4): The multiplier to use for the hidden dimension.
        dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
        activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.
Kashif Rasul's avatar
Kashif Rasul committed
503
        final_dropout (`bool` *optional*, defaults to False): Apply a final dropout.
504
        bias (`bool`, defaults to True): Whether to use a bias in the linear layer.
Kashif Rasul's avatar
Kashif Rasul committed
505
506
507
    """

    def __init__(
Will Berman's avatar
Will Berman committed
508
509
510
511
512
513
        self,
        dim: int,
        dim_out: Optional[int] = None,
        mult: int = 4,
        dropout: float = 0.0,
        activation_fn: str = "geglu",
Kashif Rasul's avatar
Kashif Rasul committed
514
        final_dropout: bool = False,
515
        bias: bool = True,
Kashif Rasul's avatar
Kashif Rasul committed
516
    ):
Patrick von Platen's avatar
Patrick von Platen committed
517
518
        super().__init__()
        inner_dim = int(dim * mult)
519
        dim_out = dim_out if dim_out is not None else dim
520
        linear_cls = LoRACompatibleLinear if not USE_PEFT_BACKEND else nn.Linear
Patrick von Platen's avatar
Patrick von Platen committed
521

522
        if activation_fn == "gelu":
523
            act_fn = GELU(dim, inner_dim, bias=bias)
Kashif Rasul's avatar
Kashif Rasul committed
524
        if activation_fn == "gelu-approximate":
525
            act_fn = GELU(dim, inner_dim, approximate="tanh", bias=bias)
526
        elif activation_fn == "geglu":
527
            act_fn = GEGLU(dim, inner_dim, bias=bias)
Will Berman's avatar
Will Berman committed
528
        elif activation_fn == "geglu-approximate":
529
            act_fn = ApproximateGELU(dim, inner_dim, bias=bias)
Will Berman's avatar
Will Berman committed
530
531

        self.net = nn.ModuleList([])
532
        # project in
533
        self.net.append(act_fn)
534
535
536
        # project dropout
        self.net.append(nn.Dropout(dropout))
        # project out
537
        self.net.append(linear_cls(inner_dim, dim_out, bias=bias))
Kashif Rasul's avatar
Kashif Rasul committed
538
539
540
        # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
        if final_dropout:
            self.net.append(nn.Dropout(dropout))
Patrick von Platen's avatar
Patrick von Platen committed
541

542
    def forward(self, hidden_states: torch.Tensor, scale: float = 1.0) -> torch.Tensor:
543
        compatible_cls = (GEGLU,) if USE_PEFT_BACKEND else (GEGLU, LoRACompatibleLinear)
544
        for module in self.net:
545
            if isinstance(module, compatible_cls):
546
547
548
                hidden_states = module(hidden_states, scale)
            else:
                hidden_states = module(hidden_states)
549
        return hidden_states