"vscode:/vscode.git/clone" did not exist on "4a65b050de16b7fbca2788891e7b6b534816c0fc"
activations.py 554 Bytes
Newer Older
1
2
3
from torch import nn


4
5
6
7
8
9
10
11
12
def get_activation(act_fn: str) -> nn.Module:
    """Helper function to get activation function from string.

    Args:
        act_fn (str): Name of activation function.

    Returns:
        nn.Module: Activation function.
    """
13
14
15
16
17
18
    if act_fn in ["swish", "silu"]:
        return nn.SiLU()
    elif act_fn == "mish":
        return nn.Mish()
    elif act_fn == "gelu":
        return nn.GELU()
19
20
    elif act_fn == "relu":
        return nn.ReLU()
21
22
    else:
        raise ValueError(f"Unsupported activation function: {act_fn}")