Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
add_layernorm.py 3.68 KB
Newer Older
yanjl1's avatar
Initial  
yanjl1 committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120 
import hipdnn
import torch


def build_layernorm_fusion_graph(
    hipdnn_handle,
    torch_tensor_x1,
    torch_tensor_x2,
    torch_tensor_scale,
    torch_tensor_bias,
    torch_tensor_epsilon,
    mode,
    eps,
    hipdnn_data_type,
):
    # Create graph
    graph = hipdnn.pygraph(
        handle=hipdnn_handle,
        io_data_type=hipdnn_data_type,
        intermediate_data_type=hipdnn.data_type.FLOAT,
        compute_data_type=hipdnn.data_type.FLOAT,
        name="layernorm_fusion_inference",
    )

    # Create hipdnn tensors
    hipdnn_tensor_x1 = graph.tensor_like(torch_tensor_x1)
    hipdnn_tensor_x2 = graph.tensor_like(torch_tensor_x2)
    hipdnn_tensor_scale = graph.tensor_like(torch_tensor_scale)
    hipdnn_tensor_bias = graph.tensor_like(torch_tensor_bias)
    hipdnn_tensor_epsilon = graph.tensor_like(torch_tensor_epsilon)
    hipdnn_tensor_epsilon.set_value(eps)

    # Create op
    hipdnn_tensor_add_output = graph.add(a=hipdnn_tensor_x1, b=hipdnn_tensor_x2, name="add")
    hipdnn_tensor_add_output.set_output(True)

    hipdnn_tensor_y, hipdnn_tensor_mean, hipdnn_tensor_inv_var = graph.layernorm(
        mode,
        hipdnn_tensor_add_output,
        hipdnn_tensor_scale,
        hipdnn_tensor_bias,
        hipdnn_tensor_epsilon,
        hipdnn.data_type.FLOAT,
        name="layernorm",
    )
    hipdnn_tensor_y.set_output(True)
    graph.build(hipdnn_handle)

    return (
        graph,
        hipdnn_tensor_x1,
        hipdnn_tensor_x2,
        hipdnn_tensor_scale,
        hipdnn_tensor_bias,
        hipdnn_tensor_add_output,
        hipdnn_tensor_y,
    )


if __name__ == "__main__":
    # Input dimensions
    batch = 16  # Batch size
    seq_len = 32  # Number of input seq
    embedding_dim = 64  # Number of feature
    mode = hipdnn.norm_forward_phase.INFERENCE  # Mode
    eps = 1e-5

    hipdnn_data_type = hipdnn.data_type.FLOAT
    torch_data_type = torch.float32

    torch_tensor_x1 = torch.rand(
        (batch, seq_len, embedding_dim), dtype=torch_data_type, device="cuda"
    )
    torch_tensor_x2 = torch.rand(
        (batch, seq_len, embedding_dim), dtype=torch_data_type, device="cuda"
    )
    torch_tensor_scale = torch.rand(embedding_dim, dtype=torch_data_type, device="cuda")
    torch_tensor_bias = torch.rand(embedding_dim, dtype=torch_data_type, device="cuda")
    torch_tensor_epsilon = torch.full(
        (1, 1, 1, 1), eps, dtype=torch.float32, requires_grad=False, device="cpu"
    )

    hipdnn_handle = hipdnn.create_handle()

    (
        graph,
        hipdnn_tensor_x1,
        hipdnn_tensor_x2,
        hipdnn_tensor_scale,
        hipdnn_tensor_bias,
        hipdnn_tensor_add_output,
        hipdnn_tensor_y,
    ) = build_layernorm_fusion_graph(
        hipdnn_handle,
        torch_tensor_x1,
        torch_tensor_x2,
        torch_tensor_scale,
        torch_tensor_bias,
        torch_tensor_epsilon,
        mode,
        eps,
        hipdnn_data_type,
    )

    torch_tensor_addoutput = torch.empty(
        hipdnn_tensor_add_output.get_dim(), dtype=torch_data_type, device="cuda"
    )
    torch_tensor_y = torch.empty(hipdnn_tensor_y.get_dim(), dtype=torch_data_type, device="cuda")
    variant_pack = {
        hipdnn_tensor_x1: torch_tensor_x1.data_ptr(),
        hipdnn_tensor_x2: torch_tensor_x2.data_ptr(),
        hipdnn_tensor_scale: torch_tensor_scale.data_ptr(),
        hipdnn_tensor_bias: torch_tensor_bias.data_ptr(),
        hipdnn_tensor_add_output: torch_tensor_addoutput.data_ptr(),
        hipdnn_tensor_y: torch_tensor_y.data_ptr(),
    }
    workspace = torch.empty(graph.get_workspace_size(), dtype=torch.uint8, device="cuda")

    graph.exec(variant_pack=variant_pack, workspace=workspace.data_ptr())
    print("add_layernorm graph execution complete.")