rope_forward.py

import hipdnn
import torch


def build_rope_forward_graph(
    hipdnn_handle, torch_tensor_x, torch_tensor_cos, torch_tensor_sin, hipdnn_data_type
):
    graph = hipdnn.pygraph(
        handle=hipdnn_handle,
        io_data_type=hipdnn_data_type,
        intermediate_data_type=hipdnn.data_type.FLOAT,
        compute_data_type=hipdnn.data_type.FLOAT,
        name="rope_forward_inference",
    )
    hipdnn_tensor_x = graph.tensor_like(torch_tensor_x)
    hipdnn_tensor_cos = graph.tensor_like(torch_tensor_cos)
    hipdnn_tensor_sin = graph.tensor_like(torch_tensor_sin)
    hipdnn_tensor_y = graph.rope_forward(
        x=hipdnn_tensor_x, cos=hipdnn_tensor_cos, sin=hipdnn_tensor_sin, name="rope_forward"
    )
    hipdnn_tensor_y.set_output(True)
    graph.build(hipdnn_handle)
    return (graph, hipdnn_tensor_x, hipdnn_tensor_cos, hipdnn_tensor_sin, hipdnn_tensor_y)


if __name__ == "__main__":
    batch, seq_len, dim = 2, 4, 8
    hipdnn_data_type = hipdnn.data_type.FLOAT
    torch_data_type = torch.float32
    torch_tensor_x = torch.rand(batch, seq_len, dim, dtype=torch_data_type, device="cuda")
    torch_tensor_cos = torch.rand(batch, seq_len, dim, dtype=torch_data_type, device="cuda")
    torch_tensor_sin = torch.rand(batch, seq_len, dim, dtype=torch_data_type, device="cuda")
    hipdnn_handle = hipdnn.create_handle()
    graph, hipdnn_tensor_x, hipdnn_tensor_cos, hipdnn_tensor_sin, hipdnn_tensor_y = (
        build_rope_forward_graph(
            hipdnn_handle, torch_tensor_x, torch_tensor_cos, torch_tensor_sin, hipdnn_data_type
        )
    )
    torch_tensor_y = torch.empty(batch, seq_len, dim, dtype=torch_data_type, device="cuda")
    variant_pack = {
        hipdnn_tensor_x: torch_tensor_x.data_ptr(),
        hipdnn_tensor_cos: torch_tensor_cos.data_ptr(),
        hipdnn_tensor_sin: torch_tensor_sin.data_ptr(),
        hipdnn_tensor_y: torch_tensor_y.data_ptr(),
    }
    workspace = torch.empty(graph.get_workspace_size(), dtype=torch.uint8, device="cuda")
    graph.exec(variant_pack=variant_pack, workspace=workspace.data_ptr())
    print("rope_forward graph execution complete.")