"vscode:/vscode.git/clone" did not exist on "84cb0598196e1e02df6b5ac9bb4fc702d07fa9a0"
test_layernorm.cpp 3.31 KB
Newer Older
1
2
3
4
5
6

#include "verify_program.hpp"
#include <migraphx/program.hpp>
#include <migraphx/generate.hpp>
#include <migraphx/operators.hpp>

Paul Fultz II's avatar
Paul Fultz II committed
7
8
migraphx::instruction_ref
add_layernorm(migraphx::module& m, migraphx::instruction_ref x, std::vector<size_t> dims)
9
10
{
    auto scale =
Paul Fultz II's avatar
Paul Fultz II committed
11
        m.add_parameter("scale", migraphx::shape{migraphx::shape::float_type, {dims.back()}});
12
    auto bias =
Paul Fultz II's avatar
Paul Fultz II committed
13
14
15
        m.add_parameter("bias", migraphx::shape{migraphx::shape::float_type, {dims.back()}});
    auto epsilon  = m.add_literal(1e-12f);
    auto exponent = m.add_literal(2.0f);
16

Paul Fultz II's avatar
Paul Fultz II committed
17
18
19
20
21
22
    auto mean            = m.add_instruction(migraphx::op::reduce_mean({2}), x);
    auto mean_mbcast     = m.add_instruction(migraphx::op::multibroadcast{{dims}}, mean);
    auto sub             = m.add_instruction(migraphx::op::sub{}, x, mean_mbcast);
    auto exponent_mbcast = m.add_instruction(migraphx::op::multibroadcast{{dims}}, exponent);
    auto pow             = m.add_instruction(migraphx::op::pow{}, sub, exponent_mbcast);
    auto var             = m.add_instruction(migraphx::op::reduce_mean({2}), pow);
23
    auto epsilon_mbcast =
Paul Fultz II's avatar
Paul Fultz II committed
24
25
26
27
28
29
30
31
32
        m.add_instruction(migraphx::op::multibroadcast{{1, dims.at(1), 1}}, epsilon);
    auto add_epsilon  = m.add_instruction(migraphx::op::add{}, var, epsilon_mbcast);
    auto sqrt         = m.add_instruction(migraphx::op::sqrt{}, add_epsilon);
    auto sqrt_mbcast  = m.add_instruction(migraphx::op::multibroadcast{dims}, sqrt);
    auto div          = m.add_instruction(migraphx::op::div{}, sub, sqrt_mbcast);
    auto scale_mbcast = m.add_instruction(migraphx::op::multibroadcast{dims}, scale);
    auto mul          = m.add_instruction(migraphx::op::mul{}, scale_mbcast, div);
    auto bias_mbcast  = m.add_instruction(migraphx::op::multibroadcast{dims}, bias);
    return m.add_instruction(migraphx::op::add{}, mul, bias_mbcast);
33
34
35
36
37
38
39
}

struct test_layernorm : verify_program<test_layernorm>
{
    migraphx::program create_program() const
    {
        migraphx::program p;
Paul Fultz II's avatar
Paul Fultz II committed
40
41
42
43
        auto* mm                 = p.get_main_module();
        std::vector<size_t> dims = {1, 1, 5};
        auto x = mm->add_parameter("x", migraphx::shape{migraphx::shape::float_type, dims});
        add_layernorm(*mm, x, dims);
44
45
46
47
48
49
50
51
52
        return p;
    }
};

struct test_layernorm2 : verify_program<test_layernorm2>
{
    migraphx::program create_program() const
    {
        migraphx::program p;
Paul Fultz II's avatar
Paul Fultz II committed
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
        auto* mm                 = p.get_main_module();
        std::vector<size_t> dims = {1, 4, 24};
        auto x = mm->add_parameter("x", migraphx::shape{migraphx::shape::float_type, dims});
        add_layernorm(*mm, x, dims);
        return p;
    }
};

struct test_layernorm_triadd : verify_program<test_layernorm_triadd>
{
    migraphx::program create_program() const
    {
        migraphx::program p;
        auto* mm                 = p.get_main_module();
        std::vector<size_t> dims = {1, 4, 24};
        auto x    = mm->add_parameter("x", migraphx::shape{migraphx::shape::float_type, dims});
        auto y    = mm->add_parameter("y", migraphx::shape{migraphx::shape::float_type, dims});
        auto z    = mm->add_parameter("z", migraphx::shape{migraphx::shape::float_type, dims});
        auto add1 = mm->add_instruction(migraphx::op::add{}, x, y);
        auto add2 = mm->add_instruction(migraphx::op::add{}, add1, z);
        add_layernorm(*mm, add2, dims);
74
75
76
        return p;
    }
};