fuse_ck.cpp 3.48 KB
Newer Older
Paul's avatar
Paul committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#include <migraphx/gpu/fuse_ck.hpp>
#include <migraphx/matcher.hpp>
#include <migraphx/pass_manager.hpp>
#include <migraphx/make_op.hpp>
#include <migraphx/register_op.hpp>

namespace migraphx {
inline namespace MIGRAPHX_INLINE_NS {

struct module;

namespace gpu {

struct ck_gemm
{
    operation op = make_op("dot");

    template <class Self, class F>
    static auto reflect(Self& self, F f)
    {
        return pack(f(self.op, "op"));
    }

    std::string name() const { return "gpu::ck_gemm"; }
Paul's avatar
Paul committed
25
26
27

    void check_gemm_shape(const shape& s) const
    {
Paul's avatar
Paul committed
28
        if(not contains(range(s.strides().rbegin(), s.strides().rbegin()+3), 1))
Paul's avatar
Paul committed
29
30
31
            MIGRAPHX_THROW("Invalid shape for ck_gemm");
    }

Paul's avatar
Paul committed
32
33
    shape compute_shape(std::vector<shape> inputs, const std::vector<module_ref>& mods) const
    {
Paul's avatar
Paul committed
34
        check_shapes{inputs, *this}.same_ndims();
Paul's avatar
Paul committed
35
36
37
38
        // if(mods.size() != 1)
        //     MIGRAPHX_THROW("should have one submodule.");
        if(inputs.size() < 2)
            MIGRAPHX_THROW("should have at least two inputs.");
Paul's avatar
Paul committed
39
40
        auto a = inputs[0];
        auto b = inputs[1];
Paul's avatar
Format  
Paul committed
41
        for(const auto& input : inputs)
Paul's avatar
Paul committed
42
            check_gemm_shape(input);
Paul's avatar
Paul committed
43
44
45
46
47
48
49
50
51
52
53
54
55
        return op.compute_shape({a, b});
    }
};
MIGRAPHX_REGISTER_OP(ck_gemm);

namespace {

MIGRAPHX_PRED_MATCHER(is_ck_gemm, instruction_ref ins)
{
    if(ins->name() != "dot")
        return false;
    auto a = ins->inputs().front()->get_shape();
    auto b = ins->inputs().back()->get_shape();
Paul's avatar
Paul committed
56
    if(a.lens().back() > 2048)
Paul's avatar
Paul committed
57
        return false;
Paul's avatar
Paul committed
58
    return true;
Paul's avatar
Paul committed
59
60
61
62
}

struct find_ck_gemm
{
Paul's avatar
Paul committed
63
    // Find a gemm followed by a pointwise operation.
Paul's avatar
Format  
Paul committed
64
65
66
67
    auto matcher() const
    {
        auto gemm =
            match::skip(match::name("contiguous"))(match::name("dot")(is_ck_gemm().bind("gemm")));
Paul's avatar
Paul committed
68
69
        return match::name("pointwise")(match::any_of[match::inputs()](gemm.bind("x")));
    }
Paul's avatar
Paul committed
70
71
72

    void apply(module_pass_manager& mpm, const match::matcher_result& r) const
    {
Paul's avatar
Paul committed
73
74
75
76
77
78
        auto ins      = r.result;
        auto gemm_ins = r.instructions["gemm"];
        auto x_ins    = r.instructions["x"]; // input after contiguous
        auto* pm      = ins->module_inputs().front();
        auto names    = pm->get_parameter_names();
        std::sort(names.begin(), names.end());
Paul's avatar
Format  
Paul committed
79
80
        auto inputs   = ins->inputs();
        auto gemm_it  = std::find(inputs.begin(), inputs.end(), x_ins);
Paul's avatar
Paul committed
81
82
        auto gemm_idx = gemm_it - inputs.begin();
        assert(gemm_it != inputs.end());
Paul's avatar
Paul committed
83
84
        if (ins->get_shape().type() != shape::half_type)
            return;
Paul's avatar
Format  
Paul committed
85
        if(gemm_idx != 0)
Paul's avatar
Paul committed
86
        {
Paul's avatar
Format  
Paul committed
87
88
            auto first_param    = pm->get_parameter(names[0]);
            auto gemm_param     = pm->get_parameter(names[gemm_idx]);
Paul's avatar
Paul committed
89
            auto new_gemm_param = pm->add_parameter(names[0] + ".0", gemm_param->get_shape());
Paul's avatar
Format  
Paul committed
90
91
            auto new_first_param =
                pm->add_parameter(names[gemm_idx] + ".0", first_param->get_shape());
Paul's avatar
Paul committed
92
93
94
95
96
97
98
99
100
            pm->replace_instruction(gemm_param, new_gemm_param);
            pm->replace_instruction(first_param, new_first_param);
            pm->remove_instruction(first_param);
            pm->remove_instruction(gemm_param);
        }
        inputs.erase(gemm_it);
        inputs.insert(inputs.begin(), gemm_ins->inputs().begin(), gemm_ins->inputs().end());

        mpm.get_module().replace_instruction(ins, ck_gemm{}, inputs, {pm});
Paul's avatar
Paul committed
101
102
103
104
105
    }
};

} // namespace

Paul's avatar
Format  
Paul committed
106
void fuse_ck::apply(module_pass_manager& mpm) const { match::find_matches(mpm, find_ck_gemm{}); }
Paul's avatar
Paul committed
107
108
109
110
111

} // namespace gpu

} // namespace MIGRAPHX_INLINE_NS
} // namespace migraphx