lowering.cpp 14.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/*
 * The MIT License (MIT)
 *
 * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
Shucai Xiao's avatar
Shucai Xiao committed
24
#include <iterator>
Paul's avatar
Paul committed
25
26
27
#include <migraphx/gpu/lowering.hpp>
#include <migraphx/manage_ptr.hpp>
#include <migraphx/instruction.hpp>
28
#include <migraphx/make_op.hpp>
29
30
#include <migraphx/instruction_ref.hpp>
#include <migraphx/stringutils.hpp>
31
32
33
34

#include <migraphx/op/convolution.hpp>
#include <migraphx/op/deconvolution.hpp>
#include <migraphx/op/dot.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
35
#include <migraphx/op/if_op.hpp>
36
37
38
39
#include <migraphx/op/reshape.hpp>
#include <migraphx/op/quant_convolution.hpp>
#include <migraphx/op/quant_dot.hpp>

Paul's avatar
Paul committed
40
41
#include <migraphx/gpu/context.hpp>
#include <migraphx/gpu/convolution.hpp>
42
#include <migraphx/gpu/device_name.hpp>
Paul's avatar
Paul committed
43
#include <migraphx/gpu/gemm.hpp>
44
#include <migraphx/gpu/int8_conv_pack.hpp>
45
46
#include <migraphx/gpu/miopen.hpp>
#include <migraphx/gpu/rocblas.hpp>
47
#include <migraphx/gpu/compiler.hpp>
48
#include <migraphx/iterator_for.hpp>
49
#include <migraphx/program.hpp>
Paul's avatar
Paul committed
50
#include <utility>
51
#include <functional>
Khalique's avatar
Khalique committed
52
#include <algorithm>
Shucai Xiao's avatar
Shucai Xiao committed
53
#include <map>
Paul's avatar
Paul committed
54

Paul's avatar
Paul committed
55
namespace migraphx {
Paul's avatar
Paul committed
56
inline namespace MIGRAPHX_INLINE_NS {
Paul's avatar
Paul committed
57
namespace gpu {
Paul's avatar
Paul committed
58
59
60

struct miopen_apply
{
Shucai Xiao's avatar
Shucai Xiao committed
61
    module* mod          = nullptr;
62
    const lowering* pass = nullptr;
Shucai Xiao's avatar
Shucai Xiao committed
63
    std::unordered_map<std::string, std::function<instruction_ref(instruction_ref)>> apply_map{};
Shucai Xiao's avatar
Shucai Xiao committed
64
    instruction_ref last{};
Shucai Xiao's avatar
Shucai Xiao committed
65
66
    bool offload_copy   = false;
    bool int8_x4_format = true;
67
    bool compute_fp32   = false;
Paul's avatar
Paul committed
68

69
    context& get_context() const
70
71
72
73
74
75
    {
        assert(pass != nullptr);
        assert(pass->ctx != nullptr);
        return *pass->ctx;
    }

Paul's avatar
Paul committed
76
77
78
79
80
81
82
    void check_shape(shape x, instruction_ref i)
    {
        assert(x == i->get_shape());
        (void)x;
        (void)i;
    }

83
84
    void init()
    {
Shucai Xiao's avatar
Shucai Xiao committed
85
        assert(mod != nullptr);
86
        assert(pass != nullptr);
87

88
89
90
        auto& ctx      = get_context();
        int8_x4_format = get_int8_x4_format(ctx);
        compute_fp32   = get_compute_fp32_flag();
Shucai Xiao's avatar
Shucai Xiao committed
91

Shucai Xiao's avatar
Shucai Xiao committed
92
        offload_copy = (mod->name() == "main") ? pass->offload_copy : false;
Paul's avatar
Paul committed
93

94
95
96
97
98
99
        add_generic_op("contiguous");

        add_extend_op("argmax");
        add_extend_op("argmin");
        add_extend_op("gather");
        add_extend_op("logsoftmax");
Shucai Xiao's avatar
Shucai Xiao committed
100
        add_extend_op("lrn");
turneram's avatar
turneram committed
101
        add_extend_op("multinomial");
Shucai Xiao's avatar
Shucai Xiao committed
102
        add_extend_op("nonzero");
103
        add_extend_op("pooling");
104
        add_extend_op("prefix_scan_sum");
Cagri Eryilmaz's avatar
Cagri Eryilmaz committed
105
        add_extend_op("reverse");
106
107
108
        add_extend_op("rnn_var_sl_last_output");
        add_extend_op("rnn_var_sl_shift_output");
        add_extend_op("rnn_var_sl_shift_sequence");
109
        add_extend_op("scatter_none");
Shucai Xiao's avatar
Shucai Xiao committed
110
        add_extend_op("topk");
111

112
113
114
        add_convolution_op<op::convolution>("convolution");
        add_convolution_op<op::deconvolution>("deconvolution");
        add_convolution_op<op::quant_convolution>("quant_convolution");
Shucai Xiao's avatar
Shucai Xiao committed
115
116
        add_gemm_op<op::dot>("dot");
        add_gemm_op<op::quant_dot>("quant_dot");
Shucai Xiao's avatar
Shucai Xiao committed
117
        add_if_op();
Shucai Xiao's avatar
Shucai Xiao committed
118
        add_loop_op();
Shucai Xiao's avatar
Shucai Xiao committed
119
        add_neg_op();
120
        add_nms_op();
121
122
    }

123
    void copy_params() const
124
    {
Shucai Xiao's avatar
Shucai Xiao committed
125
        if(not offload_copy)
126
            return;
127

Shucai Xiao's avatar
Shucai Xiao committed
128
        for(auto ins : iterator_for(*mod))
129
130
131
        {
            if(ins->name() != "@param")
                continue;
132

Shucai Xiao's avatar
Shucai Xiao committed
133
134
135
136
            // parameter no outputs, no need to insert copy to gpu
            if(ins->outputs().empty())
                continue;

137
138
            auto pos = std::next(ins);
            auto a   = insert_allocation(pos, ins->get_shape());
139
            auto c   = mod->insert_instruction(pos, make_op("hip::copy_to_gpu"), ins, a);
Shucai Xiao's avatar
Shucai Xiao committed
140
            mod->replace_instruction(ins, c);
141
        }
142
143

        // return instruction
Shucai Xiao's avatar
Shucai Xiao committed
144
        auto ret = std::prev(mod->end());
145
146
        if(ret->name() == "@return")
        {
147
            const auto& inputs = ret->inputs();
148
149
150

            // each input of ret need to be copied from gpu to host, and replace
            // output with copy output
151
            for(const auto& in : inputs)
152
            {
153
                auto p_output = mod->insert_instruction(ret, make_op("hip::copy_from_gpu"), in);
154
155
156
157
158
159
                instruction::replace_argument(ret, in, p_output);
            }
        }
        // else branch to handle legacy program without the return instruction
        else
        {
160
            mod->add_instruction(make_op("hip::copy_from_gpu"), ret);
161
        }
162
163
    }

Paul's avatar
Paul committed
164
165
    void apply()
    {
166
        init();
Shucai Xiao's avatar
Shucai Xiao committed
167
        for(auto it = mod->begin(); it != mod->end(); it++)
Paul's avatar
Paul committed
168
        {
169
170
            auto s     = it->get_shape();
            auto attrs = it->get_operator().attributes();
171
            if(apply_map.count(it->name()) > 0)
172
            {
173
                check_shape(s, apply_map.at(it->name())(it));
Paul's avatar
Paul committed
174
            }
175
176
177
178
            else if(has_compiler_for(it->name()))
            {
                check_shape(s, insert_precompile_op(it));
            }
179
180
181
182
            else if(attrs.contains("target"))
            {
                check_shape(s, insert_custom_op(it, attrs));
            }
Paul's avatar
Paul committed
183
        }
184
        copy_params();
Paul's avatar
Paul committed
185
186
    }

187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
    instruction_ref insert_custom_op(instruction_ref ins, const value& attrs) const
    {
        const auto& custom_op = ins->get_operator();
        if(attrs.at("target") == "cpu")
        {
            auto s = ins->get_shape();
            std::vector<instruction_ref> cpu_inputs;
            auto inputs = ins->inputs();
            auto output = inputs.back();
            std::transform(
                inputs.begin(), inputs.end(), std::back_inserter(cpu_inputs), [&](auto in) {
                    return mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), in);
                });
            cpu_inputs.front() =
                mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_inputs);
            auto cpu_out = mod->insert_instruction(ins, custom_op, cpu_inputs);
            auto gpu_out =
                mod->insert_instruction(ins, make_op("hip::copy_to_gpu"), cpu_out, output);
            return mod->replace_instruction(ins, gpu_out);
        }
        return ins;
    }

210
    instruction_ref insert_precompile_op(instruction_ref ins) const
211
212
213
214
215
216
217
218
219
220
221
222
    {
        auto output                       = insert_allocation(ins, ins->get_shape());
        std::vector<instruction_ref> refs = ins->inputs();
        refs.push_back(output);

        return mod->replace_instruction(
            ins,
            make_op("gpu::precompile_op", {{"op", to_value(ins->get_operator())}}),
            refs,
            ins->module_inputs());
    }

223
    instruction_ref insert_allocation(instruction_ref ins, const shape& s) const
Paul's avatar
Paul committed
224
    {
225
        return mod->insert_instruction(ins, make_op("allocate", {{"shape", to_value(s)}}));
Paul's avatar
Paul committed
226
227
    }

228
229
    template <typename Op>
    void add_gemm_op(const std::string& name)
230
231
    {
        apply_map.emplace(name, [=](instruction_ref ins) {
232
            std::vector<instruction_ref> refs = ins->inputs();
233
234
235
            assert(refs.size() == 2);
            auto output = insert_allocation(ins, ins->get_shape());
            refs.push_back(output);
Shucai Xiao's avatar
Shucai Xiao committed
236
            return mod->replace_instruction(
237
                ins, rocblas_gemm<Op>{Op{}, 1, 0, int8_x4_format, compute_fp32}, refs);
238
239
240
        });
    }

241
242
    template <typename Op>
    void add_convolution_op(const std::string& name)
243
    {
244
245
246
247
248
249
250
251
252
        apply_map.emplace(name, [=](instruction_ref ins) {
            operation conv =
                miopen_convolution<Op>{any_cast<Op>(ins->get_operator()), int8_x4_format};
            migraphx::context ctx         = get_context();
            size_t ws_bytes               = 0;
            auto compile_conv_with_format = [&](bool format) {
                conv     = miopen_convolution<Op>{any_cast<Op>(ins->get_operator()), format};
                auto ws  = conv.compile(ctx, ins->get_shape(), to_shapes(ins->inputs()));
                ws_bytes = ws.get("workspace", 0);
253
254
255
            };

            try
256
257
            { // for the regular convolution and deconvolution, this try would always succeed
                compile_conv_with_format(int8_x4_format);
258
259
260
261
            }
            catch(migraphx::exception&)
            {
                // In case no solver supports the default format, retry using the other format.
262
                compile_conv_with_format(not int8_x4_format);
263
            }
264

Shucai Xiao's avatar
Shucai Xiao committed
265
            auto args      = ins->inputs();
Shucai Xiao's avatar
Shucai Xiao committed
266
            auto output    = insert_allocation(ins, ins->get_shape());
267
            auto workspace = insert_allocation(ins, shape{shape::int8_type, {ws_bytes}});
Shucai Xiao's avatar
Shucai Xiao committed
268
            return mod->replace_instruction(ins, conv, args[0], args[1], workspace, output);
Shucai Xiao's avatar
Shucai Xiao committed
269
270
271
        });
    }

272
273
274
    // add_generic_op just constructs the operator with no fields whereas add_extend_op copies over
    // the fields Since it doesn't have fields its default constructed

275
276
277
    void add_generic_op(const std::string& name) { add_generic_op(name, "gpu::" + name); }

    void add_generic_op(const std::string& op_name, const std::string& gpu_name)
Paul's avatar
Paul committed
278
    {
279
        apply_map.emplace(op_name, [=](instruction_ref ins) {
280
281
282
            auto output                       = insert_allocation(ins, ins->get_shape());
            std::vector<instruction_ref> refs = ins->inputs();
            refs.push_back(output);
Paul's avatar
Paul committed
283

Shucai Xiao's avatar
Shucai Xiao committed
284
            return mod->replace_instruction(ins, make_op(gpu_name), refs);
285
        });
Paul's avatar
Paul committed
286
    }
Paul's avatar
Paul committed
287

288
289
290
    void add_extend_op(const std::string& name) { add_extend_op(name, "gpu::" + name); }

    void add_extend_op(const std::string& op_name, const std::string& gpu_name)
Khalique's avatar
Khalique committed
291
    {
292
293
        apply_map.emplace(op_name, [=](instruction_ref ins) {
            auto&& op                         = ins->get_operator();
294
295
296
            auto output                       = insert_allocation(ins, ins->get_shape());
            std::vector<instruction_ref> refs = ins->inputs();
            refs.push_back(output);
Paul's avatar
Paul committed
297

Shucai Xiao's avatar
Shucai Xiao committed
298
            return mod->replace_instruction(ins, make_op(gpu_name, op.to_value()), refs);
299
        });
Khalique's avatar
Khalique committed
300
301
    }

Shucai Xiao's avatar
Shucai Xiao committed
302
303
304
305
306
307
    // use 0 - input to represent neg
    void add_neg_op()
    {
        apply_map.emplace("neg", [=](instruction_ref ins) {
            auto s = ins->get_shape();
            std::vector<float> zeros(s.elements(), 0.0f);
Shucai Xiao's avatar
Shucai Xiao committed
308
            auto l0     = mod->add_literal(literal(s, zeros));
Shucai Xiao's avatar
Shucai Xiao committed
309
            auto output = insert_allocation(ins, s);
Shucai Xiao's avatar
Shucai Xiao committed
310
            return mod->replace_instruction(
311
                ins, make_op("gpu::sub"), l0, ins->inputs().front(), output);
Shucai Xiao's avatar
Shucai Xiao committed
312
313
        });
    }
Shucai Xiao's avatar
Shucai Xiao committed
314

Shucai Xiao's avatar
Shucai Xiao committed
315
    // add input and output argument for the if operator
Shucai Xiao's avatar
Shucai Xiao committed
316
317
318
319
    void add_if_op()
    {
        apply_map.emplace("if", [=](instruction_ref ins) {
            std::vector<instruction_ref> inputs = ins->inputs();
320
321
322
            auto cpu_cond =
                mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), inputs.front());
            auto sync_cond = mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_cond);
Shucai Xiao's avatar
Shucai Xiao committed
323
324
            inputs.front() = sync_cond;

325
            return mod->replace_instruction(ins, ins->get_operator(), inputs, ins->module_inputs());
Shucai Xiao's avatar
Shucai Xiao committed
326
327
        });
    }
Shucai Xiao's avatar
Shucai Xiao committed
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343

    // replace the loop operator with gpu_loop operator
    void add_loop_op()
    {
        apply_map.emplace("loop", [=](instruction_ref ins) {
            std::vector<instruction_ref> inputs = ins->inputs();
            // copy max_iter from gpu to cpu
            auto cpu_max_iter =
                mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), inputs.at(0));
            auto cpu_cond =
                mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), inputs.at(1));
            auto synced_max_iter =
                mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_max_iter, cpu_cond);
            inputs.at(0)     = synced_max_iter;
            inputs.at(1)     = cpu_cond;
            auto copy_inputs = inputs;
344
345
346
347
            std::transform(copy_inputs.begin(),
                           copy_inputs.end(),
                           std::back_inserter(inputs),
                           [&](auto in) { return insert_allocation(ins, in->get_shape()); });
Shucai Xiao's avatar
Shucai Xiao committed
348
349
350
351
352

            auto mod_args = ins->module_inputs();
            auto output   = insert_allocation(ins, ins->get_shape());

            const auto* sub_mod = mod_args.front();
353
354
            auto cond_out       = insert_allocation(ins, sub_mod->get_output_shapes().front());

Shucai Xiao's avatar
Shucai Xiao committed
355
356
357
358
359
360
361
362
            // add cond and mod outputs to the argument list
            inputs.push_back(cond_out);
            inputs.push_back(output);

            return mod->replace_instruction(
                ins, make_op("gpu::loop", ins->get_operator().to_value()), inputs, mod_args);
        });
    }
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382

    void add_nms_op()
    {
        apply_map.emplace("nonmaxsuppression", [=](instruction_ref ins) {
            auto s      = ins->get_shape();
            auto output = insert_allocation(ins, s);
            std::vector<instruction_ref> cpu_inputs;
            auto inputs = ins->inputs();
            std::transform(
                inputs.begin(), inputs.end(), std::back_inserter(cpu_inputs), [&](auto in) {
                    return mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), in);
                });
            cpu_inputs.front() =
                mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_inputs);
            auto cpu_out = mod->insert_instruction(ins, ins->get_operator(), cpu_inputs);
            auto gpu_out =
                mod->insert_instruction(ins, make_op("hip::copy_to_gpu"), cpu_out, output);
            return mod->replace_instruction(ins, gpu_out);
        });
    }
Paul's avatar
Paul committed
383
384
};

Shucai Xiao's avatar
Shucai Xiao committed
385
void lowering::apply(module& m) const { miopen_apply{&m, this}.apply(); }
Shucai Xiao's avatar
Shucai Xiao committed
386

Paul's avatar
Paul committed
387
} // namespace gpu
Paul's avatar
Paul committed
388
} // namespace MIGRAPHX_INLINE_NS
Paul's avatar
Paul committed
389
} // namespace migraphx