lowering.cpp 15.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/*
 * The MIT License (MIT)
 *
 * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
Shucai Xiao's avatar
Shucai Xiao committed
24
#include <iterator>
Paul's avatar
Paul committed
25
26
27
#include <migraphx/gpu/lowering.hpp>
#include <migraphx/manage_ptr.hpp>
#include <migraphx/instruction.hpp>
28
#include <migraphx/make_op.hpp>
29
30
#include <migraphx/instruction_ref.hpp>
#include <migraphx/stringutils.hpp>
31
32
33
34

#include <migraphx/op/convolution.hpp>
#include <migraphx/op/deconvolution.hpp>
#include <migraphx/op/dot.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
35
#include <migraphx/op/if_op.hpp>
36
37
38
39
#include <migraphx/op/reshape.hpp>
#include <migraphx/op/quant_convolution.hpp>
#include <migraphx/op/quant_dot.hpp>

Paul's avatar
Paul committed
40
41
#include <migraphx/gpu/context.hpp>
#include <migraphx/gpu/convolution.hpp>
kahmed10's avatar
kahmed10 committed
42
#include <migraphx/gpu/deconvolution.hpp>
43
#include <migraphx/gpu/device_name.hpp>
Paul's avatar
Paul committed
44
#include <migraphx/gpu/gemm.hpp>
45
#include <migraphx/gpu/int8_conv_pack.hpp>
46
47
48
#include <migraphx/gpu/miopen.hpp>
#include <migraphx/gpu/quant_convolution.hpp>
#include <migraphx/gpu/rocblas.hpp>
49
#include <migraphx/gpu/compiler.hpp>
50
#include <migraphx/iterator_for.hpp>
51
#include <migraphx/program.hpp>
Paul's avatar
Paul committed
52
#include <utility>
53
#include <functional>
Khalique's avatar
Khalique committed
54
#include <algorithm>
Shucai Xiao's avatar
Shucai Xiao committed
55
#include <map>
Paul's avatar
Paul committed
56

Paul's avatar
Paul committed
57
namespace migraphx {
Paul's avatar
Paul committed
58
inline namespace MIGRAPHX_INLINE_NS {
Paul's avatar
Paul committed
59
namespace gpu {
Paul's avatar
Paul committed
60
61
62

struct miopen_apply
{
Shucai Xiao's avatar
Shucai Xiao committed
63
    module* mod          = nullptr;
64
    const lowering* pass = nullptr;
Shucai Xiao's avatar
Shucai Xiao committed
65
    std::unordered_map<std::string, std::function<instruction_ref(instruction_ref)>> apply_map{};
Shucai Xiao's avatar
Shucai Xiao committed
66
    instruction_ref last{};
Shucai Xiao's avatar
Shucai Xiao committed
67
68
    bool offload_copy   = false;
    bool int8_x4_format = true;
69
    bool compute_fp32   = false;
Paul's avatar
Paul committed
70

71
    context& get_context() const
72
73
74
75
76
77
    {
        assert(pass != nullptr);
        assert(pass->ctx != nullptr);
        return *pass->ctx;
    }

Paul's avatar
Paul committed
78
79
80
81
82
83
84
    void check_shape(shape x, instruction_ref i)
    {
        assert(x == i->get_shape());
        (void)x;
        (void)i;
    }

85
86
    void init()
    {
Shucai Xiao's avatar
Shucai Xiao committed
87
        assert(mod != nullptr);
88
        assert(pass != nullptr);
89

90
91
92
        auto& ctx      = get_context();
        int8_x4_format = get_int8_x4_format(ctx);
        compute_fp32   = get_compute_fp32_flag();
Shucai Xiao's avatar
Shucai Xiao committed
93

Shucai Xiao's avatar
Shucai Xiao committed
94
        offload_copy = (mod->name() == "main") ? pass->offload_copy : false;
Paul's avatar
Paul committed
95

96
97
98
99
        add_generic_op("contiguous");

        add_extend_op("argmax");
        add_extend_op("argmin");
Shucai Xiao's avatar
Shucai Xiao committed
100
        add_extend_op("elu");
101
        add_extend_op("gather");
Shucai Xiao's avatar
Shucai Xiao committed
102
        add_extend_op("leaky_relu");
103
        add_extend_op("logsoftmax");
Shucai Xiao's avatar
Shucai Xiao committed
104
        add_extend_op("lrn");
turneram's avatar
turneram committed
105
        add_extend_op("multinomial");
Shucai Xiao's avatar
Shucai Xiao committed
106
        add_extend_op("nonzero");
107
        add_extend_op("pooling");
108
        add_extend_op("prefix_scan_sum");
Cagri Eryilmaz's avatar
Cagri Eryilmaz committed
109
        add_extend_op("reverse");
110
111
112
        add_extend_op("rnn_var_sl_last_output");
        add_extend_op("rnn_var_sl_shift_output");
        add_extend_op("rnn_var_sl_shift_sequence");
113
        add_extend_op("scatter_none");
Shucai Xiao's avatar
Shucai Xiao committed
114
        add_extend_op("topk");
115

116
        add_convolution_op();
kahmed10's avatar
kahmed10 committed
117
        add_deconvolution_op();
Shucai Xiao's avatar
Shucai Xiao committed
118
119
        add_gemm_op<op::dot>("dot");
        add_gemm_op<op::quant_dot>("quant_dot");
Shucai Xiao's avatar
Shucai Xiao committed
120
        add_if_op();
Shucai Xiao's avatar
Shucai Xiao committed
121
        add_loop_op();
Shucai Xiao's avatar
Shucai Xiao committed
122
        add_neg_op();
123
        add_nms_op();
Shucai Xiao's avatar
Shucai Xiao committed
124
        add_quant_convolution_op();
125
126
    }

127
    void copy_params() const
128
    {
Shucai Xiao's avatar
Shucai Xiao committed
129
        if(not offload_copy)
130
            return;
131

Shucai Xiao's avatar
Shucai Xiao committed
132
        for(auto ins : iterator_for(*mod))
133
134
135
        {
            if(ins->name() != "@param")
                continue;
136

Shucai Xiao's avatar
Shucai Xiao committed
137
138
139
140
            // parameter no outputs, no need to insert copy to gpu
            if(ins->outputs().empty())
                continue;

141
142
            auto pos = std::next(ins);
            auto a   = insert_allocation(pos, ins->get_shape());
143
            auto c   = mod->insert_instruction(pos, make_op("hip::copy_to_gpu"), ins, a);
Shucai Xiao's avatar
Shucai Xiao committed
144
            mod->replace_instruction(ins, c);
145
        }
146
147

        // return instruction
Shucai Xiao's avatar
Shucai Xiao committed
148
        auto ret = std::prev(mod->end());
149
150
        if(ret->name() == "@return")
        {
151
            const auto& inputs = ret->inputs();
152
153
154

            // each input of ret need to be copied from gpu to host, and replace
            // output with copy output
155
            for(const auto& in : inputs)
156
            {
157
                auto p_output = mod->insert_instruction(ret, make_op("hip::copy_from_gpu"), in);
158
159
160
161
162
163
                instruction::replace_argument(ret, in, p_output);
            }
        }
        // else branch to handle legacy program without the return instruction
        else
        {
164
            mod->add_instruction(make_op("hip::copy_from_gpu"), ret);
165
        }
166
167
    }

Paul's avatar
Paul committed
168
169
    void apply()
    {
170
        init();
Shucai Xiao's avatar
Shucai Xiao committed
171
        for(auto it = mod->begin(); it != mod->end(); it++)
Paul's avatar
Paul committed
172
        {
173
174
            auto s     = it->get_shape();
            auto attrs = it->get_operator().attributes();
175
            if(apply_map.count(it->name()) > 0)
176
            {
177
                check_shape(s, apply_map.at(it->name())(it));
Paul's avatar
Paul committed
178
            }
179
180
181
182
            else if(has_compiler_for(it->name()))
            {
                check_shape(s, insert_precompile_op(it));
            }
183
184
185
186
            else if(attrs.contains("target"))
            {
                check_shape(s, insert_custom_op(it, attrs));
            }
Paul's avatar
Paul committed
187
        }
188
        copy_params();
Paul's avatar
Paul committed
189
190
    }

191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
    instruction_ref insert_custom_op(instruction_ref ins, const value& attrs) const
    {
        const auto& custom_op = ins->get_operator();
        if(attrs.at("target") == "cpu")
        {
            auto s = ins->get_shape();
            std::vector<instruction_ref> cpu_inputs;
            auto inputs = ins->inputs();
            auto output = inputs.back();
            std::transform(
                inputs.begin(), inputs.end(), std::back_inserter(cpu_inputs), [&](auto in) {
                    return mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), in);
                });
            cpu_inputs.front() =
                mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_inputs);
            auto cpu_out = mod->insert_instruction(ins, custom_op, cpu_inputs);
            auto gpu_out =
                mod->insert_instruction(ins, make_op("hip::copy_to_gpu"), cpu_out, output);
            return mod->replace_instruction(ins, gpu_out);
        }
        return ins;
    }

214
    instruction_ref insert_precompile_op(instruction_ref ins) const
215
216
217
218
219
220
221
222
223
224
225
226
    {
        auto output                       = insert_allocation(ins, ins->get_shape());
        std::vector<instruction_ref> refs = ins->inputs();
        refs.push_back(output);

        return mod->replace_instruction(
            ins,
            make_op("gpu::precompile_op", {{"op", to_value(ins->get_operator())}}),
            refs,
            ins->module_inputs());
    }

227
    instruction_ref insert_allocation(instruction_ref ins, const shape& s) const
Paul's avatar
Paul committed
228
    {
229
        return mod->insert_instruction(ins, make_op("allocate", {{"shape", to_value(s)}}));
Paul's avatar
Paul committed
230
231
    }

Shucai Xiao's avatar
Shucai Xiao committed
232
    void add_convolution_op()
Paul's avatar
Paul committed
233
    {
234
235
        apply_map.emplace("convolution", [=](instruction_ref ins) {
            auto&& op = any_cast<op::convolution>(ins->get_operator());
Paul's avatar
Paul committed
236

237
            auto conv = miopen_convolution{op, make_conv(op)};
238
            auto ws   = conv.find(get_context(), ins->get_shape(), to_shapes(ins->inputs()));
Paul's avatar
Paul committed
239

240
            auto workspace = insert_allocation(ins, ws);
241
            auto output    = insert_allocation(ins, ins->get_shape());
kahmed10's avatar
kahmed10 committed
242

Shucai Xiao's avatar
Shucai Xiao committed
243
            return mod->replace_instruction(
kahmed10's avatar
kahmed10 committed
244
245
246
247
248
249
250
251
252
253
                ins, conv, ins->inputs().at(0), ins->inputs().at(1), workspace, output);
        });
    }

    void add_deconvolution_op()
    {
        apply_map.emplace("deconvolution", [=](instruction_ref ins) {
            auto&& op = any_cast<op::deconvolution>(ins->get_operator());

            auto conv = miopen_deconvolution{op, make_deconv(op)};
Paul Fultz II's avatar
Paul Fultz II committed
254
            auto ws   = conv.find(get_context(), ins->get_shape(), to_shapes(ins->inputs()));
kahmed10's avatar
kahmed10 committed
255

256
            auto workspace = insert_allocation(ins, ws);
kahmed10's avatar
kahmed10 committed
257
            auto output    = insert_allocation(ins, ins->get_shape());
Paul's avatar
Paul committed
258

Shucai Xiao's avatar
Shucai Xiao committed
259
            return mod->replace_instruction(
260
261
                ins, conv, ins->inputs().at(0), ins->inputs().at(1), workspace, output);
        });
Paul's avatar
Paul committed
262
263
    }

264
265
    template <typename Op>
    void add_gemm_op(const std::string& name)
266
267
    {
        apply_map.emplace(name, [=](instruction_ref ins) {
268
            std::vector<instruction_ref> refs = ins->inputs();
269
270
271
            assert(refs.size() == 2);
            auto output = insert_allocation(ins, ins->get_shape());
            refs.push_back(output);
Shucai Xiao's avatar
Shucai Xiao committed
272
            return mod->replace_instruction(
273
                ins, rocblas_gemm<Op>{Op{}, 1, 0, int8_x4_format, compute_fp32}, refs);
274
275
276
        });
    }

277
278
279
280
    void add_quant_convolution_op()
    {
        apply_map.emplace("quant_convolution", [=](instruction_ref ins) {
            auto&& op = any_cast<op::quant_convolution>(ins->get_operator());
281
282
283
284
            shape ws;
            miopen_quant_convolution conv;
            auto compile_quant_conv_with_format = [&](bool format) {
                conv = miopen_quant_convolution{op, format, make_conv(op)};
Paul Fultz II's avatar
Paul Fultz II committed
285
                ws   = conv.find(get_context(), ins->get_shape(), to_shapes(ins->inputs()));
286
287
288
289
290
291
292
293
294
            };

            try
            {
                compile_quant_conv_with_format(int8_x4_format);
            }
            catch(migraphx::exception&)
            {
                // In case no solver supports the default format, retry using the other format.
295
                compile_quant_conv_with_format(not int8_x4_format);
296
            }
297

Shucai Xiao's avatar
Shucai Xiao committed
298
            auto args      = ins->inputs();
299
            auto workspace = insert_allocation(ins, ws);
Shucai Xiao's avatar
Shucai Xiao committed
300
301
            auto output    = insert_allocation(ins, ins->get_shape());

Shucai Xiao's avatar
Shucai Xiao committed
302
            return mod->replace_instruction(ins, conv, args[0], args[1], workspace, output);
Shucai Xiao's avatar
Shucai Xiao committed
303
304
305
        });
    }

306
307
308
    // add_generic_op just constructs the operator with no fields whereas add_extend_op copies over
    // the fields Since it doesn't have fields its default constructed

309
310
311
    void add_generic_op(const std::string& name) { add_generic_op(name, "gpu::" + name); }

    void add_generic_op(const std::string& op_name, const std::string& gpu_name)
Paul's avatar
Paul committed
312
    {
313
        apply_map.emplace(op_name, [=](instruction_ref ins) {
314
315
316
            auto output                       = insert_allocation(ins, ins->get_shape());
            std::vector<instruction_ref> refs = ins->inputs();
            refs.push_back(output);
Paul's avatar
Paul committed
317

Shucai Xiao's avatar
Shucai Xiao committed
318
            return mod->replace_instruction(ins, make_op(gpu_name), refs);
319
        });
Paul's avatar
Paul committed
320
    }
Paul's avatar
Paul committed
321

322
323
324
    void add_extend_op(const std::string& name) { add_extend_op(name, "gpu::" + name); }

    void add_extend_op(const std::string& op_name, const std::string& gpu_name)
Khalique's avatar
Khalique committed
325
    {
326
327
        apply_map.emplace(op_name, [=](instruction_ref ins) {
            auto&& op                         = ins->get_operator();
328
329
330
            auto output                       = insert_allocation(ins, ins->get_shape());
            std::vector<instruction_ref> refs = ins->inputs();
            refs.push_back(output);
Paul's avatar
Paul committed
331

Shucai Xiao's avatar
Shucai Xiao committed
332
            return mod->replace_instruction(ins, make_op(gpu_name, op.to_value()), refs);
333
        });
Khalique's avatar
Khalique committed
334
335
    }

Shucai Xiao's avatar
Shucai Xiao committed
336
337
338
339
340
341
    // use 0 - input to represent neg
    void add_neg_op()
    {
        apply_map.emplace("neg", [=](instruction_ref ins) {
            auto s = ins->get_shape();
            std::vector<float> zeros(s.elements(), 0.0f);
Shucai Xiao's avatar
Shucai Xiao committed
342
            auto l0     = mod->add_literal(literal(s, zeros));
Shucai Xiao's avatar
Shucai Xiao committed
343
            auto output = insert_allocation(ins, s);
Shucai Xiao's avatar
Shucai Xiao committed
344
            return mod->replace_instruction(
345
                ins, make_op("gpu::sub"), l0, ins->inputs().front(), output);
Shucai Xiao's avatar
Shucai Xiao committed
346
347
        });
    }
Shucai Xiao's avatar
Shucai Xiao committed
348

Shucai Xiao's avatar
Shucai Xiao committed
349
    // add input and output argument for the if operator
Shucai Xiao's avatar
Shucai Xiao committed
350
351
352
353
    void add_if_op()
    {
        apply_map.emplace("if", [=](instruction_ref ins) {
            std::vector<instruction_ref> inputs = ins->inputs();
354
355
356
            auto cpu_cond =
                mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), inputs.front());
            auto sync_cond = mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_cond);
Shucai Xiao's avatar
Shucai Xiao committed
357
358
            inputs.front() = sync_cond;

359
            return mod->replace_instruction(ins, ins->get_operator(), inputs, ins->module_inputs());
Shucai Xiao's avatar
Shucai Xiao committed
360
361
        });
    }
Shucai Xiao's avatar
Shucai Xiao committed
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377

    // replace the loop operator with gpu_loop operator
    void add_loop_op()
    {
        apply_map.emplace("loop", [=](instruction_ref ins) {
            std::vector<instruction_ref> inputs = ins->inputs();
            // copy max_iter from gpu to cpu
            auto cpu_max_iter =
                mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), inputs.at(0));
            auto cpu_cond =
                mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), inputs.at(1));
            auto synced_max_iter =
                mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_max_iter, cpu_cond);
            inputs.at(0)     = synced_max_iter;
            inputs.at(1)     = cpu_cond;
            auto copy_inputs = inputs;
378
379
380
381
            std::transform(copy_inputs.begin(),
                           copy_inputs.end(),
                           std::back_inserter(inputs),
                           [&](auto in) { return insert_allocation(ins, in->get_shape()); });
Shucai Xiao's avatar
Shucai Xiao committed
382
383
384
385
386

            auto mod_args = ins->module_inputs();
            auto output   = insert_allocation(ins, ins->get_shape());

            const auto* sub_mod = mod_args.front();
387
388
            auto cond_out       = insert_allocation(ins, sub_mod->get_output_shapes().front());

Shucai Xiao's avatar
Shucai Xiao committed
389
390
391
392
393
394
395
396
            // add cond and mod outputs to the argument list
            inputs.push_back(cond_out);
            inputs.push_back(output);

            return mod->replace_instruction(
                ins, make_op("gpu::loop", ins->get_operator().to_value()), inputs, mod_args);
        });
    }
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416

    void add_nms_op()
    {
        apply_map.emplace("nonmaxsuppression", [=](instruction_ref ins) {
            auto s      = ins->get_shape();
            auto output = insert_allocation(ins, s);
            std::vector<instruction_ref> cpu_inputs;
            auto inputs = ins->inputs();
            std::transform(
                inputs.begin(), inputs.end(), std::back_inserter(cpu_inputs), [&](auto in) {
                    return mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), in);
                });
            cpu_inputs.front() =
                mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_inputs);
            auto cpu_out = mod->insert_instruction(ins, ins->get_operator(), cpu_inputs);
            auto gpu_out =
                mod->insert_instruction(ins, make_op("hip::copy_to_gpu"), cpu_out, output);
            return mod->replace_instruction(ins, gpu_out);
        });
    }
Paul's avatar
Paul committed
417
418
};

Shucai Xiao's avatar
Shucai Xiao committed
419
void lowering::apply(module& m) const { miopen_apply{&m, this}.apply(); }
Shucai Xiao's avatar
Shucai Xiao committed
420

Paul's avatar
Paul committed
421
} // namespace gpu
Paul's avatar
Paul committed
422
} // namespace MIGRAPHX_INLINE_NS
Paul's avatar
Paul committed
423
} // namespace migraphx