quantization.cpp 19.3 KB
Newer Older
Shucai Xiao's avatar
Shucai Xiao committed
1
#include <migraphx/quantization.hpp>
2
3
4
#include <migraphx/program.hpp>
#include <migraphx/instruction.hpp>
#include <migraphx/iterator_for.hpp>
5
#include <migraphx/op/convert.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
6
7
8
#include <migraphx/op/dot.hpp>
#include <migraphx/op/mul.hpp>
#include <migraphx/op/add.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
9
#include <migraphx/op/quant_dot.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
10
#include <migraphx/op/capture.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
11
#include <migraphx/op/convolution.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
12
#include <migraphx/op/quant_convolution.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
13
#include <migraphx/op/multibroadcast.hpp>
14
#include <migraphx/stringutils.hpp>
15
#include <migraphx/ranges.hpp>
16
17
18
19
20
#include <utility>

namespace migraphx {
inline namespace MIGRAPHX_INLINE_NS {

Shucai Xiao's avatar
Shucai Xiao committed
21
22
23
24
25
26
instruction_ref insert_quant_ins(program& prog,
                                 instruction_ref& ins,
                                 shape::type_t type,
                                 std::unordered_map<instruction_ref, instruction_ref>& map_ins,
                                 float scale = 1.0f,
                                 float shift = 0.0f)
27
{
Shucai Xiao's avatar
Shucai Xiao committed
28
    if(map_ins.count(ins) > 0)
29
    {
Shucai Xiao's avatar
Shucai Xiao committed
30
31
32
33
34
35
        return map_ins[ins];
    }

    if(ins->name() == "undefined")
    {
        return ins;
36
37
    }

Shucai Xiao's avatar
Shucai Xiao committed
38
    assert(ins->get_shape().type() == shape::float_type ||
Shucai Xiao's avatar
Shucai Xiao committed
39
40
41
42
43
           ins->get_shape().type() == shape::double_type ||
           ins->get_shape().type() == shape::int32_type);
    instruction_ref quant_ins{};
    quant_ins    = prog.insert_instruction(std::next(ins), op::convert{type, scale, shift}, ins);
    map_ins[ins] = quant_ins;
44

Shucai Xiao's avatar
Shucai Xiao committed
45
    return quant_ins;
46
47
}

Shucai Xiao's avatar
Shucai Xiao committed
48
49
50
51
52
// This function is to convert any instructions specified in the input
// from double or float to float16 by inserting a convert operator.
// For the conversion, there could be cases of overflowing, but it
// is very rare in the area of deeping learning, so we just do a
// truncate of the input to get the fp16.
53
void quantize(program& prog, const std::vector<std::string>& ins_names)
54
{
55
    std::unordered_map<instruction_ref, instruction_ref> map_fp16;
Shucai Xiao's avatar
Shucai Xiao committed
56
    for(auto ins : iterator_for(prog))
57
    {
58
        // all indicates every instruction is converted
Shucai Xiao's avatar
Shucai Xiao committed
59
        if((not contains(ins_names, "all")) and (not contains(ins_names, ins->name())))
60
61
62
        {
            continue;
        }
63

64
        shape::type_t orig_type = ins->get_shape().type();
Shucai Xiao's avatar
Shucai Xiao committed
65
        // process all inputs, if input is a fp32 or fp64, convert it
66
        // to a fp16 by adding a convert operator.
67
        auto inputs = ins->inputs();
68
        std::vector<instruction_ref> converted_inputs;
Shucai Xiao's avatar
Shucai Xiao committed
69
        for(auto input : inputs)
70
71
        {
            auto s = input->get_shape();
Shucai Xiao's avatar
Shucai Xiao committed
72
            if(s.type() == shape::float_type || s.type() == shape::double_type)
73
            {
74
                // if the input is a convert operator, uses its input
75
76
                // as its current input
                instruction_ref input_fp16{};
77
                if(input->name() == "convert")
78
79
80
81
82
                {
                    input_fp16 = input->inputs().front();
                }
                else
                {
Shucai Xiao's avatar
Shucai Xiao committed
83
                    input_fp16 = insert_quant_ins(prog, input, shape::half_type, map_fp16);
84
                }
85
                converted_inputs.push_back(input_fp16);
86
            }
87
88
89
90
91
92
            else
            {
                converted_inputs.push_back(input);
            }
        }

93
        // no change for the input, go to the next instruction
Shucai Xiao's avatar
Shucai Xiao committed
94
        if(inputs == converted_inputs)
95
        {
96
            continue;
Shucai Xiao's avatar
Shucai Xiao committed
97
98
99
100
101
102
        }

        auto op        = ins->get_operator();
        auto ins_shape = compute_shape(op, converted_inputs);
        if(ins_shape.type() != orig_type)
        {
Shucai Xiao's avatar
Shucai Xiao committed
103
104
105
106
107
            // check the dead code case to avoid assert
            bool output_empty = ins->outputs().empty();
            auto ins_orig_type =
                prog.insert_instruction(std::next(ins), op::convert{orig_type}, ins);
            if(!output_empty)
108
            {
Shucai Xiao's avatar
Shucai Xiao committed
109
                prog.replace_instruction(ins, ins_orig_type);
110
            }
111
        }
Shucai Xiao's avatar
Shucai Xiao committed
112
113

        prog.replace_instruction(ins, op, converted_inputs);
114
115
116
    }
}

Shucai Xiao's avatar
Shucai Xiao committed
117
void quantize(program& prog) { quantize(prog, {"all"}); }
Shucai Xiao's avatar
Shucai Xiao committed
118

Shucai Xiao's avatar
Shucai Xiao committed
119
// int8 quantization is different from fp16 since int8 can only handle value
Shucai Xiao's avatar
Shucai Xiao committed
120
// -128 ~ 127. To convert the float or double to int8, we need a scale and
Shucai Xiao's avatar
Shucai Xiao committed
121
// a shift, then the convert can be done as v_int8 = fp * scale + shift.
Shucai Xiao's avatar
Shucai Xiao committed
122
// To simplify the changes, we consider shift as 0.0f for now.
Shucai Xiao's avatar
Shucai Xiao committed
123
124
125
void quantize_int8(program& prog,
                   const std::vector<std::string>& ins_names,
                   std::vector<std::pair<float, float>>& int8_quant_params)
Shucai Xiao's avatar
Shucai Xiao committed
126
{
Shucai Xiao's avatar
Shucai Xiao committed
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
    // // For debugging
    // auto print_gemm_res = [&](std::size_t ins_index, std::vector<migraphx::argument> args) {
    //     // scale and shift is need for only int8 type, and we do not
    //     // consider shift, so set shift to 0
    //     std::vector<float> vec_val;
    //     args.front().visit([&](auto output) { vec_val.assign(output.begin(), output.end()); });
    //     std::cout << "quant_gemm = " << std::endl;
    //     for (size_t i = 0; i < 20; i++)
    //     {
    //         std::cout << vec_val[i] << "\t";
    //     }
    //     std::cout << std::endl;
    // };

    // // For debugging
    // auto print_conv_res = [&](std::size_t ins_index, std::vector<migraphx::argument> args) {
    //     // scale and shift is need for only int8 type, and we do not
    //     // consider shift, so set shift to 0
    //     std::vector<float> vec_val;
    //     args.front().visit([&](auto output) { vec_val.assign(output.begin(), output.end()); });
    //     std::cout << "quant_conv = " << std::endl;
    //     for (size_t i = 0; i < 20; i++)
    //     {
    //         std::cout << vec_val[i] << "\t";
    //     }
    //     std::cout << std::endl;
    // };

Shucai Xiao's avatar
Shucai Xiao committed
155
156
    // For now, we only support the int8 quantization of gemm and convolution
    std::vector<std::string> op_names = {"dot", "convolution"};
Shucai Xiao's avatar
Shucai Xiao committed
157
    if(!std::all_of(ins_names.begin(), ins_names.end(), [&](auto name) {
Shucai Xiao's avatar
Shucai Xiao committed
158
           return (std::find(op_names.begin(), op_names.end(), name) != op_names.end());
Shucai Xiao's avatar
Shucai Xiao committed
159
       }))
Shucai Xiao's avatar
Shucai Xiao committed
160
161
162
163
    {
        MIGRAPHX_THROW("QUANTIZE_INT8: only support DOT and CONVOLUTION operation");
    }

Shucai Xiao's avatar
Shucai Xiao committed
164
    std::size_t quant_param_index = 0;
Shucai Xiao's avatar
Shucai Xiao committed
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
    std::unordered_map<instruction_ref, instruction_ref> map_quant_ins;
    for(auto ins : iterator_for(prog))
    {
        if(not contains(ins_names, ins->name()))
        {
            continue;
        }

        shape::type_t orig_type = ins->get_shape().type();

        // for the dot operator, there could be 2 or 3 input arguments
        // if the 3rd argument is available, convert it to an int32.
        std::vector<instruction_ref> converted_inputs;

        // process all inputs, if input is a fp32 or fp64, convert it
Shucai Xiao's avatar
Shucai Xiao committed
180
        // to a int8 type by adding a convert operator and replace
Shucai Xiao's avatar
Shucai Xiao committed
181
        // the operator with the corresponding int8 version
Shucai Xiao's avatar
Shucai Xiao committed
182
183
        auto inputs = ins->inputs();
        std::vector<std::pair<float, float>> ins_quant_params;
Shucai Xiao's avatar
Shucai Xiao committed
184
185
        for(auto input : inputs)
        {
Shucai Xiao's avatar
Shucai Xiao committed
186
187
            // In general, the target_type is int8, but for the dot
            // operation, if it has 3 inputs, then the last one should
Shucai Xiao's avatar
Shucai Xiao committed
188
189
            // be converted to int32_type
            shape::type_t quant_type = shape::int8_type;
Shucai Xiao's avatar
Shucai Xiao committed
190
191
            auto param               = int8_quant_params[quant_param_index++];
            ins_quant_params.push_back(param);
Shucai Xiao's avatar
Shucai Xiao committed
192
            if(ins->name() == "dot" and inputs.size() == 3 and input == inputs.back())
Shucai Xiao's avatar
Shucai Xiao committed
193
            {
Shucai Xiao's avatar
Shucai Xiao committed
194
195
                quant_type = shape::int32_type;
            }
Shucai Xiao's avatar
Shucai Xiao committed
196

Shucai Xiao's avatar
Shucai Xiao committed
197
            auto s = input->get_shape();
198
            if((s.type() == shape::float_type || s.type() == shape::double_type ||
Shucai Xiao's avatar
Shucai Xiao committed
199
200
                s.type() == shape::int32_type) &&
               s.type() != quant_type)
Shucai Xiao's avatar
Shucai Xiao committed
201
202
203
204
205
206
207
            {
                // if the input is a convert operator, uses its input
                // as its current input
                instruction_ref quant_input{};
                if(input->name() == "convert")
                {
                    auto tmp_ins = input->inputs().front();
Shucai Xiao's avatar
Shucai Xiao committed
208
                    if(tmp_ins->get_shape().type() == quant_type)
Shucai Xiao's avatar
Shucai Xiao committed
209
210
211
212
213
                    {
                        quant_input = input->inputs().front();
                    }
                    else
                    {
Shucai Xiao's avatar
Shucai Xiao committed
214
215
                        quant_input = insert_quant_ins(
                            prog, input, quant_type, map_quant_ins, param.first, param.second);
Shucai Xiao's avatar
Shucai Xiao committed
216
217
218
                    }
                }
                else
219
                {
Shucai Xiao's avatar
Shucai Xiao committed
220
221
                    quant_input = insert_quant_ins(
                        prog, input, quant_type, map_quant_ins, param.first, param.second);
222
                }
Shucai Xiao's avatar
Shucai Xiao committed
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
                converted_inputs.push_back(quant_input);
            }
            else
            {
                converted_inputs.push_back(input);
            }
        }

        // no change for the input, go to the next instruction
        if(inputs == converted_inputs)
        {
            continue;
        }

        // When converting from other types to int8_type, there are parameters
        // used as scale and shift(.0f), which will generate results diffrent from
        // the original results. To adjust the output to be "correct(approximatly
Shucai Xiao's avatar
Shucai Xiao committed
240
        // equal)", we need additional calculation for the adjustment
Shucai Xiao's avatar
Shucai Xiao committed
241
        if(ins->name() == "dot")
Shucai Xiao's avatar
Shucai Xiao committed
242
        {
Shucai Xiao's avatar
Shucai Xiao committed
243
244
245
246
            auto dot_op = any_cast<op::dot>(ins->get_operator());
            float new_alpha =
                dot_op.alpha / (ins_quant_params[0].first * ins_quant_params[1].first);
            float new_beta = dot_op.beta;
Shucai Xiao's avatar
Shucai Xiao committed
247
            // We need additional checking about the quant_alpha value. If
248
249
250
            // abs(quant_alpha) > 50 (some tmp value set here), we can convert
            // it to an integer as the new_alpha in the quant_dot
            float threshold = 50.0f;
Shucai Xiao's avatar
Shucai Xiao committed
251
            if(fabs(new_alpha) >= threshold && fabs(new_beta) >= threshold)
252
253
            {
                int32_t quant_alpha = static_cast<int32_t>(new_alpha);
Shucai Xiao's avatar
Shucai Xiao committed
254
255
256
                int32_t quant_beta  = static_cast<int32_t>(new_beta);
                shape quant_shape   = compute_shape(op::quant_dot{1, 0}, converted_inputs);
                if(quant_shape.type() == orig_type)
257
                {
Shucai Xiao's avatar
Shucai Xiao committed
258
259
                    prog.replace_instruction(
                        ins, op::quant_dot{quant_alpha, quant_beta}, converted_inputs);
260
261
262
                }
                else
                {
Shucai Xiao's avatar
Shucai Xiao committed
263
264
                    auto quant_dot = prog.insert_instruction(
                        ins, op::quant_dot{quant_alpha, quant_beta}, converted_inputs);
265
266
267
                    prog.replace_instruction(ins, op::convert{orig_type}, quant_dot);
                }
            }
Shucai Xiao's avatar
Shucai Xiao committed
268
269
            // only alpha can be quantized, quantization of beta will cause
            // big error, so we have to manually do the multiplication and
270
            // addition
Shucai Xiao's avatar
Shucai Xiao committed
271
            else if(fabs(new_alpha) >= threshold)
272
273
            {
                int32_t quant_alpha = static_cast<int32_t>(new_alpha);
Shucai Xiao's avatar
Shucai Xiao committed
274
275
                int32_t quant_beta  = 0;
                if(orig_type == shape::int32_type)
276
                {
Shucai Xiao's avatar
Shucai Xiao committed
277
                    if(inputs.size() == 2 or dot_op.beta == 0.0f)
278
                    {
Shucai Xiao's avatar
Shucai Xiao committed
279
280
                        prog.replace_instruction(
                            ins, op::quant_dot{quant_alpha, quant_beta}, converted_inputs);
281
282
283
284
                    }
                    // if there are 3 inputs, we need to consider the third argument
                    else
                    {
Shucai Xiao's avatar
Shucai Xiao committed
285
286
                        auto q_dot = prog.insert_instruction(
                            ins, op::quant_dot{quant_alpha, quant_beta}, converted_inputs);
287
288
                        std::vector<float> vec_beta(q_dot->get_shape().elements(), dot_op.beta);
                        auto l_beta = prog.add_literal(literal{orig_type, vec_beta});
Shucai Xiao's avatar
Shucai Xiao committed
289
290
                        auto beta_c =
                            prog.insert_instruction(ins, op::mul{}, l_beta, inputs.back());
291
292
293
294
295
                        prog.replace_instruction(ins, op::add{}, q_dot, beta_c);
                    }
                }
                else
                {
Shucai Xiao's avatar
Shucai Xiao committed
296
                    if(inputs.size() == 2 or dot_op.beta == 0.0f)
297
                    {
Shucai Xiao's avatar
Shucai Xiao committed
298
299
                        auto q_dot = prog.insert_instruction(
                            ins, op::quant_dot{quant_alpha, quant_beta}, converted_inputs);
300
301
302
303
304
                        prog.replace_instruction(ins, op::convert{orig_type}, q_dot);
                    }
                    // if there are 3 inputs, we need to consider the third argument
                    else
                    {
Shucai Xiao's avatar
Shucai Xiao committed
305
306
                        auto q_dot = prog.insert_instruction(
                            ins, op::quant_dot{quant_alpha, quant_beta}, converted_inputs);
307
308
309
                        auto oq_dot = prog.insert_instruction(ins, op::convert{orig_type}, q_dot);
                        std::vector<float> vec_beta(q_dot->get_shape().elements(), dot_op.beta);
                        auto l_beta = prog.add_literal(literal{oq_dot->get_shape(), vec_beta});
Shucai Xiao's avatar
Shucai Xiao committed
310
311
                        auto beta_c =
                            prog.insert_instruction(ins, op::mul{}, l_beta, inputs.back());
312
313
314
315
316
317
318
319
                        prog.replace_instruction(ins, op::add{}, q_dot, beta_c);
                    }
                }
            }
            else
            {
                auto q_dot = prog.insert_instruction(ins, op::quant_dot{1, 0}, converted_inputs);
                std::vector<float> vec_alpha(q_dot->get_shape().elements(), new_alpha);
Shucai Xiao's avatar
Shucai Xiao committed
320
                if(orig_type == shape::int32_type)
321
322
                {
                    auto l_alpha = prog.add_literal(literal(ins->get_shape(), vec_alpha));
Shucai Xiao's avatar
Shucai Xiao committed
323
                    if(converted_inputs.size() == 2 or dot_op.beta == 0.0f)
324
325
326
327
328
329
330
                    {
                        prog.replace_instruction(ins, op::mul{}, l_alpha, q_dot);
                    }
                    // case of 3 arguments
                    else
                    {
                        std::vector<float> vec_beta(ins->get_shape().elements(), new_beta);
Shucai Xiao's avatar
Shucai Xiao committed
331
                        auto l_beta   = prog.add_literal(literal(ins->get_shape(), vec_beta));
332
                        auto alpha_ab = prog.insert_instruction(ins, op::mul{}, l_alpha, q_dot);
Shucai Xiao's avatar
Shucai Xiao committed
333
334
                        auto beta_c =
                            prog.insert_instruction(ins, op::mul{}, l_beta, inputs.back());
335
336
337
338
339
                        prog.replace_instruction(ins, op::add{}, alpha_ab, beta_c);
                    }
                }
                else
                {
Shucai Xiao's avatar
Shucai Xiao committed
340
                    auto oq_dot  = prog.insert_instruction(ins, op::convert{orig_type}, q_dot);
341
                    auto l_alpha = prog.add_literal(literal(ins->get_shape(), vec_alpha));
Shucai Xiao's avatar
Shucai Xiao committed
342
                    if(converted_inputs.size() == 2 or dot_op.beta == 0.0f)
343
344
345
346
347
348
349
                    {
                        prog.replace_instruction(ins, op::mul{}, l_alpha, oq_dot);
                    }
                    // case of 3 arguments
                    else
                    {
                        std::vector<float> vec_beta(ins->get_shape().elements(), new_beta);
Shucai Xiao's avatar
Shucai Xiao committed
350
                        auto l_beta   = prog.add_literal(literal(ins->get_shape(), vec_beta));
351
                        auto alpha_ab = prog.insert_instruction(ins, op::mul{}, l_alpha, oq_dot);
Shucai Xiao's avatar
Shucai Xiao committed
352
353
                        auto beta_c =
                            prog.insert_instruction(ins, op::mul{}, l_beta, inputs.back());
354
355
356
357
                        prog.replace_instruction(ins, op::add{}, alpha_ab, beta_c);
                    }
                }
            }
Shucai Xiao's avatar
Shucai Xiao committed
358
        }
Shucai Xiao's avatar
Shucai Xiao committed
359
        else if(ins->name() == "convolution")
Shucai Xiao's avatar
Shucai Xiao committed
360
        {
Shucai Xiao's avatar
Shucai Xiao committed
361
            // Current MIOpen convolution does not support alpha and beta,
Shucai Xiao's avatar
Shucai Xiao committed
362
            // so we need a separate multiply to adjust the output
Shucai Xiao's avatar
Shucai Xiao committed
363
364
365
366
367
368
            auto conv_op       = any_cast<op::convolution>(ins->get_operator());
            auto padding       = conv_op.padding;
            auto stride        = conv_op.stride;
            auto dilation      = conv_op.dilation;
            auto padding_mode  = conv_op.padding_mode;
            auto group         = conv_op.group;
Shucai Xiao's avatar
Shucai Xiao committed
369
            auto adjust_factor = 1.0 / (ins_quant_params[0].first * ins_quant_params[1].first);
Shucai Xiao's avatar
Shucai Xiao committed
370

Shucai Xiao's avatar
Shucai Xiao committed
371
372
373
            shape quant_shape =
                compute_shape(op::quant_convolution{padding, stride, dilation, padding_mode, group},
                              converted_inputs);
374
            std::vector<float> vec_factor(quant_shape.elements(), adjust_factor);
Shucai Xiao's avatar
Shucai Xiao committed
375
376
            auto fl = prog.add_literal(literal{{orig_type, quant_shape.lens()}, vec_factor});
            if(quant_shape.type() == orig_type)
377
            {
Shucai Xiao's avatar
Shucai Xiao committed
378
                if(adjust_factor == 1.0f)
379
                {
Shucai Xiao's avatar
Shucai Xiao committed
380
381
382
383
                    prog.replace_instruction(
                        ins,
                        op::quant_convolution{padding, stride, dilation, padding_mode, group},
                        converted_inputs);
384
385
386
                }
                else
                {
Shucai Xiao's avatar
Shucai Xiao committed
387
                    auto quant_conv = prog.insert_instruction(
Shucai Xiao's avatar
Shucai Xiao committed
388
389
390
                        ins,
                        op::quant_convolution{padding, stride, dilation, padding_mode, group},
                        converted_inputs);
391
392
393
394
395
                    prog.replace_instruction(ins, op::mul{}, quant_conv, fl);
                }
            }
            else
            {
Shucai Xiao's avatar
Shucai Xiao committed
396
397
398
399
400
                auto quant_conv = prog.insert_instruction(
                    ins,
                    op::quant_convolution{padding, stride, dilation, padding_mode, group},
                    converted_inputs);
                if(adjust_factor == 1.0f)
401
402
403
404
405
406
407
408
409
                {
                    prog.replace_instruction(ins, op::convert{orig_type}, quant_conv);
                }
                else
                {
                    auto oq_conv = prog.insert_instruction(ins, op::convert{orig_type}, quant_conv);
                    prog.replace_instruction(ins, op::mul{}, oq_conv, fl);
                }
            }
Shucai Xiao's avatar
Shucai Xiao committed
410
411
412
413
414
        }
        else
        {
            MIGRAPHX_THROW("INT8_QUANTIZE: does not support operator" + ins->name());
        }
415
416
417
    }
}

Shucai Xiao's avatar
Shucai Xiao committed
418
419
// For the input of each input argument, we need to insert a
// capture operator to compute the scale and shift
Shucai Xiao's avatar
Shucai Xiao committed
420
421
422
423
void capture_arguments(program& prog,
                       const std::vector<std::string>& ins_names,
                       std::size_t& num_quant_params,
                       std::function<void(std::size_t, std::vector<argument> args)> func)
Shucai Xiao's avatar
Shucai Xiao committed
424
{
Shucai Xiao's avatar
Shucai Xiao committed
425
    num_quant_params = 0;
Shucai Xiao's avatar
Shucai Xiao committed
426
    // the int8 quantization only support dot and convolution
Shucai Xiao's avatar
Shucai Xiao committed
427
    std::vector<std::string> op_names = {"dot", "convolution", "quant_dot", "quant_convolution"};
Shucai Xiao's avatar
Shucai Xiao committed
428
429
430
    if(!std::all_of(ins_names.begin(), ins_names.end(), [&](auto name) {
           return std::find(op_names.begin(), op_names.end(), name) != op_names.end();
       }))
Shucai Xiao's avatar
Shucai Xiao committed
431
432
433
434
435
436
437
    {
        MIGRAPHX_THROW("CAPTURE_ARGUMENTS: input operator is not supported");
    }

    std::unordered_map<instruction_ref, instruction_ref> ins_map;
    for(auto ins : iterator_for(prog))
    {
Shucai Xiao's avatar
Shucai Xiao committed
438
        if(not contains(ins_names, ins->name()))
Shucai Xiao's avatar
Shucai Xiao committed
439
440
441
442
443
444
        {
            continue;
        }

        auto inputs = ins->inputs();
        std::vector<instruction_ref> new_args;
Shucai Xiao's avatar
Shucai Xiao committed
445
        for(auto input : inputs)
Shucai Xiao's avatar
Shucai Xiao committed
446
447
        {
            instruction_ref new_ins{};
Shucai Xiao's avatar
Shucai Xiao committed
448
            if(ins_map.count(input) > 0)
Shucai Xiao's avatar
Shucai Xiao committed
449
450
451
452
453
            {
                new_ins = ins_map[input];
            }
            else
            {
Shucai Xiao's avatar
Shucai Xiao committed
454
                new_ins = prog.insert_instruction(
Shucai Xiao's avatar
Shucai Xiao committed
455
                    std::next(input), op::capture{num_quant_params++, func}, input);
Shucai Xiao's avatar
Shucai Xiao committed
456
457
458
459
460
461
462
463
                ins_map[input] = new_ins;
            }
            new_args.push_back(new_ins);
        }
        instruction::replace(ins, ins->get_operator(), ins->get_shape(), new_args);
    }
}

464
465
} // namespace MIGRAPHX_INLINE_NS
} // namespace migraphx