multitarget_test.cpp 22 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
/*
 * The MIT License (MIT)
 *
 * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
#include <iostream>
#include <set>
#include <unordered_set>
#include <vector>
#include <random>
#include <cmath>
#include <migraphx/program.hpp>
#include <migraphx/target.hpp>
#include <migraphx/stringutils.hpp>
#include <migraphx/module.hpp>
#include <migraphx/literal.hpp>
#include <migraphx/instruction.hpp>
#include <migraphx/shape.hpp>
#include <migraphx/make_op.hpp>
#include <migraphx/check_shapes.hpp>
#include <migraphx/functional.hpp>
#include <migraphx/compile_options.hpp>
#include <migraphx/register_target.hpp>
umangyadav's avatar
umangyadav committed
42
#include <migraphx/generate.hpp>
umang yadav's avatar
umang yadav committed
43
#include <migraphx/generate_root_modules.hpp>
umang yadav's avatar
umang yadav committed
44
#include <migraphx/target_assignments.hpp>
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
#include "test.hpp"

// check if it is custom_op or run_on_module operator
bool has_target_attr(const migraphx::instruction& ins)
{
    return ins.get_operator().attributes().contains("target");
}

auto nonprefixed_ops()
{
    // ops without prefixes
    static std::unordered_set<std::string> op_map = {
        "select_module", "load", "if", "nonmaxsuppression", "multibroadcast"};
    return op_map;
}

bool is_compiled_gpu_module(const migraphx::module& m)
{
    return std::all_of(m.begin(), m.end(), [](auto ins) {
        auto ins_name = ins.name();
        if(not migraphx::starts_with(ins_name, "@"))
        {
            if(not migraphx::starts_with(ins_name, "gpu::") and
               not migraphx::starts_with(ins_name, "hip::") and
               not migraphx::starts_with(ins_name, "check_context") and
               not migraphx::contains(nonprefixed_ops(), ins_name) and not has_target_attr(ins))
            {
                return false;
            }
        }
        return true;
    });
}

bool is_compiled_fpga_module(const migraphx::module& m)
{
    return std::all_of(m.begin(), m.end(), [](auto ins) {
        auto ins_name = ins.name();
        if(not migraphx::starts_with(ins_name, "@"))
        {
            if(not migraphx::starts_with(ins_name, "fpga::") and
               not migraphx::starts_with(ins_name, "check_context") and
               not migraphx::contains(nonprefixed_ops(), ins_name) and not has_target_attr(ins))
            {
                return false;
            }
        }
        return true;
    });
}

bool is_compiled_cpu_module(const migraphx::module& m)
{
    return std::all_of(m.begin(), m.end(), [](auto ins) {
        auto ins_name = ins.name();
        if(not migraphx::starts_with(ins_name, "@"))
        {
            if(not migraphx::starts_with(ins_name, "cpu::") and
               not migraphx::starts_with(ins_name, "dnnl::") and
               not migraphx::starts_with(ins_name, "check_context") and not has_target_attr(ins) and
               not migraphx::contains(nonprefixed_ops(), ins_name))
            {
                return false;
            }
        }
        return true;
    });
}

bool is_compiled_ref_module(const migraphx::module& m)
{
    return std::all_of(m.begin(), m.end(), [](auto ins) {
        auto ins_name = ins.name();
        if(not migraphx::starts_with(ins_name, "@"))
        {
            if((not migraphx::starts_with(ins_name, "ref::") and
                not migraphx::starts_with(ins_name, "check_context") and
                not has_target_attr(ins)) and
               not migraphx::contains(nonprefixed_ops(), ins_name))
            {
                return false;
            }
        }
        return true;
    });
}

// NOLINT
bool check_compiled_program(const migraphx::program& p,
                            const std::vector<migraphx::target>& targets)
{
    auto mods           = p.get_modules();
Umang Yadav's avatar
Umang Yadav committed
137
    bool rot_ins        = false;
138
139
140
141
142
143
144
    bool check_compiled = true;
    for(const auto* mod : mods)
    {
        for(const auto& ins : *mod)
        {
            if(ins.name() == "run_on_target")
            {
Umang Yadav's avatar
Umang Yadav committed
145
                rot_ins |= true;
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
                auto* mod_input = ins.module_inputs().front();
                std::size_t target_id =
                    ins.get_operator().to_value()["target_id"].to<std::size_t>();
                auto target_name = targets.at(target_id).name();
                if(target_name == "gpu")
                    check_compiled &= is_compiled_gpu_module(*mod_input);
                else if(target_name == "cpu")
                    check_compiled &= is_compiled_cpu_module(*mod_input);
                else if(target_name == "fpga")
                    check_compiled &= is_compiled_fpga_module(*mod_input);
                else if(target_name == "ref")
                    check_compiled &= is_compiled_ref_module(*mod_input);
            }
        }
    }
Umang Yadav's avatar
Umang Yadav committed
161
    return check_compiled and rot_ins;
162
163
}

164
TEST_CASE(multitarget_compile_cpu_gpu)
165
166
167
168
169
170
171
172
173
174
175
176
177
{
    migraphx::program p;
    auto* mm     = p.get_main_module();
    auto s       = migraphx::shape{migraphx::shape::float_type, {8}};
    auto x_param = mm->add_parameter("x", s);
    auto y_param = mm->add_parameter("y", s);
    auto z_param = mm->add_parameter("z", s);
    auto cpu_ins = mm->add_instruction(migraphx::make_op("add"), x_param, y_param);
    auto gpu_ins = mm->add_instruction(migraphx::make_op("add"), cpu_ins, z_param);
    mm->add_return({gpu_ins});
    migraphx::target_assignments tass;
    tass.insert(tass.begin(), std::make_pair(cpu_ins, 1));
    tass.insert(tass.begin(), std::make_pair(gpu_ins, 0));
umang yadav's avatar
umang yadav committed
178
    migraphx::generate_root_modules(p, tass);
umangyadav's avatar
umangyadav committed
179
180
181
    migraphx::compile_options gpu_opts;
    gpu_opts.offload_copy = true;
    p.compile({migraphx::make_target("gpu"), migraphx::make_target("cpu")}, {gpu_opts});
182
    EXPECT(check_compiled_program(p, {migraphx::make_target("gpu"), migraphx::make_target("cpu")}));
umangyadav's avatar
umangyadav committed
183
    migraphx::parameter_map params;
umangyadav's avatar
umangyadav committed
184
185
186
    params["x"] = migraphx::fill_argument(s, 1);
    params["y"] = migraphx::fill_argument(s, 2);
    params["z"] = migraphx::fill_argument(s, 3);
umangyadav's avatar
umangyadav committed
187
    auto result = p.eval(params).back();
umangyadav's avatar
umangyadav committed
188
189
    auto gold   = migraphx::fill_argument(s, 6);
    EXPECT(gold == result);
190
191
}

umangyadav's avatar
umangyadav committed
192
TEST_CASE(single_target_multi_compile)
193
194
195
{
    migraphx::program p;
    migraphx::shape boxes_s{migraphx::shape::float_type, {1, 6, 4}};
196
197
    auto* mm         = p.get_main_module();
    auto boxes_param = mm->add_parameter("boxes", boxes_s);
198
199
200

    migraphx::shape scores_s{migraphx::shape::float_type, {1, 1, 6}};
    std::vector<float> scores_vec = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3};
201
202
203
204
205
    auto scores_l                 = mm->add_literal(migraphx::literal(scores_s, scores_vec));
    auto max_out_l                = mm->add_literal(int64_t{4});
    auto iou_threshold            = mm->add_literal(0.5f);
    auto score_threshold          = mm->add_literal(0.0f);
    auto r                        = mm->add_instruction(
206
        migraphx::make_op("nonmaxsuppression",
umangyadav's avatar
umangyadav committed
207
                                                 {{"center_point_box", true}, {"use_dyn_output", true}}),
208
        boxes_param,
209
210
211
212
        scores_l,
        max_out_l,
        iou_threshold,
        score_threshold);
213
    mm->add_return({r});
umang yadav's avatar
umang yadav committed
214
    // do target assignments
215
216
    migraphx::target_assignments tass;
    tass.insert(tass.begin(), std::make_pair(r, 0));
umang yadav's avatar
umang yadav committed
217
    migraphx::generate_root_modules(p, tass);
218
    // compile using multi-target compilation path
umangyadav's avatar
umangyadav committed
219
220
    migraphx::compile_options gpu_opts;
    gpu_opts.offload_copy = true;
221
222
223
224
225
    // need to add "ref" to avoid ambigious call to "compile()"
    p.compile({migraphx::make_target("gpu"), migraphx::make_target("ref")}, {gpu_opts});
    EXPECT(check_compiled_program(p, {migraphx::make_target("gpu"), migraphx::make_target("ref")}));
    // eval
    migraphx::parameter_map params;
umangyadav's avatar
umangyadav committed
226
    std::vector<float> boxes_vec  = {0.5, 0.5,  1.0, 1.0, 0.5, 0.6,  1.0, 1.0, 0.5, 0.4,   1.0, 1.0,
227
                                     0.5, 10.5, 1.0, 1.0, 0.5, 10.6, 1.0, 1.0, 0.5, 100.5, 1.0, 1.0};
umangyadav's avatar
umangyadav committed
228
229
230
231
232
233
    params["boxes"]               = migraphx::argument(boxes_s, boxes_vec.data());
    auto output                   = p.eval(params).back();
    std::vector<int64_t> gold_vec = {0, 0, 3, 0, 0, 0, 0, 0, 5};
    auto gold =
        migraphx::argument(migraphx::shape{migraphx::shape::int64_type, {3, 3}}, gold_vec.data());
    EXPECT(output == gold);
234
235
}

umang yadav's avatar
umang yadav committed
236
TEST_CASE(multitarget_compile_if_then_else)
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
{
    migraphx::program p;
    auto* mm = p.get_main_module();
    migraphx::shape cond_s{migraphx::shape::bool_type};
    auto cond = mm->add_parameter("cond", cond_s);
    migraphx::shape ds{migraphx::shape::float_type, {2, 3}};
    auto x = mm->add_parameter("x", ds);
    auto y = mm->add_parameter("y", ds);

    auto* then_mod = p.create_module("if_gpu_mod");
    std::vector<float> data1(ds.elements(), 1);
    auto l1 = then_mod->add_literal(migraphx::literal(ds, data1));
    auto a1 = then_mod->add_instruction(migraphx::make_op("add"), x, l1);
    then_mod->add_return({a1});

    auto* else_mod = p.create_module("else_cpu_mod");
    std::vector<float> data2(ds.elements(), 2);
    auto l2 = else_mod->add_literal(migraphx::literal(ds, data2));
    auto a2 = else_mod->add_instruction(migraphx::make_op("mul"), y, l2);
    else_mod->add_return({a2});

    auto ret = mm->add_instruction(migraphx::make_op("if"), {cond}, {then_mod, else_mod});
    auto r   = mm->add_instruction(migraphx::make_op("get_tuple_elem", {{"index", 0}}), ret);
    mm->add_return({r});
    migraphx::target_assignments tass;
    tass.insert(tass.begin(), std::make_pair(l1, 0));
    tass.insert(tass.begin(), std::make_pair(a1, 0));
    tass.insert(tass.begin(), std::make_pair(l2, 1));
    tass.insert(tass.begin(), std::make_pair(a2, 1));
umang yadav's avatar
umang yadav committed
266
    migraphx::generate_root_modules(p, tass);
267
268
269
    // compile
    migraphx::compile_options gpu_opts;
    gpu_opts.offload_copy = true;
umangyadav's avatar
umangyadav committed
270
271
272
273
274
275
276
    p.compile(
        {migraphx::make_target("gpu"), migraphx::make_target("cpu"), migraphx::make_target("ref")},
        {gpu_opts});
    EXPECT(check_compiled_program(p,
                                  {migraphx::make_target("gpu"),
                                   migraphx::make_target("cpu"),
                                   migraphx::make_target("ref")}));
277
278
279
280
281
282
283
284
285
286
287
288
    migraphx::parameter_map params;
    params["x"] = migraphx::fill_argument(ds, 2);
    params["y"] = migraphx::fill_argument(ds, 3);
    for(bool cond_val : {true, false})
    {
        params["cond"] = migraphx::argument(cond_s, &cond_val);
        auto result    = p.eval(params).back();
        auto gold      = migraphx::fill_argument(ds, (cond_val ? 3 : 6));
        EXPECT(gold == result);
    }
}

umangyadav's avatar
umangyadav committed
289
// TODO : FPGA compilation is broken right now, below test mentions fpga but doesn't compile for it
290
TEST_CASE(multitarget_compile_nested_if_then_else)
291
292
293
294
295
296
297
{
    std::unordered_map<std::size_t, std::size_t> counter_map = {{0, 0}, {1, 0}};
    migraphx::shape ds{migraphx::shape::float_type, {2, 3}};
    migraphx::target_assignments tass;
    migraphx::program p;
    auto* mm = p.get_main_module();
    migraphx::shape cond_s{migraphx::shape::bool_type};
umang yadav's avatar
umang yadav committed
298
299
300
301
302
303
    auto cond_0 = mm->add_parameter("cond_0", cond_s);
    auto cond_1 = mm->add_parameter("cond_1", cond_s);
    auto x      = mm->add_parameter("x", ds);
    auto y      = mm->add_parameter("y", ds);
    auto z      = mm->add_parameter("z", ds);
    auto create_test_module =
Umang Yadav's avatar
Umang Yadav committed
304
        [&](migraphx::program& prog, std::size_t tid, const std::string& param_prefix) {
umang yadav's avatar
umang yadav committed
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
            std::string mod_name =
                "target_" + std::to_string(tid) + "_" + std::to_string(counter_map[tid]++);
            auto* test_mod = prog.create_module(mod_name);
            std::vector<float> data(ds.elements(), -1);
            auto l1               = test_mod->add_literal(migraphx::literal(ds, data));
            auto test_mod_param_0 = test_mod->add_parameter(param_prefix + "_param_0", ds);
            auto test_mod_param_1 = test_mod->add_parameter(param_prefix + "_param_1", ds);
            auto test_mod_param_2 = test_mod->add_parameter(param_prefix + "_param_2", ds);
            auto ins1 = test_mod->add_instruction(migraphx::make_op("add"), test_mod_param_0, l1);
            auto ins2 = test_mod->add_instruction(migraphx::make_op("mul"), ins1, test_mod_param_1);
            auto ins3 = test_mod->add_instruction(migraphx::make_op("sub"), ins2, test_mod_param_2);
            test_mod->add_return({ins3});
            tass.insert(tass.begin(), std::make_pair(ins1, tid));
            tass.insert(tass.begin(), std::make_pair(ins2, tid));
            tass.insert(tass.begin(), std::make_pair(ins3, tid));
            return test_mod;
        };
322
323
324
325
326
327
328
329
330

    auto* then_mod        = p.create_module("then_mod");
    auto then_mod_cond    = then_mod->add_parameter("then_mod_cond", cond_s);
    auto then_mod_param_0 = then_mod->add_parameter("then_mod_param_0", ds);
    auto then_mod_param_1 = then_mod->add_parameter("then_mod_param_1", ds);
    auto then_mod_param_2 = then_mod->add_parameter("then_mod_param_2", ds);
    auto then_mod_ref_ins =
        then_mod->add_instruction(migraphx::make_op("add"), then_mod_param_0, then_mod_param_1);
    tass.insert(tass.begin(), std::make_pair(then_mod_ref_ins, 3));
umang yadav's avatar
umang yadav committed
331
332
333
334
335
336
337
338
339
    auto then_mod_if =
        then_mod->add_instruction(migraphx::make_op("if"),
                                  {then_mod_cond,
                                   then_mod_param_0,
                                   then_mod_param_1,
                                   then_mod_param_2,
                                   then_mod_ref_ins,
                                   then_mod_param_1,
                                   then_mod_param_2},
umang yadav's avatar
umang yadav committed
340
                                  {create_test_module(p, 1, "1_"), create_test_module(p, 0, "2_")});
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
    auto then_mod_if_0 =
        then_mod->add_instruction(migraphx::make_op("get_tuple_elem", {{"index", 0}}), then_mod_if);
    then_mod->add_return({then_mod_if_0});

    // create nested else_mod with multiple targets.
    // else_mod has one instruction that runs a module on "fpga" and another instruction that
    // creates nested modules using "If" that runs on "cpu" and "gpu"
    auto* else_mod        = p.create_module("else_mod");
    auto else_mod_cond    = else_mod->add_parameter("else_mod_cond", cond_s);
    auto else_mod_param_0 = else_mod->add_parameter("else_mod_param_0", ds);
    auto else_mod_param_1 = else_mod->add_parameter("else_mod_param_1", ds);
    auto else_mod_param_2 = else_mod->add_parameter("else_mod_param_2", ds);
    auto else_mod_fpga_ins =
        else_mod->add_instruction(migraphx::make_op("add"), else_mod_param_0, else_mod_param_2);
    tass.insert(tass.begin(), std::make_pair(else_mod_fpga_ins, 2));
umang yadav's avatar
umang yadav committed
356
357
358
359
360
361
362
363
364
    auto else_mod_if =
        else_mod->add_instruction(migraphx::make_op("if"),
                                  {else_mod_cond,
                                   else_mod_fpga_ins,
                                   else_mod_param_0,
                                   else_mod_param_1,
                                   else_mod_param_2,
                                   else_mod_param_1,
                                   else_mod_param_0},
umang yadav's avatar
umang yadav committed
365
                                  {create_test_module(p, 0, "1_"), create_test_module(p, 1, "2_")});
366
367
368
369
370
371
372
373
374
375
376
377
378
    auto else_mod_if_0 =
        else_mod->add_instruction(migraphx::make_op("get_tuple_elem", {{"index", 0}}), else_mod_if);
    else_mod->add_return({else_mod_if_0});

    // Create nested and multi-target main module using "If"
    auto main_if_ins = mm->add_instruction(
        migraphx::make_op("if"), {cond_0, cond_1, x, y, z, cond_1, x, y, z}, {then_mod, else_mod});
    auto r = mm->add_instruction(migraphx::make_op("get_tuple_elem", {{"index", 0}}), main_if_ins);
    mm->add_return({r});

    // compile
    migraphx::compile_options gpu_opts;
    gpu_opts.offload_copy = true;
umang yadav's avatar
umang yadav committed
379
    migraphx::generate_root_modules(p, tass);
380
381
382
383
384
    p.compile({migraphx::make_target("gpu"),
               migraphx::make_target("cpu"),
               migraphx::make_target("ref"),
               migraphx::make_target("ref")},
              {gpu_opts});
385
386
387
    EXPECT(check_compiled_program(p,
                                  {migraphx::make_target("gpu"),
                                   migraphx::make_target("cpu"),
umangyadav's avatar
umangyadav committed
388
                                   migraphx::make_target("ref"),
389
                                   migraphx::make_target("ref")}));
umangyadav's avatar
umangyadav committed
390
391
    // do evaluation using different conditions
    migraphx::parameter_map params;
umangyadav's avatar
umangyadav committed
392
393
394
395
396
397
    float x_i   = 2.0;
    float y_i   = 3.0;
    float z_i   = 4.0;
    params["x"] = migraphx::fill_argument(ds, x_i);
    params["y"] = migraphx::fill_argument(ds, y_i);
    params["z"] = migraphx::fill_argument(ds, z_i);
umangyadav's avatar
umangyadav committed
398
399
400
401
402
403
404
405
406
407
408
    // cover all paths with different combination of conditions
    std::vector<std::pair<bool, bool>> test_conds = {
        {true, true}, {true, false}, {false, true}, {false, false}};
    for(auto [cond_val_0, cond_val_1] : test_conds)
    {
        params["cond_0"] = migraphx::argument(cond_s, &cond_val_0);
        params["cond_1"] = migraphx::argument(cond_s, &cond_val_1);
        auto result      = p.eval(params).back();
        // main has one instruction that is : if_then_else
        // then mod is doing : {tmp = x+y; (cond) ? (((x-1)*y)-z)  : (((tmp-1)*y)-z);}
        // else mod is doing : {tmp = x+z; (cond) ? (((tmp-1)*x)-y) : (((z-1)*y)-x);}
umangyadav's avatar
umangyadav committed
409
        float gold_i = -1.0;
umangyadav's avatar
umangyadav committed
410
411
        if(cond_val_0)
        {
umangyadav's avatar
umangyadav committed
412
413
            float tmp_i = x_i + y_i;
            gold_i      = (cond_val_1) ? (((x_i - 1) * y_i) - z_i) : (((tmp_i - 1) * y_i) - z_i);
umangyadav's avatar
umangyadav committed
414
415
416
        }
        else
        {
umangyadav's avatar
umangyadav committed
417
418
            float tmp_i = x_i + z_i;
            gold_i      = (cond_val_1) ? (((tmp_i - 1) * x_i) - y_i) : (((z_i - 1) * y_i) - x_i);
umangyadav's avatar
umangyadav committed
419
        }
umangyadav's avatar
umangyadav committed
420
421
        auto gold = migraphx::fill_argument(ds, gold_i);
        EXPECT(gold == result);
umangyadav's avatar
umangyadav committed
422
    }
423
424
}

umangyadav's avatar
umangyadav committed
425
// TODO : FPGA compilation is broken right now, below test mentions fpga but doesn't compile for it
426
427
428
TEST_CASE(multitarget_select_module)
{
    migraphx::program p;
umang yadav's avatar
umang yadav committed
429
    migraphx::target_assignments tass;
430
431
432
433
434
    // create batch submodules
    auto create_submodule = [&](std::size_t batch_size, const std::string& module_name) {
        auto* submod = p.create_module(module_name);
        migraphx::shape sm_shape{migraphx::shape::float_type, {batch_size, 4}};
        auto sm_input = submod->add_parameter("data", sm_shape);
umangyadav's avatar
umangyadav committed
435
436
        migraphx::shape lit_s{migraphx::shape{migraphx::shape::float_type, {1}}};
        auto literal_ins = submod->add_literal(migraphx::literal{lit_s, {6}});
437
438
439
440
        auto broadcast_lit =
            submod->add_instruction(migraphx::make_op("multibroadcast"), literal_ins, sm_input);
        auto add_ins0 = submod->add_instruction(migraphx::make_op("add"), sm_input, broadcast_lit);
        auto add_ins1 = submod->add_instruction(migraphx::make_op("add"), add_ins0, broadcast_lit);
umang yadav's avatar
umang yadav committed
441
442
443
        tass.insert(tass.begin(), std::make_pair(broadcast_lit, batch_size - 1));
        tass.insert(tass.begin(), std::make_pair(add_ins0, batch_size - 1));
        tass.insert(tass.begin(), std::make_pair(add_ins1, batch_size - 1));
umangyadav's avatar
umangyadav committed
444
        submod->add_return({add_ins1});
445
446
447
448
449
450
451
        return submod;
    };
    auto* batch1 = create_submodule(1, "batch_1");
    auto* batch2 = create_submodule(2, "batch_2");
    auto* batch3 = create_submodule(3, "batch_3");
    auto* batch4 = create_submodule(4, "batch_4");

umangyadav's avatar
umangyadav committed
452
453
454
    auto* mm = p.get_main_module();
    migraphx::shape dyn_s{migraphx::shape::float_type, {{1, 4}, {4, 4}}};
    auto input                              = mm->add_parameter("data", dyn_s);
455
456
457
458
459
460
461
    std::vector<migraphx::shape> sub_shapes = {};
    sub_shapes.push_back(migraphx::shape{migraphx::shape::float_type, {{1, 4}, {4, 4}}});
    sub_shapes.push_back(migraphx::shape{migraphx::shape::float_type, {{1, 4}, {4, 4}}});
    migraphx::shape out_attr = migraphx::shape{sub_shapes};
    auto sm_ins              = mm->add_instruction(
        migraphx::make_op("select_module", {{"output_dyn_shapes", migraphx::to_value(out_attr)}}),
        {input},
umang yadav's avatar
umang yadav committed
462
        {batch1, batch2, batch3, batch4});
463
    auto ret0 = mm->add_instruction(migraphx::make_op("get_tuple_elem", {{"index", 0}}), sm_ins);
umangyadav's avatar
umangyadav committed
464
    mm->add_return({ret0});
465
    // compile
umangyadav's avatar
umangyadav committed
466
467
    migraphx::compile_options gpu_opts;
    gpu_opts.offload_copy = true;
umang yadav's avatar
umang yadav committed
468
    migraphx::generate_root_modules(p, tass);
469
470
    p.compile({migraphx::make_target("gpu"),
               migraphx::make_target("cpu"),
umangyadav's avatar
umangyadav committed
471
472
473
               migraphx::make_target("ref"),
               migraphx::make_target("ref")},
              {gpu_opts});
474
475
476
    EXPECT(check_compiled_program(p,
                                  {migraphx::make_target("gpu"),
                                   migraphx::make_target("cpu"),
umangyadav's avatar
umangyadav committed
477
                                   migraphx::make_target("ref"),
478
                                   migraphx::make_target("ref")}));
umangyadav's avatar
umangyadav committed
479
480
481
482
483
    // program does the 12+x where x has dynamic shape {{1, 4}, {4, 4}}
    for(const size_t bs : {1, 2, 3, 4})
    {
        migraphx::shape arg_shape{migraphx::shape::float_type, {bs, 4}};
        migraphx::parameter_map params;
umangyadav's avatar
umangyadav committed
484
485
486
487
488
489
490
491
        params["data"] = migraphx::generate_argument(arg_shape, arg_shape.elements());
        std::vector<float> input_data;
        params["data"].visit([&](const auto& vec) { input_data.assign(vec.begin(), vec.end()); });
        std::transform(input_data.begin(), input_data.end(), input_data.begin(), [](const auto& i) {
            return i + 12.0;
        });
        auto result = p.eval(params).back();
        EXPECT(migraphx::argument(arg_shape, input_data.data()) == result);
umangyadav's avatar
umangyadav committed
492
    }
493
494
495
}

int main(int argc, const char* argv[]) { test::run(argc, argv); }