program.cpp 30.9 KB
Newer Older
Paul's avatar
Paul committed
1
2
3
#include <migraphx/program.hpp>
#include <migraphx/stringutils.hpp>
#include <migraphx/instruction.hpp>
4
#include <migraphx/op/identity.hpp>
Paul's avatar
Paul committed
5
#include <migraphx/target.hpp>
Paul's avatar
Paul committed
6
7
8
#include <migraphx/env.hpp>
#include <migraphx/ranges.hpp>
#include <migraphx/time.hpp>
9
#include <migraphx/pass_manager.hpp>
10
#include <migraphx/register_target.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
11
#include <migraphx/iterator_for.hpp>
12
#include <migraphx/iterator.hpp>
13
#include <migraphx/algorithm.hpp>
14
#include <migraphx/output_iterator.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
15
#include <migraphx/make_op.hpp>
16
#include <migraphx/marker.hpp>
Paul's avatar
Paul committed
17
#include <iostream>
Paul's avatar
Paul committed
18
#include <sstream>
Paul's avatar
Paul committed
19
#include <algorithm>
20
#include <set>
Paul's avatar
Paul committed
21
#include <utility>
22

23
#include <unordered_set>
Shucai Xiao's avatar
Shucai Xiao committed
24
25
#include <map>
#include <cassert>
Paul's avatar
Paul committed
26

Paul's avatar
Paul committed
27
namespace migraphx {
Paul's avatar
Paul committed
28
inline namespace MIGRAPHX_INLINE_NS {
Paul's avatar
Paul committed
29

30
31
using milliseconds = std::chrono::duration<double, std::milli>;

Paul's avatar
Paul committed
32
33
struct program_impl
{
Shucai Xiao's avatar
Shucai Xiao committed
34
    // A map is used to keep references to modules of the program
35
    std::unordered_map<std::string, module> modules;
Paul's avatar
Paul committed
36
    context ctx;
37
    std::string target_name;
Paul's avatar
Paul committed
38
39
};

40
program::program() : impl(std::make_unique<program_impl>()) { this->create_module("main"); }
Paul's avatar
Paul committed
41

Paul's avatar
Paul committed
42
program::program(program&&) noexcept = default;
Shucai Xiao's avatar
Shucai Xiao committed
43
program::~program() noexcept         = default;
Paul's avatar
Paul committed
44

45
// copy constructor
Shucai Xiao's avatar
Shucai Xiao committed
46
program::program(const program& p) { assign(p); }
47
48

// copy assignment operator
Shucai Xiao's avatar
Shucai Xiao committed
49
program& program::operator=(program p)
50
{
Shucai Xiao's avatar
Shucai Xiao committed
51
    std::swap(p.impl, this->impl);
52
53
54
    return *this;
}

Shucai Xiao's avatar
Shucai Xiao committed
55
void program::assign(const program& p)
56
{
Shucai Xiao's avatar
Shucai Xiao committed
57
    if(!impl)
58
59
60
    {
        impl = std::make_unique<program_impl>();
    }
Shucai Xiao's avatar
Shucai Xiao committed
61
    else if(!impl->modules.empty())
62
    {
Shucai Xiao's avatar
Shucai Xiao committed
63
        impl->modules.clear();
64
    }
Shucai Xiao's avatar
Shucai Xiao committed
65

66
    impl->ctx         = p.impl->ctx;
Shucai Xiao's avatar
Shucai Xiao committed
67
68
    impl->target_name = p.impl->target_name;
    impl->modules     = p.impl->modules;
Shucai Xiao's avatar
Shucai Xiao committed
69
70
71
72

    // build a map from old ins to new ins
    // Build a map from old module to new module
    std::unordered_map<module_ref, module_ref> mod_map;
Paul's avatar
Paul committed
73
74
75
76
77
    std::transform(
        impl->modules.begin(),
        impl->modules.end(),
        std::inserter(mod_map, mod_map.begin()),
        [&](auto&& xp) { return std::make_pair(&p.impl->modules.at(xp.first), &xp.second); });
Shucai Xiao's avatar
Shucai Xiao committed
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

    std::unordered_map<instruction_ref, instruction_ref> ins_map;
    for(auto&& pp : mod_map)
    {
        auto old_ins = iterator_for(*pp.first);
        auto new_ins = iterator_for(*pp.second);
        std::transform(old_ins.begin(),
                       old_ins.end(),
                       new_ins.begin(),
                       std::inserter(ins_map, ins_map.begin()),
                       [](auto x, auto y) { return std::make_pair(x, y); });
    }

    // Update all references from all modules
    for(auto&& mp : impl->modules)
    {
94
        for(auto ins : iterator_for(mp.second))
Shucai Xiao's avatar
Shucai Xiao committed
95
96
            instruction::replace_refs(ins, ins_map, mod_map);
    }
97
98
}

Paul's avatar
Paul committed
99
shape program::get_parameter_shape(std::string name) const
Paul's avatar
Paul committed
100
{
Shucai Xiao's avatar
Shucai Xiao committed
101
102
    const auto* mm = this->get_main_module();
    return mm->get_parameter_shape(std::move(name));
Paul's avatar
Paul committed
103
104
}

105
106
std::vector<std::string> program::get_parameter_names() const
{
Shucai Xiao's avatar
Shucai Xiao committed
107
108
    const auto* mm = this->get_main_module();
    return mm->get_parameter_names();
109
110
}

mei-ye's avatar
mei-ye committed
111
112
instruction_ref program::get_parameter(std::string name) const
{
Shucai Xiao's avatar
Shucai Xiao committed
113
114
    const auto* mm = this->get_main_module();
    return mm->get_parameter(std::move(name));
mei-ye's avatar
mei-ye committed
115
116
}

Paul's avatar
Paul committed
117
118
std::unordered_map<std::string, shape> program::get_parameter_shapes() const
{
Shucai Xiao's avatar
Shucai Xiao committed
119
120
    const auto* mm = this->get_main_module();
    return mm->get_parameter_shapes();
Paul's avatar
Paul committed
121
122
}

Shucai Xiao's avatar
Shucai Xiao committed
123
std::size_t program::size() const { return impl->modules.size(); }
124

125
126
std::vector<shape> program::get_output_shapes() const
{
Shucai Xiao's avatar
Shucai Xiao committed
127
128
    const auto* mm = this->get_main_module();
    return mm->get_output_shapes();
129
}
Paul's avatar
Paul committed
130

Paul's avatar
Paul committed
131
132
context& program::get_context() const { return impl->ctx; }

Paul's avatar
Paul committed
133
134
instruction_ref program::validate() const
{
Shucai Xiao's avatar
Shucai Xiao committed
135
136
    const auto* mm = this->get_main_module();
    return mm->validate();
Paul's avatar
Paul committed
137
138
}

139
140
bool program::is_compiled() const { return not this->impl->target_name.empty(); }

umangyadav's avatar
umangyadav committed
141
void program::compile(const target& t, compile_options options, const std::string& ir_dump_path)
Paul's avatar
Paul committed
142
{
143
144
145
    assert(not this->is_compiled());
    this->impl->target_name = t.name();
    this->impl->ctx         = t.get_context();
146
    if(enabled(MIGRAPHX_DUMP_PASSES_TO_FILE{}))
umangyadav's avatar
umangyadav committed
147
        options.trace = tracer{t.name() + "_" + ir_dump_path};
148
149
    else if(enabled(MIGRAPHX_TRACE_COMPILE{})) 
        options.trace = tracer{std::cout};
Shucai Xiao's avatar
Shucai Xiao committed
150

151
    options.trace("input_program", *this);
Shucai Xiao's avatar
Shucai Xiao committed
152

Shucai Xiao's avatar
Shucai Xiao committed
153
    auto&& passes = t.get_passes(this->impl->ctx, options);
154
155
156
    run_passes(*this, passes, options.trace);

    auto mods = this->get_modules();
Shucai Xiao's avatar
Shucai Xiao committed
157

158
159
    // Validate and finalize
    for(const auto& mod : reverse(mods))
Paul's avatar
Paul committed
160
    {
Shucai Xiao's avatar
Shucai Xiao committed
161
162
163
164
165
166
        auto invalid = mod->validate();
        if(invalid != mod->end())
        {
            MIGRAPHX_THROW("Invalid module " + mod->name() + " from compilation at instruction " +
                           std::to_string(std::distance(mod->begin(), invalid)));
        }
167
168
169
170
171
172
173
        auto dangling = mod->find_dangling_reference();
        if(dangling != mod->end())
        {
            auto index = std::distance(mod->begin(), dangling);
            MIGRAPHX_THROW("Dangling reference in module " + mod->name() + " from instruction " +
                           std::to_string(index));
        }
Shucai Xiao's avatar
Shucai Xiao committed
174
        mod->finalize(this->impl->ctx);
Paul's avatar
Paul committed
175
    }
Paul's avatar
Paul committed
176
177
178
179
}

void program::finalize()
{
Shucai Xiao's avatar
Shucai Xiao committed
180
181
    auto* mm = this->get_main_module();
    mm->finalize(this->impl->ctx);
Paul's avatar
Paul committed
182
183
}

184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
template <class T>
std::string classify(T x)
{
    switch(std::fpclassify(x))
    {
    case FP_INFINITE: return "inf";
    case FP_NAN: return "nan";
    case FP_NORMAL: return "normal";
    case FP_SUBNORMAL: return "subnormal";
    case FP_ZERO: return "zero";
    default: return "unknown";
    }
}

std::unordered_set<std::string> classify_argument(const argument& a)
{
    std::unordered_set<std::string> result;
    a.visit(
        [&](auto t) {
            for(const auto& x : t)
                result.insert(classify(x));
        },
        [&](const auto& xs) {
            for(const auto& x : xs)
            {
                auto r = classify_argument(x);
                result.insert(r.begin(), r.end());
            }
        });
    return result;
}

void preview_argument(std::ostream& os, const argument& a)
{
    a.visit(
        [&](auto t) {
            if(t.size() <= 10)
            {
                os << t;
            }
            else
            {
                os << to_string_range(t.begin(), t.begin() + 5);
                os << ", ..., ";
                os << to_string_range(t.end() - 5, t.end());
            }
        },
        [&](const auto& xs) {
            for(const auto& x : xs)
            {
                os << '{';
                preview_argument(os, x);
                os << '}';
            }
        });
}

Paul's avatar
Paul committed
241
template <class F>
Shucai Xiao's avatar
Shucai Xiao committed
242
std::vector<argument> generic_eval(const module* mod,
243
244
                                   context& ctx,
                                   std::unordered_map<std::string, argument> params,
Shucai Xiao's avatar
Shucai Xiao committed
245
                                   std::unordered_map<instruction_ref, argument> results,
246
                                   F make_trace)
Paul's avatar
Paul committed
247
{
Shucai Xiao's avatar
Shucai Xiao committed
248
249
    assert(mod->validate() == mod->end());
    results.reserve(mod->size() * 2);
Paul's avatar
Paul committed
250
251
    std::vector<argument> values;
    values.reserve(16);
252
    auto trace = make_trace(mod);
Shucai Xiao's avatar
Shucai Xiao committed
253
    for(auto ins : iterator_for(*mod))
Paul's avatar
Paul committed
254
    {
255
        assert(results.find(ins) == results.end());
256
257
        const auto& name = ins->name();
        if(name == "@literal")
Paul's avatar
Paul committed
258
        {
Paul's avatar
Paul committed
259
            results.emplace(ins, trace(ins, [&] { return ins->get_literal().get_argument(); }));
Paul's avatar
Paul committed
260
        }
261
        else if(name == "@param")
Paul's avatar
Paul committed
262
        {
Paul's avatar
Paul committed
263
264
265
266
267
            results.emplace(
                ins, trace(ins, [&] {
                    auto param_name = any_cast<builtin::param>(ins->get_operator()).parameter;
                    if(not contains(params, param_name))
                        MIGRAPHX_THROW("Parameter not found: " + param_name);
268
                    auto param = params[param_name];
Paul's avatar
Paul committed
269
270
271
272
273
                    if(param.get_shape() != ins->get_shape())
                        MIGRAPHX_THROW("Incorrect shape {" + to_string(param.get_shape()) +
                                       "} for parameter: " + param_name);
                    return param;
                }));
Paul's avatar
Paul committed
274
        }
275
        else if(name == "@outline")
Paul's avatar
Paul committed
276
        {
Paul's avatar
Paul committed
277
            results.emplace(ins, trace(ins, [&] { return argument{ins->get_shape(), nullptr}; }));
Paul's avatar
Paul committed
278
        }
279
280
281
282
283
284
285
286
287
288
289
290
291
        else if(name == "@return")
        {
            std::vector<argument> prog_outputs;
            std::transform(ins->inputs().begin(),
                           ins->inputs().end(),
                           std::back_inserter(prog_outputs),
                           [&](instruction_ref i) {
                               assert(results.find(i) != results.end());
                               return results[i];
                           });

            return prog_outputs;
        }
Paul's avatar
Paul committed
292
293
        else
        {
Paul's avatar
Paul committed
294
            values.resize(ins->inputs().size());
Paul's avatar
Paul committed
295
296
297
298
299
            std::transform(
                ins->inputs().begin(), ins->inputs().end(), values.begin(), [&](instruction_ref i) {
                    assert(results.find(i) != results.end());
                    return results[i];
                });
Shucai Xiao's avatar
Shucai Xiao committed
300
301
302
303

            const auto& mod_args = ins->module_inputs();
            auto module_eval     = [&](module_ref smod,
                                   const std::unordered_map<std::string, argument>& inputs) {
Shucai Xiao's avatar
Shucai Xiao committed
304
305
                auto ssctx = ctx;
                return generic_eval(smod, ssctx, inputs, results, make_trace);
Shucai Xiao's avatar
Shucai Xiao committed
306
307
            };

Shucai Xiao's avatar
Shucai Xiao committed
308
309
310
311
            results.emplace(ins, trace(ins, [&] {
                                return ins->normalized_operator().compute(
                                    ctx, ins->get_shape(), values, mod_args, module_eval);
                            }));
Paul's avatar
Paul committed
312
        }
313
        assert(results.find(ins) != results.end());
314
        assert(results.at(ins).get_shape() == ins->get_shape());
Paul's avatar
Paul committed
315
    }
Shucai Xiao's avatar
Shucai Xiao committed
316
    return {results.at(std::prev(mod->end()))};
Paul's avatar
Paul committed
317
318
}

Shucai Xiao's avatar
Shucai Xiao committed
319
320
321
322
template <class F>
std::vector<argument> generic_eval(const program& p,
                                   context& ctx,
                                   std::unordered_map<std::string, argument> params,
323
                                   F make_trace)
Shucai Xiao's avatar
Shucai Xiao committed
324
{
Shucai Xiao's avatar
Shucai Xiao committed
325
    const module* mm = p.get_main_module();
326
    return generic_eval(mm, ctx, params, {}, make_trace);
Shucai Xiao's avatar
Shucai Xiao committed
327
328
}

329
std::vector<argument> program::eval(parameter_map params) const
Paul's avatar
Paul committed
330
{
Paul's avatar
Paul committed
331
332
    auto& ctx = this->impl->ctx;
#ifndef NDEBUG
333
334
335
336
337
338
339
340
341
342
343
    auto with_check_context = [&](auto f) {
        return [=, &ctx](auto&&) {
            auto sctx          = std::make_shared<context>(ctx);
            auto check_context = [=, &ctx](auto g) {
                assert(is_shared(ctx, *sctx));
                auto x = g();
                *sctx  = ctx;
                return x;
            };
            return [=](auto&&... xs) { return f(xs..., check_context); };
        };
Paul's avatar
Paul committed
344
345
    };
#else
346
347
348
349
350
    auto with_check_context = [](auto f) {
        return [=](auto&&) {
            return [=](auto&&... xs) { return f(xs..., [](auto g) { return g(); }); };
        };
    };
Paul's avatar
Paul committed
351
#endif
Paul's avatar
Paul committed
352
353
354
355

    auto trace_level = value_of(MIGRAPHX_TRACE_EVAL{});

    if(trace_level > 0)
Paul's avatar
Paul committed
356
    {
Shucai Xiao's avatar
Shucai Xiao committed
357
358
359
360
361
362
363
364
        std::unordered_map<instruction_ref, std::string> ins_out;
        // get instruction names
        this->print([&](auto x, auto ins_names) {
            std::stringstream ss;
            instruction::print(ss, x, ins_names);
            ins_out[x] = ss.str();
        });

365
366
367
368
369
        return generic_eval(*this,
                            ctx,
                            std::move(params),
                            with_check_context([&](auto& ins, auto f, auto&& check_context) {
                                ctx.finish();
Shucai Xiao's avatar
Shucai Xiao committed
370
                                std::cout << "Run instruction: " << ins_out.at(ins) << std::endl;
371
372
373
374
375
376
377
                                timer t{};
                                auto result = check_context(f);
                                double t1   = t.record<milliseconds>();
                                ctx.finish();
                                double t2 = t.record<milliseconds>();
                                std::cout << "Time: " << t1 << "ms, " << t2 << "ms" << std::endl;
                                if(trace_level > 1 and ins->name().front() != '@' and
Shucai Xiao's avatar
Shucai Xiao committed
378
379
                                   ins->name() != "load" and not result.empty())
                                {
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
                                    target tgt  = make_target(this->impl->target_name);
                                    auto buffer = tgt.copy_from(result);
                                    if(trace_level == 2)
                                    {
                                        std::cout << "Output has "
                                                  << to_string_range(classify_argument(buffer))
                                                  << std::endl;
                                        std::cout << "Output: ";
                                        preview_argument(std::cout, buffer);
                                        std::cout << std::endl;
                                    }
                                    else
                                    {
                                        std::cout << "Output: " << buffer << std::endl;
                                    }
Shucai Xiao's avatar
Shucai Xiao committed
395
                                }
396
397
                                return result;
                            }));
Paul's avatar
Paul committed
398
399
400
    }
    else
    {
401
402
403
404
405
406
        return generic_eval(*this,
                            ctx,
                            std::move(params),
                            with_check_context([&](auto&, auto f, auto&& check_context) {
                                return check_context(f);
                            }));
Paul's avatar
Paul committed
407
    }
Paul's avatar
Paul committed
408
409
}

410
const int program_file_version = 5;
411
412
413
414
415
416
417
418

value program::to_value() const
{
    value result;
    result["version"] = program_file_version;
    result["target"]  = this->impl->target_name;
    if(not this->impl->target_name.empty())
        result["context"] = this->impl->ctx.to_value();
Shucai Xiao's avatar
Shucai Xiao committed
419

420
    value module_vals = value::object{};
Shucai Xiao's avatar
Shucai Xiao committed
421
    std::unordered_map<instruction_ref, std::string> names;
422
    for(auto& mod : this->get_modules())
Shucai Xiao's avatar
Shucai Xiao committed
423
    {
Shucai Xiao's avatar
Shucai Xiao committed
424
425
        value mod_val;
        value nodes;
426
427
        mod_val["name"] = mod->name();
        names           = mod->print(
Shucai Xiao's avatar
Shucai Xiao committed
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
            [&](auto ins, auto ins_names) {
                value node;
                node["output"]     = ins_names.at(ins);
                node["name"]       = ins->name();
                node["shape"]      = migraphx::to_value(ins->get_shape());
                node["normalized"] = ins->is_normalized();
                if(ins->name() == "@literal")
                    node["literal"] = migraphx::to_value(ins->get_literal());
                node["operator"] = ins->get_operator().to_value();
                std::vector<std::string> inputs;
                std::transform(ins->inputs().begin(),
                               ins->inputs().end(),
                               std::back_inserter(inputs),
                               [&](auto i) {
                                   assert(contains(ins_names, i));
                                   return ins_names.at(i);
                               });
                node["inputs"]   = inputs;
                auto module_args = ins->module_inputs();
                if(not module_args.empty())
                {
                    std::vector<std::string> module_inputs;
                    std::transform(module_args.begin(),
                                   module_args.end(),
                                   std::back_inserter(module_inputs),
                                   [&](auto mod_ref) { return mod_ref->name(); });
                    node["module_inputs"] = module_inputs;
                }

                nodes.push_back(node);
            },
            names);
        mod_val["nodes"] = nodes;

462
        module_vals[mod->name()] = mod_val;
Shucai Xiao's avatar
Shucai Xiao committed
463
    }
Shucai Xiao's avatar
Shucai Xiao committed
464
465
466

    result["modules"] = module_vals;

467
468
    return result;
}
Shucai Xiao's avatar
Shucai Xiao committed
469

Shucai Xiao's avatar
Shucai Xiao committed
470
471
472
473
474
static void mod_from_val(module_ref mod,
                         const value& v,
                         std::unordered_map<std::string, instruction_ref>& instructions,
                         const std::unordered_map<std::string, module_ref>& map_mods)
{
475
    const auto& module_val = v.at(mod->name());
Shucai Xiao's avatar
Shucai Xiao committed
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
    for(const value& node : module_val.at("nodes"))
    {
        instruction_ref output;
        auto name       = node.at("name").to<std::string>();
        auto fields     = node.at("operator");
        auto normalized = node.at("normalized").to<bool>();

        if(name == "@param")
        {
            output = mod->add_parameter(fields["parameter"].to<std::string>(),
                                        migraphx::from_value<shape>(node.at("shape")));
        }
        else if(name == "@literal")
        {
            output = mod->add_literal(migraphx::from_value<literal>(node.at("literal")));
        }
        else
        {
            auto op = make_op(name, fields);
            std::vector<instruction_ref> inputs;
            std::transform(node.at("inputs").begin(),
                           node.at("inputs").end(),
                           std::back_inserter(inputs),
                           [&](const value& i) {
                               auto i_name = i.to<std::string>();
                               assert(contains(instructions, i_name));
                               return instructions.at(i_name);
                           });

            std::vector<module_ref> module_inputs;
            if(node.contains("module_inputs"))
            {
                std::transform(node.at("module_inputs").begin(),
                               node.at("module_inputs").end(),
                               std::back_inserter(module_inputs),
                               [&](const value& i) { return map_mods.at(i.to<std::string>()); });

                for(auto& smod : module_inputs)
                {
                    mod_from_val(smod, v, instructions, map_mods);
                }
            }

            if(name == "@return")
            {
                output = mod->add_return(inputs);
            }
            else if(module_inputs.empty())
            {
                output = mod->add_instruction(op, inputs);
            }
            else
            {
                output = mod->add_instruction(op, inputs, module_inputs);
            }
        }
        output->set_normalized(normalized);
        instructions[node.at("output").to<std::string>()] = output;
    }
}

537
538
539
540
void program::from_value(const value& v)
{
    auto version = v.at("version").to<int>();
    if(version != program_file_version)
Shucai Xiao's avatar
Shucai Xiao committed
541
542
543
544
    {
        MIGRAPHX_THROW("Warning: Program version mismatch");
    }

545
546
547
548
549
550
551
552
    this->impl->target_name = v.at("target").to<std::string>();
    if(not this->impl->target_name.empty())
    {
        target t        = make_target(this->impl->target_name);
        this->impl->ctx = t.get_context();
        this->impl->ctx.from_value(v.at("context"));
    }

Shucai Xiao's avatar
Shucai Xiao committed
553
554
    auto module_vals = v.at("modules");
    for(const auto& vv : module_vals)
555
    {
556
        const auto& name = vv.get_key();
Shucai Xiao's avatar
Shucai Xiao committed
557
558
        if(name == "main")
            continue;
559
        impl->modules.emplace(name, name);
560
    }
561
    std::unordered_map<std::string, module_ref> map_mods;
Paul's avatar
Paul committed
562
563
564
565
    std::transform(impl->modules.begin(),
                   impl->modules.end(),
                   std::inserter(map_mods, map_mods.end()),
                   [&](auto&& pp) { return std::make_pair(pp.first, &pp.second); });
Shucai Xiao's avatar
Shucai Xiao committed
566
567
568
569
570

    std::unordered_map<std::string, instruction_ref> map_insts;
    auto* mm = get_main_module();
    mod_from_val(mm, module_vals, map_insts, map_mods);

571
572
573
    this->finalize();
}

Paul's avatar
Paul committed
574
575
576
double common_average(const std::vector<double>& v)
{
    std::size_t n = v.size() / 4;
Paul's avatar
Paul committed
577
578
    double total  = std::accumulate(v.begin() + n, v.end() - n, 0.0);
    return total / std::distance(v.begin() + n, v.end() - n);
Paul's avatar
Paul committed
579
580
}

Paul Fultz II's avatar
Paul Fultz II committed
581
582
583
584
585
586
587
588
std::string perf_group(const operation& op)
{
    auto attr = op.attributes();
    if(attr.contains("group"))
        return attr.at("group").to<std::string>();
    return op.name();
}

589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
void program::mark(const parameter_map& params, marker&& m)
{
    auto& ctx = this->impl->ctx;
    // Run once by itself
    eval(params);
    ctx.finish();
    // Start marking
    m.mark_start(*this);
    generic_eval(*this, ctx, params, always([&](auto ins, auto f) {
        argument result;
        m.mark_start(ins);
        result = f();
        m.mark_stop(ins);
        return result;
    }));
    m.mark_stop(*this);
}

607
608
609
610
void program::perf_report(std::ostream& os,
                          std::size_t n,
                          parameter_map params,
                          std::size_t batch) const
Paul's avatar
Paul committed
611
{
612
    auto& ctx = this->impl->ctx;
Paul's avatar
Paul committed
613
614
    // Run once by itself
    eval(params);
Paul's avatar
Paul committed
615
    ctx.finish();
Paul's avatar
Paul committed
616
    // Run and time entire program
Paul's avatar
Paul committed
617
618
    std::vector<double> total_vec;
    total_vec.reserve(n);
Paul's avatar
Paul committed
619
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
620
    {
Paul's avatar
Paul committed
621
622
623
624
        total_vec.push_back(time<milliseconds>([&] {
            eval(params);
            ctx.finish();
        }));
Paul's avatar
Paul committed
625
    }
Paul's avatar
Paul committed
626
627
    std::sort(total_vec.begin(), total_vec.end());
    std::unordered_map<instruction_ref, std::vector<double>> ins_vec;
Paul's avatar
Paul committed
628
    // Fill the map
629
    generic_eval(*this, ctx, params, always([&](auto ins, auto) {
Paul's avatar
Paul committed
630
        ins_vec[ins].reserve(n);
631
        return argument{ins->get_shape(), nullptr};
632
    }));
633

Paul's avatar
Paul committed
634
    // Run and time each instruction
Paul's avatar
Paul committed
635
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
636
    {
637
        generic_eval(*this, ctx, params, always([&](auto ins, auto f) {
638
            argument result;
Paul's avatar
Paul committed
639
640
641
642
            ins_vec[ins].push_back(time<milliseconds>([&] {
                result = f();
                ctx.finish();
            }));
643
            return result;
644
        }));
Paul's avatar
Paul committed
645
    }
Paul's avatar
Paul committed
646
647
    for(auto&& p : ins_vec)
        std::sort(p.second.begin(), p.second.end());
Paul's avatar
Paul committed
648
    // Run and time implicit overhead
Paul's avatar
Paul committed
649
650
    std::vector<double> overhead_vec;
    overhead_vec.reserve(n);
Paul's avatar
Paul committed
651
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
652
    {
Paul's avatar
Paul committed
653
        overhead_vec.push_back(time<milliseconds>([&] { dry_run(params); }));
Paul's avatar
Paul committed
654
655
    }

Paul's avatar
Paul committed
656
    double total_time             = common_average(total_vec);
Paul's avatar
Paul committed
657
    double rate                   = 1000.0 / total_time;
Paul's avatar
Paul committed
658
    double overhead_time          = common_average(overhead_vec);
Paul's avatar
Paul committed
659
    double overhead_percent       = overhead_time * 100.0 / total_time;
Paul's avatar
Paul committed
660
    double total_instruction_time = 0.0;
Paul's avatar
Paul committed
661
    std::unordered_map<std::string, double> op_times;
Paul's avatar
Paul committed
662
    for(auto&& p : ins_vec)
Paul's avatar
Paul committed
663
664
    {
        double avg = common_average(p.second);
Paul Fultz II's avatar
Paul Fultz II committed
665
        op_times[perf_group(p.first->get_operator())] += avg;
Paul's avatar
Paul committed
666
667
        total_instruction_time += avg;
    }
Paul's avatar
Paul committed
668
669
    double calculate_overhead_time    = total_time - total_instruction_time;
    double calculate_overhead_percent = calculate_overhead_time * 100.0 / total_time;
Paul's avatar
Paul committed
670

Shucai Xiao's avatar
Shucai Xiao committed
671
672
673
    std::unordered_map<instruction_ref, std::string> names;
    this->print(names, [&](auto ins, auto ins_names) {
        instruction::print(std::cout, ins, ins_names);
674
675
676
677
678

        // skip return instruction
        if(ins->name() == "@return")
            return;

Paul's avatar
Paul committed
679
680
681
        double avg     = common_average(ins_vec[ins]);
        double percent = std::ceil(100.0 * avg / total_instruction_time);
        os << ": " << avg << "ms, " << percent << "%";
682
        os << std::endl;
Paul's avatar
Paul committed
683
    });
Paul's avatar
Paul committed
684
685
686

    os << std::endl;
    os << "Summary:" << std::endl;
687
688
689
690
691
692
693
    std::vector<std::pair<double, std::string>> op_times_sorted;
    std::transform(op_times.begin(),
                   op_times.end(),
                   std::back_inserter(op_times_sorted),
                   [](auto p) { return std::make_pair(p.second, p.first); });
    std::sort(op_times_sorted.begin(), op_times_sorted.end(), std::greater<>{});
    for(auto&& p : op_times_sorted)
Paul's avatar
Paul committed
694
    {
695
696
        auto&& name    = p.second;
        double avg     = p.first;
Paul's avatar
Paul committed
697
698
699
700
701
        double percent = std::ceil(100.0 * avg / total_instruction_time);
        os << name << ": " << avg << "ms, " << percent << "%" << std::endl;
    }

    os << std::endl;
Paul's avatar
Paul committed
702

703
704
    os << "Batch size: " << batch << std::endl;
    os << "Rate: " << rate * batch << "/sec" << std::endl;
Paul's avatar
Paul committed
705
706
    os << "Total time: " << total_time << "ms" << std::endl;
    os << "Total instructions time: " << total_instruction_time << "ms" << std::endl;
Paul's avatar
Paul committed
707
708
709
710
    os << "Overhead time: " << overhead_time << "ms"
       << ", " << calculate_overhead_time << "ms" << std::endl;
    os << "Overhead: " << std::round(overhead_percent) << "%"
       << ", " << std::round(calculate_overhead_percent) << "%" << std::endl;
Paul's avatar
Paul committed
711
712
}

Paul's avatar
Paul committed
713
714
void program::debug_print() const { std::cout << *this << std::endl; }
void program::debug_print(instruction_ref ins) const
Paul's avatar
Paul committed
715
{
Shucai Xiao's avatar
Shucai Xiao committed
716
    std::unordered_map<instruction_ref, std::string> names;
717
    if(std::any_of(this->impl->modules.begin(), this->impl->modules.end(), [&](const auto& pp) {
718
           return is_end(pp.second.end(), ins);
Shucai Xiao's avatar
Shucai Xiao committed
719
       }))
Paul's avatar
Paul committed
720
721
722
723
    {
        std::cout << "End instruction" << std::endl;
        return;
    }
724
725
    else if(std::none_of(this->impl->modules.begin(),
                         this->impl->modules.end(),
726
                         [&](const auto& pp) { return pp.second.has_instruction(ins); }))
Paul's avatar
Paul committed
727
728
729
730
    {
        std::cout << "Instruction not part of program" << std::endl;
        return;
    }
Shucai Xiao's avatar
Shucai Xiao committed
731

Paul's avatar
Paul committed
732
    std::stringstream ss;
Shucai Xiao's avatar
Shucai Xiao committed
733
    this->print(names, [&](auto x, auto ins_names) {
Paul's avatar
Paul committed
734
        if(x == ins)
Paul's avatar
Paul committed
735
        {
Shucai Xiao's avatar
Shucai Xiao committed
736
            instruction::print(std::cout, x, ins_names);
Paul's avatar
Paul committed
737
738
739
740
741
            std::cout << std::endl;
        }
    });
}

Shucai Xiao's avatar
Shucai Xiao committed
742
743
744
745
void program::print(
    std::unordered_map<instruction_ref, std::string>& names,
    const std::function<void(instruction_ref, std::unordered_map<instruction_ref, std::string>)>&
        print_func) const
746
{
747
    for(const auto& pp : this->impl->modules)
748
    {
749
        names = pp.second.print(print_func, names);
750
751
752
    }
}

Shucai Xiao's avatar
Shucai Xiao committed
753
754
755
756
757
758
759
760
void program::print(
    const std::function<void(instruction_ref ins,
                             std::unordered_map<instruction_ref, std::string>)>& print_func) const
{
    std::unordered_map<instruction_ref, std::string> names;
    this->print(names, print_func);
}

Shucai Xiao's avatar
Shucai Xiao committed
761
void program::print_graph(std::ostream& os, bool brief) const
762
{
Shucai Xiao's avatar
Shucai Xiao committed
763
764
    const auto* mm = this->get_main_module();
    mm->print_graph(os, brief);
765
766
767
768
}

void program::print_cpp(std::ostream& os) const
{
Shucai Xiao's avatar
Shucai Xiao committed
769
770
771
772
773
774
775
776
    auto vec_modules = this->get_modules();
    std::unordered_map<instruction_ref, std::string> names;
    for(auto& mod : vec_modules)
    {
        os << "module: \"" << mod->name() << "\"" << std::endl;
        names = mod->print_cpp(os, names);
        os << std::endl;
    }
777
778
}

Paul's avatar
Paul committed
779
780
void program::dry_run(std::unordered_map<std::string, argument> params) const
{
Paul's avatar
Paul committed
781
    auto& ctx = this->impl->ctx;
782
783
784
    generic_eval(*this, ctx, std::move(params), always([](auto ins, auto&&...) {
        return argument{ins->get_shape(), nullptr};
    }));
Paul's avatar
Paul committed
785
786
}

Shucai Xiao's avatar
Shucai Xiao committed
787
void program::annotate(std::ostream& os, const std::function<void(instruction_ref)>& a) const
Paul's avatar
Paul committed
788
{
789
    for(auto& pp : this->impl->modules)
Shucai Xiao's avatar
Shucai Xiao committed
790
    {
791
792
        std::cout << pp.first << ":" << std::endl;
        pp.second.annotate(os, a);
Shucai Xiao's avatar
Shucai Xiao committed
793
    }
Paul's avatar
Paul committed
794
795
}

Paul's avatar
Paul committed
796
const module* program::get_module(const std::string& name) const { return &impl->modules.at(name); }
Shucai Xiao's avatar
Shucai Xiao committed
797
798
799

module* program::create_module(const std::string& name)
{
800
    assert(not contains(impl->modules, name));
801
802
    auto r = impl->modules.emplace(name, name);
    return &(r.first->second);
Shucai Xiao's avatar
Shucai Xiao committed
803
804
}

Paul's avatar
Paul committed
805
module* program::get_module(const std::string& name) { return &impl->modules.at(name); }
Shucai Xiao's avatar
Shucai Xiao committed
806
807
808
809
810

module* program::get_main_module() { return get_module("main"); }

const module* program::get_main_module() const { return get_module("main"); }

Paul's avatar
Paul committed
811
template <class T>
812
std::vector<T*> generic_get_modules(T* mm)
Shucai Xiao's avatar
Shucai Xiao committed
813
{
814
    std::vector<T*> vec_modules;
Shucai Xiao's avatar
Shucai Xiao committed
815
816
817
818
819
    vec_modules.push_back(mm);
    auto sub_modules = mm->get_sub_modules();
    vec_modules.insert(vec_modules.end(), sub_modules.begin(), sub_modules.end());
    return vec_modules;
}
Shucai Xiao's avatar
Shucai Xiao committed
820

Paul's avatar
Paul committed
821
template <class Map, class T, class OutputIterator>
822
void generic_get_unused_modules(Map& m, const std::vector<T*>& mods, OutputIterator out)
Shucai Xiao's avatar
Shucai Xiao committed
823
{
824
825
826
827
    std::unordered_set<std::string> used;
    std::transform(mods.begin(), mods.end(), std::inserter(used, used.end()), [](auto&& mod) {
        return mod->name();
    });
bpickrel's avatar
bpickrel committed
828
829
830
831
832
833
    transform_if(
        m.begin(),
        m.end(),
        out,
        [&](auto&& pp) { return not contains(used, pp.first); },
        [](auto&& pp) { return &pp.second; });
834
}
Shucai Xiao's avatar
Shucai Xiao committed
835

836
837
838
839
840
841
842
843
844
845
846
847
std::vector<const module*> program::get_modules() const
{
    auto result = generic_get_modules(this->get_main_module());
    generic_get_unused_modules(impl->modules, result, std::back_inserter(result));
    return result;
}

std::vector<module*> program::get_modules()
{
    auto result = generic_get_modules(this->get_main_module());
    generic_get_unused_modules(impl->modules, result, std::back_inserter(result));
    return result;
Shucai Xiao's avatar
Shucai Xiao committed
848
849
}

850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
template <class Map, class T>
bool is_unused_module(Map& m, const std::vector<T*>& mods, const std::string& name)
{
    bool is_unused = false;
    generic_get_unused_modules(m, mods, make_function_output_iterator([&](auto* mod) {
                                   if(mod->name() == name)
                                       is_unused = true;
                               }));
    return is_unused;
}

template <class Map>
bool references_instruction(Map& m, const instruction& ins, const std::string& name)
{
    return std::any_of(m.begin(), m.end(), [&](auto&& p) {
        if(p.first == name)
            return false;
        return std::any_of(p.second.begin(), p.second.end(), [&](auto&& i) {
            return std::any_of(i.inputs().begin(), i.inputs().end(), [&](auto&& j) {
                return std::addressof(*j) == std::addressof(ins);
            });
        });
    });
}

void program::remove_module(const std::string& name)
{
    // cppcheck-suppress assertWithSideEffect
    assert(is_unused_module(impl->modules, generic_get_modules(this->get_main_module()), name) &&
           "Module used in program");
    assert(std::none_of(
               impl->modules.at(name).begin(),
               impl->modules.at(name).end(),
               [&](auto&& ins) { return references_instruction(impl->modules, ins, name); }) &&
           "Instruction referenced in another module");
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900

    // if an instruction has an input out side of the current module, need to remove
    // the instruction from its input's outputs
    auto& mod = impl->modules.at(name);
    for(auto ins : iterator_for(mod))
    {
        auto inputs = ins->inputs();
        for(auto in : inputs)
        {
            if(not mod.has_instruction(in))
            {
                in->remove_output(ins);
            }
        }
    }

901
902
903
904
905
906
907
908
909
910
911
912
    impl->modules.erase(name);
}

void program::remove_unused_modules()
{
    std::vector<module*> unused;
    generic_get_unused_modules(
        impl->modules, generic_get_modules(this->get_main_module()), std::back_inserter(unused));
    for(auto* m : unused)
        this->remove_module(m->name());
}

913
914
program& program::sort()
{
915
    for(auto& pp : this->impl->modules)
Shucai Xiao's avatar
Shucai Xiao committed
916
    {
917
        pp.second.sort();
Shucai Xiao's avatar
Shucai Xiao committed
918
919
    }

920
921
922
    return *this;
}

Paul's avatar
Paul committed
923
bool operator==(const program& x, const program& y) { return to_string(x) == to_string(y); }
Paul's avatar
Paul committed
924

Paul's avatar
Paul committed
925
std::ostream& operator<<(std::ostream& os, const program& p)
Paul's avatar
Paul committed
926
{
Shucai Xiao's avatar
Shucai Xiao committed
927
928
929
    auto vec_modules = p.get_modules();
    std::unordered_map<instruction_ref, std::string> names;
    for(auto& mod : vec_modules)
Shucai Xiao's avatar
Shucai Xiao committed
930
    {
Shucai Xiao's avatar
Shucai Xiao committed
931
932
933
934
935
936
937
        os << "module: \"" << mod->name() << "\"" << std::endl;
        names = mod->print(
            [&](auto ins, auto ins_names) {
                instruction::print(os, ins, ins_names);
                os << std::endl;
            },
            names);
938
        os << std::endl;
Shucai Xiao's avatar
Shucai Xiao committed
939
940
    }

Paul's avatar
Paul committed
941
    return os;
Paul's avatar
Paul committed
942
}
Paul's avatar
Paul committed
943

Paul's avatar
Paul committed
944
} // namespace MIGRAPHX_INLINE_NS
Paul's avatar
Paul committed
945
} // namespace migraphx