program.cpp 31.1 KB
Newer Older
1
#include <functional>
Paul's avatar
Paul committed
2
3
4
#include <migraphx/program.hpp>
#include <migraphx/stringutils.hpp>
#include <migraphx/instruction.hpp>
5
#include <migraphx/op/identity.hpp>
Paul's avatar
Paul committed
6
#include <migraphx/target.hpp>
Paul's avatar
Paul committed
7
8
9
#include <migraphx/env.hpp>
#include <migraphx/ranges.hpp>
#include <migraphx/time.hpp>
10
#include <migraphx/pass_manager.hpp>
11
#include <migraphx/register_target.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
12
#include <migraphx/iterator_for.hpp>
13
#include <migraphx/iterator.hpp>
14
#include <migraphx/algorithm.hpp>
15
#include <migraphx/output_iterator.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
16
#include <migraphx/make_op.hpp>
17
#include <migraphx/marker.hpp>
Paul's avatar
Paul committed
18
#include <iostream>
19
#include <numeric>
Paul's avatar
Paul committed
20
#include <sstream>
Paul's avatar
Paul committed
21
#include <algorithm>
22
#include <set>
Paul's avatar
Paul committed
23
#include <utility>
24
#include <iomanip>
25

26
#include <unordered_set>
Shucai Xiao's avatar
Shucai Xiao committed
27
28
#include <map>
#include <cassert>
Paul's avatar
Paul committed
29

Paul's avatar
Paul committed
30
namespace migraphx {
Paul's avatar
Paul committed
31
inline namespace MIGRAPHX_INLINE_NS {
Paul's avatar
Paul committed
32

33
34
using milliseconds = std::chrono::duration<double, std::milli>;

Paul's avatar
Paul committed
35
36
struct program_impl
{
Shucai Xiao's avatar
Shucai Xiao committed
37
    // A map is used to keep references to modules of the program
38
    std::unordered_map<std::string, module> modules;
Paul's avatar
Paul committed
39
    context ctx;
40
    std::string target_name;
Paul's avatar
Paul committed
41
42
};

43
program::program() : impl(std::make_unique<program_impl>()) { this->create_module("main"); }
Paul's avatar
Paul committed
44

Paul's avatar
Paul committed
45
program::program(program&&) noexcept = default;
Shucai Xiao's avatar
Shucai Xiao committed
46
program::~program() noexcept         = default;
Paul's avatar
Paul committed
47

48
// copy constructor
Shucai Xiao's avatar
Shucai Xiao committed
49
program::program(const program& p) { assign(p); }
50
51

// copy assignment operator
Shucai Xiao's avatar
Shucai Xiao committed
52
program& program::operator=(program p)
53
{
Shucai Xiao's avatar
Shucai Xiao committed
54
    std::swap(p.impl, this->impl);
55
56
57
    return *this;
}

Shucai Xiao's avatar
Shucai Xiao committed
58
void program::assign(const program& p)
59
{
Shucai Xiao's avatar
Shucai Xiao committed
60
    if(!impl)
61
62
63
    {
        impl = std::make_unique<program_impl>();
    }
Shucai Xiao's avatar
Shucai Xiao committed
64
    else if(!impl->modules.empty())
65
    {
Shucai Xiao's avatar
Shucai Xiao committed
66
        impl->modules.clear();
67
    }
Shucai Xiao's avatar
Shucai Xiao committed
68

69
    impl->ctx         = p.impl->ctx;
Shucai Xiao's avatar
Shucai Xiao committed
70
71
    impl->target_name = p.impl->target_name;
    impl->modules     = p.impl->modules;
Shucai Xiao's avatar
Shucai Xiao committed
72
73
74
75

    // build a map from old ins to new ins
    // Build a map from old module to new module
    std::unordered_map<module_ref, module_ref> mod_map;
Paul's avatar
Paul committed
76
77
78
79
80
    std::transform(
        impl->modules.begin(),
        impl->modules.end(),
        std::inserter(mod_map, mod_map.begin()),
        [&](auto&& xp) { return std::make_pair(&p.impl->modules.at(xp.first), &xp.second); });
Shucai Xiao's avatar
Shucai Xiao committed
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

    std::unordered_map<instruction_ref, instruction_ref> ins_map;
    for(auto&& pp : mod_map)
    {
        auto old_ins = iterator_for(*pp.first);
        auto new_ins = iterator_for(*pp.second);
        std::transform(old_ins.begin(),
                       old_ins.end(),
                       new_ins.begin(),
                       std::inserter(ins_map, ins_map.begin()),
                       [](auto x, auto y) { return std::make_pair(x, y); });
    }

    // Update all references from all modules
    for(auto&& mp : impl->modules)
    {
97
        for(auto ins : iterator_for(mp.second))
Shucai Xiao's avatar
Shucai Xiao committed
98
99
            instruction::replace_refs(ins, ins_map, mod_map);
    }
100
101
}

Paul's avatar
Paul committed
102
shape program::get_parameter_shape(std::string name) const
Paul's avatar
Paul committed
103
{
Shucai Xiao's avatar
Shucai Xiao committed
104
105
    const auto* mm = this->get_main_module();
    return mm->get_parameter_shape(std::move(name));
Paul's avatar
Paul committed
106
107
}

108
109
std::vector<std::string> program::get_parameter_names() const
{
Shucai Xiao's avatar
Shucai Xiao committed
110
111
    const auto* mm = this->get_main_module();
    return mm->get_parameter_names();
112
113
}

mei-ye's avatar
mei-ye committed
114
115
instruction_ref program::get_parameter(std::string name) const
{
Shucai Xiao's avatar
Shucai Xiao committed
116
117
    const auto* mm = this->get_main_module();
    return mm->get_parameter(std::move(name));
mei-ye's avatar
mei-ye committed
118
119
}

Paul's avatar
Paul committed
120
121
std::unordered_map<std::string, shape> program::get_parameter_shapes() const
{
Shucai Xiao's avatar
Shucai Xiao committed
122
123
    const auto* mm = this->get_main_module();
    return mm->get_parameter_shapes();
Paul's avatar
Paul committed
124
125
}

Shucai Xiao's avatar
Shucai Xiao committed
126
std::size_t program::size() const { return impl->modules.size(); }
127

128
129
std::vector<shape> program::get_output_shapes() const
{
Shucai Xiao's avatar
Shucai Xiao committed
130
131
    const auto* mm = this->get_main_module();
    return mm->get_output_shapes();
132
}
Paul's avatar
Paul committed
133

Paul's avatar
Paul committed
134
135
context& program::get_context() const { return impl->ctx; }

Paul's avatar
Paul committed
136
137
instruction_ref program::validate() const
{
Shucai Xiao's avatar
Shucai Xiao committed
138
139
    const auto* mm = this->get_main_module();
    return mm->validate();
Paul's avatar
Paul committed
140
141
}

142
143
bool program::is_compiled() const { return not this->impl->target_name.empty(); }

144
void program::compile(const target& t, compile_options options)
Paul's avatar
Paul committed
145
{
146
147
148
    assert(not this->is_compiled());
    this->impl->target_name = t.name();
    this->impl->ctx         = t.get_context();
Paul's avatar
Paul committed
149
    if(enabled(MIGRAPHX_TRACE_COMPILE{}))
150
        options.trace = tracer{std::cout};
Shucai Xiao's avatar
Shucai Xiao committed
151

152
153
    options.trace(*this);
    options.trace();
Shucai Xiao's avatar
Shucai Xiao committed
154

Shucai Xiao's avatar
Shucai Xiao committed
155
    auto&& passes = t.get_passes(this->impl->ctx, options);
156
157
158
    run_passes(*this, passes, options.trace);

    auto mods = this->get_modules();
Shucai Xiao's avatar
Shucai Xiao committed
159

160
161
    // Validate and finalize
    for(const auto& mod : reverse(mods))
Paul's avatar
Paul committed
162
    {
Shucai Xiao's avatar
Shucai Xiao committed
163
164
165
166
167
168
        auto invalid = mod->validate();
        if(invalid != mod->end())
        {
            MIGRAPHX_THROW("Invalid module " + mod->name() + " from compilation at instruction " +
                           std::to_string(std::distance(mod->begin(), invalid)));
        }
169
170
171
172
173
174
175
        auto dangling = mod->find_dangling_reference();
        if(dangling != mod->end())
        {
            auto index = std::distance(mod->begin(), dangling);
            MIGRAPHX_THROW("Dangling reference in module " + mod->name() + " from instruction " +
                           std::to_string(index));
        }
Shucai Xiao's avatar
Shucai Xiao committed
176
        mod->finalize(this->impl->ctx);
Paul's avatar
Paul committed
177
    }
Paul's avatar
Paul committed
178
179
180
181
}

void program::finalize()
{
Shucai Xiao's avatar
Shucai Xiao committed
182
183
    auto* mm = this->get_main_module();
    mm->finalize(this->impl->ctx);
Paul's avatar
Paul committed
184
185
}

Paul's avatar
Paul committed
186
template <class F>
Shucai Xiao's avatar
Shucai Xiao committed
187
std::vector<argument> generic_eval(const module* mod,
188
189
                                   context& ctx,
                                   std::unordered_map<std::string, argument> params,
Shucai Xiao's avatar
Shucai Xiao committed
190
                                   std::unordered_map<instruction_ref, argument> results,
191
                                   F make_trace)
Paul's avatar
Paul committed
192
{
Shucai Xiao's avatar
Shucai Xiao committed
193
194
    assert(mod->validate() == mod->end());
    results.reserve(mod->size() * 2);
Paul's avatar
Paul committed
195
196
    std::vector<argument> values;
    values.reserve(16);
197
    auto trace = make_trace(mod);
Shucai Xiao's avatar
Shucai Xiao committed
198
    for(auto ins : iterator_for(*mod))
Paul's avatar
Paul committed
199
    {
200
        assert(results.find(ins) == results.end());
201
202
        const auto& name = ins->name();
        if(name == "@literal")
Paul's avatar
Paul committed
203
        {
Paul's avatar
Paul committed
204
            results.emplace(ins, trace(ins, [&] { return ins->get_literal().get_argument(); }));
Paul's avatar
Paul committed
205
        }
206
        else if(name == "@param")
Paul's avatar
Paul committed
207
        {
Paul's avatar
Paul committed
208
209
210
211
212
            results.emplace(
                ins, trace(ins, [&] {
                    auto param_name = any_cast<builtin::param>(ins->get_operator()).parameter;
                    if(not contains(params, param_name))
                        MIGRAPHX_THROW("Parameter not found: " + param_name);
213
                    auto param = params[param_name];
Paul's avatar
Paul committed
214
215
216
217
218
                    if(param.get_shape() != ins->get_shape())
                        MIGRAPHX_THROW("Incorrect shape {" + to_string(param.get_shape()) +
                                       "} for parameter: " + param_name);
                    return param;
                }));
Paul's avatar
Paul committed
219
        }
220
        else if(name == "@outline")
Paul's avatar
Paul committed
221
        {
Paul's avatar
Paul committed
222
            results.emplace(ins, trace(ins, [&] { return argument{ins->get_shape(), nullptr}; }));
Paul's avatar
Paul committed
223
        }
224
225
226
227
228
229
230
231
232
233
234
235
236
        else if(name == "@return")
        {
            std::vector<argument> prog_outputs;
            std::transform(ins->inputs().begin(),
                           ins->inputs().end(),
                           std::back_inserter(prog_outputs),
                           [&](instruction_ref i) {
                               assert(results.find(i) != results.end());
                               return results[i];
                           });

            return prog_outputs;
        }
Paul's avatar
Paul committed
237
238
        else
        {
Paul's avatar
Paul committed
239
            values.resize(ins->inputs().size());
Paul's avatar
Paul committed
240
241
242
243
244
            std::transform(
                ins->inputs().begin(), ins->inputs().end(), values.begin(), [&](instruction_ref i) {
                    assert(results.find(i) != results.end());
                    return results[i];
                });
Shucai Xiao's avatar
Shucai Xiao committed
245
246
247
248

            const auto& mod_args = ins->module_inputs();
            auto module_eval     = [&](module_ref smod,
                                   const std::unordered_map<std::string, argument>& inputs) {
Shucai Xiao's avatar
Shucai Xiao committed
249
250
                auto ssctx = ctx;
                return generic_eval(smod, ssctx, inputs, results, make_trace);
Shucai Xiao's avatar
Shucai Xiao committed
251
252
            };

Shucai Xiao's avatar
Shucai Xiao committed
253
254
255
256
            results.emplace(ins, trace(ins, [&] {
                                return ins->normalized_operator().compute(
                                    ctx, ins->get_shape(), values, mod_args, module_eval);
                            }));
Paul's avatar
Paul committed
257
        }
258
        assert(results.find(ins) != results.end());
259
        assert(results.at(ins).get_shape() == ins->get_shape());
Paul's avatar
Paul committed
260
    }
Shucai Xiao's avatar
Shucai Xiao committed
261
    return {results.at(std::prev(mod->end()))};
Paul's avatar
Paul committed
262
263
}

Shucai Xiao's avatar
Shucai Xiao committed
264
265
266
267
template <class F>
std::vector<argument> generic_eval(const program& p,
                                   context& ctx,
                                   std::unordered_map<std::string, argument> params,
268
                                   F make_trace)
Shucai Xiao's avatar
Shucai Xiao committed
269
{
Shucai Xiao's avatar
Shucai Xiao committed
270
    const module* mm = p.get_main_module();
271
    return generic_eval(mm, ctx, params, {}, make_trace);
Shucai Xiao's avatar
Shucai Xiao committed
272
273
}

274
std::vector<argument> program::eval(parameter_map params) const
Paul's avatar
Paul committed
275
{
Paul's avatar
Paul committed
276
277
    auto& ctx = this->impl->ctx;
#ifndef NDEBUG
278
279
280
281
282
283
284
285
286
287
288
    auto with_check_context = [&](auto f) {
        return [=, &ctx](auto&&) {
            auto sctx          = std::make_shared<context>(ctx);
            auto check_context = [=, &ctx](auto g) {
                assert(is_shared(ctx, *sctx));
                auto x = g();
                *sctx  = ctx;
                return x;
            };
            return [=](auto&&... xs) { return f(xs..., check_context); };
        };
Paul's avatar
Paul committed
289
290
    };
#else
291
292
293
294
295
    auto with_check_context = [](auto f) {
        return [=](auto&&) {
            return [=](auto&&... xs) { return f(xs..., [](auto g) { return g(); }); };
        };
    };
Paul's avatar
Paul committed
296
#endif
Paul's avatar
Paul committed
297
298
299
300

    auto trace_level = value_of(MIGRAPHX_TRACE_EVAL{});

    if(trace_level > 0)
Paul's avatar
Paul committed
301
    {
302
303
304
305
306
307
308
309
310
311
312
313
314
315
        return generic_eval(*this,
                            ctx,
                            std::move(params),
                            with_check_context([&](auto& ins, auto f, auto&& check_context) {
                                ctx.finish();
                                std::cout << "Run instruction: ";
                                this->debug_print(ins);
                                timer t{};
                                auto result = check_context(f);
                                double t1   = t.record<milliseconds>();
                                ctx.finish();
                                double t2 = t.record<milliseconds>();
                                std::cout << "Time: " << t1 << "ms, " << t2 << "ms" << std::endl;
                                if(trace_level > 1 and ins->name().front() != '@' and
Shucai Xiao's avatar
Shucai Xiao committed
316
317
318
319
320
                                   ins->name() != "load" and not result.empty())
                                {
                                    target tgt = make_target(this->impl->target_name);
                                    std::cout << "Output: " << tgt.copy_from(result) << std::endl;
                                }
321
322
                                return result;
                            }));
Paul's avatar
Paul committed
323
324
325
    }
    else
    {
326
327
328
329
330
331
        return generic_eval(*this,
                            ctx,
                            std::move(params),
                            with_check_context([&](auto&, auto f, auto&& check_context) {
                                return check_context(f);
                            }));
Paul's avatar
Paul committed
332
    }
Paul's avatar
Paul committed
333
334
}

335
const int program_file_version = 5;
336
337
338
339
340
341
342
343

value program::to_value() const
{
    value result;
    result["version"] = program_file_version;
    result["target"]  = this->impl->target_name;
    if(not this->impl->target_name.empty())
        result["context"] = this->impl->ctx.to_value();
Shucai Xiao's avatar
Shucai Xiao committed
344

345
    value module_vals = value::object{};
Shucai Xiao's avatar
Shucai Xiao committed
346
    std::unordered_map<instruction_ref, std::string> names;
347
    for(auto& mod : this->get_modules())
Shucai Xiao's avatar
Shucai Xiao committed
348
    {
Shucai Xiao's avatar
Shucai Xiao committed
349
350
        value mod_val;
        value nodes;
351
352
        mod_val["name"] = mod->name();
        names           = mod->print(
Shucai Xiao's avatar
Shucai Xiao committed
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
            [&](auto ins, auto ins_names) {
                value node;
                node["output"]     = ins_names.at(ins);
                node["name"]       = ins->name();
                node["shape"]      = migraphx::to_value(ins->get_shape());
                node["normalized"] = ins->is_normalized();
                if(ins->name() == "@literal")
                    node["literal"] = migraphx::to_value(ins->get_literal());
                node["operator"] = ins->get_operator().to_value();
                std::vector<std::string> inputs;
                std::transform(ins->inputs().begin(),
                               ins->inputs().end(),
                               std::back_inserter(inputs),
                               [&](auto i) {
                                   assert(contains(ins_names, i));
                                   return ins_names.at(i);
                               });
                node["inputs"]   = inputs;
                auto module_args = ins->module_inputs();
                if(not module_args.empty())
                {
                    std::vector<std::string> module_inputs;
                    std::transform(module_args.begin(),
                                   module_args.end(),
                                   std::back_inserter(module_inputs),
                                   [&](auto mod_ref) { return mod_ref->name(); });
                    node["module_inputs"] = module_inputs;
                }

                nodes.push_back(node);
            },
            names);
        mod_val["nodes"] = nodes;

387
        module_vals[mod->name()] = mod_val;
Shucai Xiao's avatar
Shucai Xiao committed
388
    }
Shucai Xiao's avatar
Shucai Xiao committed
389
390
391

    result["modules"] = module_vals;

392
393
    return result;
}
Shucai Xiao's avatar
Shucai Xiao committed
394

Shucai Xiao's avatar
Shucai Xiao committed
395
396
397
398
399
static void mod_from_val(module_ref mod,
                         const value& v,
                         std::unordered_map<std::string, instruction_ref>& instructions,
                         const std::unordered_map<std::string, module_ref>& map_mods)
{
400
    const auto& module_val = v.at(mod->name());
Shucai Xiao's avatar
Shucai Xiao committed
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
    for(const value& node : module_val.at("nodes"))
    {
        instruction_ref output;
        auto name       = node.at("name").to<std::string>();
        auto fields     = node.at("operator");
        auto normalized = node.at("normalized").to<bool>();

        if(name == "@param")
        {
            output = mod->add_parameter(fields["parameter"].to<std::string>(),
                                        migraphx::from_value<shape>(node.at("shape")));
        }
        else if(name == "@literal")
        {
            output = mod->add_literal(migraphx::from_value<literal>(node.at("literal")));
        }
        else
        {
            auto op = make_op(name, fields);
            std::vector<instruction_ref> inputs;
            std::transform(node.at("inputs").begin(),
                           node.at("inputs").end(),
                           std::back_inserter(inputs),
                           [&](const value& i) {
                               auto i_name = i.to<std::string>();
                               assert(contains(instructions, i_name));
                               return instructions.at(i_name);
                           });

            std::vector<module_ref> module_inputs;
            if(node.contains("module_inputs"))
            {
                std::transform(node.at("module_inputs").begin(),
                               node.at("module_inputs").end(),
                               std::back_inserter(module_inputs),
                               [&](const value& i) { return map_mods.at(i.to<std::string>()); });

                for(auto& smod : module_inputs)
                {
                    mod_from_val(smod, v, instructions, map_mods);
                }
            }

            if(name == "@return")
            {
                output = mod->add_return(inputs);
            }
            else if(module_inputs.empty())
            {
                output = mod->add_instruction(op, inputs);
            }
            else
            {
                output = mod->add_instruction(op, inputs, module_inputs);
            }
        }
        output->set_normalized(normalized);
        instructions[node.at("output").to<std::string>()] = output;
    }
}

462
463
464
465
void program::from_value(const value& v)
{
    auto version = v.at("version").to<int>();
    if(version != program_file_version)
Shucai Xiao's avatar
Shucai Xiao committed
466
467
468
469
    {
        MIGRAPHX_THROW("Warning: Program version mismatch");
    }

470
471
472
473
474
475
476
477
    this->impl->target_name = v.at("target").to<std::string>();
    if(not this->impl->target_name.empty())
    {
        target t        = make_target(this->impl->target_name);
        this->impl->ctx = t.get_context();
        this->impl->ctx.from_value(v.at("context"));
    }

Shucai Xiao's avatar
Shucai Xiao committed
478
479
    auto module_vals = v.at("modules");
    for(const auto& vv : module_vals)
480
    {
481
        const auto& name = vv.get_key();
Shucai Xiao's avatar
Shucai Xiao committed
482
483
        if(name == "main")
            continue;
484
        impl->modules.emplace(name, name);
485
    }
486
    std::unordered_map<std::string, module_ref> map_mods;
Paul's avatar
Paul committed
487
488
489
490
    std::transform(impl->modules.begin(),
                   impl->modules.end(),
                   std::inserter(map_mods, map_mods.end()),
                   [&](auto&& pp) { return std::make_pair(pp.first, &pp.second); });
Shucai Xiao's avatar
Shucai Xiao committed
491
492
493
494
495

    std::unordered_map<std::string, instruction_ref> map_insts;
    auto* mm = get_main_module();
    mod_from_val(mm, module_vals, map_insts, map_mods);

496
497
498
    this->finalize();
}

Paul's avatar
Paul committed
499
500
501
double common_average(const std::vector<double>& v)
{
    std::size_t n = v.size() / 4;
Paul's avatar
Paul committed
502
503
    double total  = std::accumulate(v.begin() + n, v.end() - n, 0.0);
    return total / std::distance(v.begin() + n, v.end() - n);
Paul's avatar
Paul committed
504
505
}

Paul Fultz II's avatar
Paul Fultz II committed
506
507
508
509
510
511
512
513
std::string perf_group(const operation& op)
{
    auto attr = op.attributes();
    if(attr.contains("group"))
        return attr.at("group").to<std::string>();
    return op.name();
}

514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
void program::mark(const parameter_map& params, marker&& m)
{
    auto& ctx = this->impl->ctx;
    // Run once by itself
    eval(params);
    ctx.finish();
    // Start marking
    m.mark_start(*this);
    generic_eval(*this, ctx, params, always([&](auto ins, auto f) {
        argument result;
        m.mark_start(ins);
        result = f();
        m.mark_stop(ins);
        return result;
    }));
    m.mark_stop(*this);
}

532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
static void print_space(std::ostream& os, int n)
{
    for(int i = 0; i < n; ++i)
    {
        os << ' ';
    }
}

using op_flops = std::function<double(const std::vector<shape>& vec_ss)>;

auto get_flops_funcs()
{
    static std::unordered_map<std::string, op_flops> op_funcs;
    op_funcs.emplace("gemm", [&](const std::vector<shape>& vec_ss) {
        assert(vec_ss.size() >= 2);
Shucai Xiao's avatar
Shucai Xiao committed
547
548
549
        auto sa     = vec_ss.front();
        auto sb     = vec_ss.at(1);
        auto batch  = 1;
550
        auto lens_a = sa.lens();
Shucai Xiao's avatar
Shucai Xiao committed
551
552
553
554
        batch =
            std::accumulate(lens_a.rbegin() + 2, lens_a.rend(), 1, std::multiplies<std::size_t>{});
        auto m      = lens_a[lens_a.size() - 2];
        auto k      = lens_a.back();
555
556
557
558
559
560
561
562
563
564
        auto lens_b = sb.lens();
        assert(k == lens_b[lens_b.size() - 2]);
        auto n = lens_b.back();

        return 2.0 * m * n * k * batch;
    });

    return op_funcs;
}

565
566
567
568
void program::perf_report(std::ostream& os,
                          std::size_t n,
                          parameter_map params,
                          std::size_t batch) const
Paul's avatar
Paul committed
569
{
570
    auto& ctx = this->impl->ctx;
Paul's avatar
Paul committed
571
572
    // Run once by itself
    eval(params);
Paul's avatar
Paul committed
573
    ctx.finish();
Paul's avatar
Paul committed
574
    // Run and time entire program
Paul's avatar
Paul committed
575
576
    std::vector<double> total_vec;
    total_vec.reserve(n);
Paul's avatar
Paul committed
577
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
578
    {
Paul's avatar
Paul committed
579
580
581
582
        total_vec.push_back(time<milliseconds>([&] {
            eval(params);
            ctx.finish();
        }));
Paul's avatar
Paul committed
583
    }
Paul's avatar
Paul committed
584
585
    std::sort(total_vec.begin(), total_vec.end());
    std::unordered_map<instruction_ref, std::vector<double>> ins_vec;
Paul's avatar
Paul committed
586
    // Fill the map
587
    generic_eval(*this, ctx, params, always([&](auto ins, auto) {
Paul's avatar
Paul committed
588
        ins_vec[ins].reserve(n);
589
        return argument{ins->get_shape(), nullptr};
590
    }));
591

Paul's avatar
Paul committed
592
    // Run and time each instruction
Paul's avatar
Paul committed
593
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
594
    {
595
        generic_eval(*this, ctx, params, always([&](auto ins, auto f) {
596
            argument result;
Paul's avatar
Paul committed
597
598
599
600
            ins_vec[ins].push_back(time<milliseconds>([&] {
                result = f();
                ctx.finish();
            }));
601
            return result;
602
        }));
Paul's avatar
Paul committed
603
    }
Paul's avatar
Paul committed
604
605
    for(auto&& p : ins_vec)
        std::sort(p.second.begin(), p.second.end());
Paul's avatar
Paul committed
606
    // Run and time implicit overhead
Paul's avatar
Paul committed
607
608
    std::vector<double> overhead_vec;
    overhead_vec.reserve(n);
Paul's avatar
Paul committed
609
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
610
    {
Paul's avatar
Paul committed
611
        overhead_vec.push_back(time<milliseconds>([&] { dry_run(params); }));
Paul's avatar
Paul committed
612
613
    }

Paul's avatar
Paul committed
614
    double total_time             = common_average(total_vec);
Paul's avatar
Paul committed
615
    double rate                   = 1000.0 / total_time;
Paul's avatar
Paul committed
616
    double overhead_time          = common_average(overhead_vec);
Paul's avatar
Paul committed
617
    double overhead_percent       = overhead_time * 100.0 / total_time;
Paul's avatar
Paul committed
618
    double total_instruction_time = 0.0;
Paul's avatar
Paul committed
619
    std::unordered_map<std::string, double> op_times;
Paul's avatar
Paul committed
620
    for(auto&& p : ins_vec)
Paul's avatar
Paul committed
621
622
    {
        double avg = common_average(p.second);
Paul Fultz II's avatar
Paul Fultz II committed
623
        op_times[perf_group(p.first->get_operator())] += avg;
Paul's avatar
Paul committed
624
625
        total_instruction_time += avg;
    }
Paul's avatar
Paul committed
626
627
    double calculate_overhead_time    = total_time - total_instruction_time;
    double calculate_overhead_percent = calculate_overhead_time * 100.0 / total_time;
Paul's avatar
Paul committed
628

Shucai Xiao's avatar
Shucai Xiao committed
629
    std::unordered_map<instruction_ref, std::string> names;
630
631
632
633
634
635

    // count max instruction length
    int max_ins_len = 0;
    this->print(names, [&](auto ins, auto ins_names) {
        std::stringstream ss;
        instruction::print(ss, ins, ins_names);
Shucai Xiao's avatar
Shucai Xiao committed
636
        if(max_ins_len < ss.str().length())
637
638
639
640
641
642
643
644
645
646
        {
            max_ins_len = ss.str().length();
        }

        // skip return instruction
        if(ins->name() == "@return")
            return;
    });

    int print_ins_len = max_ins_len + 4;
Shucai Xiao's avatar
Shucai Xiao committed
647
    std::string str   = "Instructions";
648
649
    os << str;
    print_space(os, print_ins_len - str.length());
Shucai Xiao's avatar
Shucai Xiao committed
650
    std::string time_str    = "Time(ms)        ";
651
    std::string percent_str = "Percentage    ";
Shucai Xiao's avatar
Shucai Xiao committed
652
    std::string flops_str   = "Flops(GFlops/s)";
653
654
655
    os << time_str << percent_str << flops_str << std::endl;

    auto flops_funcs = get_flops_funcs();
Shucai Xiao's avatar
Shucai Xiao committed
656
    this->print(names, [&](auto ins, auto ins_names) {
657
658
659
        std::stringstream ss;
        instruction::print(ss, ins, ins_names);
        os << ss.str();
660
661
662
663
664

        // skip return instruction
        if(ins->name() == "@return")
            return;

665
666
        print_space(os, print_ins_len - ss.str().length());

Shucai Xiao's avatar
Shucai Xiao committed
667
        double avg      = common_average(ins_vec[ins]);
668
669
        std::string tms = std::to_string(avg);
        tms.append(time_str.length() - tms.length(), ' ');
Shucai Xiao's avatar
Shucai Xiao committed
670
        double percent   = 100.0 * avg / total_instruction_time;
671
        std::string pers = std::to_string(percent);
Shucai Xiao's avatar
Shucai Xiao committed
672
673
        auto loc         = pers.find('.');
        if(loc != std::string::npos)
674
675
676
677
678
679
680
681
        {
            pers.erase(pers.begin() + loc + 6, pers.end());
        }
        pers.append(percent_str.length() - pers.length(), ' ');

        // calculate flops
        std::string flps;
        std::string op_name = ins->name();
Shucai Xiao's avatar
Shucai Xiao committed
682
        auto nloc           = op_name.find("::");
683
        op_name.erase(op_name.begin(), op_name.begin() + nloc + 2);
Shucai Xiao's avatar
Shucai Xiao committed
684
        if(contains(flops_funcs, op_name))
685
686
        {
            auto op_flop_func = flops_funcs.at(op_name);
Shucai Xiao's avatar
Shucai Xiao committed
687
688
            auto inss         = to_shapes(ins->inputs());
            double flops      = op_flop_func(inss);
689
690
691
            flops /= avg;
            // convert to GFlops
            flops /= 1.0e6;
Shucai Xiao's avatar
Shucai Xiao committed
692
            flps      = std::to_string(flops);
693
            auto floc = flps.find('.');
Shucai Xiao's avatar
Shucai Xiao committed
694
            if(floc != std::string::npos)
695
696
697
698
699
            {
                flps.erase(flps.begin() + floc + 4, flps.end());
            }
        }
        os << tms << pers << flps << std::endl;
Paul's avatar
Paul committed
700
    });
Paul's avatar
Paul committed
701
702
703

    os << std::endl;
    os << "Summary:" << std::endl;
704
705
706
707
708
709
710
    std::vector<std::pair<double, std::string>> op_times_sorted;
    std::transform(op_times.begin(),
                   op_times.end(),
                   std::back_inserter(op_times_sorted),
                   [](auto p) { return std::make_pair(p.second, p.first); });
    std::sort(op_times_sorted.begin(), op_times_sorted.end(), std::greater<>{});
    for(auto&& p : op_times_sorted)
Paul's avatar
Paul committed
711
    {
712
713
        auto&& name    = p.second;
        double avg     = p.first;
Paul's avatar
Paul committed
714
715
716
717
718
        double percent = std::ceil(100.0 * avg / total_instruction_time);
        os << name << ": " << avg << "ms, " << percent << "%" << std::endl;
    }

    os << std::endl;
Paul's avatar
Paul committed
719

720
721
    os << "Batch size: " << batch << std::endl;
    os << "Rate: " << rate * batch << "/sec" << std::endl;
Paul's avatar
Paul committed
722
723
    os << "Total time: " << total_time << "ms" << std::endl;
    os << "Total instructions time: " << total_instruction_time << "ms" << std::endl;
Paul's avatar
Paul committed
724
725
726
727
    os << "Overhead time: " << overhead_time << "ms"
       << ", " << calculate_overhead_time << "ms" << std::endl;
    os << "Overhead: " << std::round(overhead_percent) << "%"
       << ", " << std::round(calculate_overhead_percent) << "%" << std::endl;
Paul's avatar
Paul committed
728
729
}

Paul's avatar
Paul committed
730
731
void program::debug_print() const { std::cout << *this << std::endl; }
void program::debug_print(instruction_ref ins) const
Paul's avatar
Paul committed
732
{
Shucai Xiao's avatar
Shucai Xiao committed
733
    std::unordered_map<instruction_ref, std::string> names;
734
    if(std::any_of(this->impl->modules.begin(), this->impl->modules.end(), [&](const auto& pp) {
735
           return is_end(pp.second.end(), ins);
Shucai Xiao's avatar
Shucai Xiao committed
736
       }))
Paul's avatar
Paul committed
737
738
739
740
    {
        std::cout << "End instruction" << std::endl;
        return;
    }
741
742
    else if(std::none_of(this->impl->modules.begin(),
                         this->impl->modules.end(),
743
                         [&](const auto& pp) { return pp.second.has_instruction(ins); }))
Paul's avatar
Paul committed
744
745
746
747
    {
        std::cout << "Instruction not part of program" << std::endl;
        return;
    }
Shucai Xiao's avatar
Shucai Xiao committed
748

Paul's avatar
Paul committed
749
    std::stringstream ss;
Shucai Xiao's avatar
Shucai Xiao committed
750
    this->print(names, [&](auto x, auto ins_names) {
Paul's avatar
Paul committed
751
        if(x == ins)
Paul's avatar
Paul committed
752
        {
Shucai Xiao's avatar
Shucai Xiao committed
753
            instruction::print(std::cout, x, ins_names);
Paul's avatar
Paul committed
754
755
756
757
758
            std::cout << std::endl;
        }
    });
}

Shucai Xiao's avatar
Shucai Xiao committed
759
760
761
762
void program::print(
    std::unordered_map<instruction_ref, std::string>& names,
    const std::function<void(instruction_ref, std::unordered_map<instruction_ref, std::string>)>&
        print_func) const
763
{
764
    for(const auto& pp : this->impl->modules)
765
    {
766
        names = pp.second.print(print_func, names);
767
768
769
    }
}

Shucai Xiao's avatar
Shucai Xiao committed
770
void program::print_graph(std::ostream& os, bool brief) const
771
{
Shucai Xiao's avatar
Shucai Xiao committed
772
773
    const auto* mm = this->get_main_module();
    mm->print_graph(os, brief);
774
775
776
777
}

void program::print_cpp(std::ostream& os) const
{
Shucai Xiao's avatar
Shucai Xiao committed
778
779
780
781
782
783
784
785
    auto vec_modules = this->get_modules();
    std::unordered_map<instruction_ref, std::string> names;
    for(auto& mod : vec_modules)
    {
        os << "module: \"" << mod->name() << "\"" << std::endl;
        names = mod->print_cpp(os, names);
        os << std::endl;
    }
786
787
}

Paul's avatar
Paul committed
788
789
void program::dry_run(std::unordered_map<std::string, argument> params) const
{
Paul's avatar
Paul committed
790
    auto& ctx = this->impl->ctx;
791
792
793
    generic_eval(*this, ctx, std::move(params), always([](auto ins, auto&&...) {
        return argument{ins->get_shape(), nullptr};
    }));
Paul's avatar
Paul committed
794
795
}

Shucai Xiao's avatar
Shucai Xiao committed
796
void program::annotate(std::ostream& os, const std::function<void(instruction_ref)>& a) const
Paul's avatar
Paul committed
797
{
798
    for(auto& pp : this->impl->modules)
Shucai Xiao's avatar
Shucai Xiao committed
799
    {
800
801
        std::cout << pp.first << ":" << std::endl;
        pp.second.annotate(os, a);
Shucai Xiao's avatar
Shucai Xiao committed
802
    }
Paul's avatar
Paul committed
803
804
}

Paul's avatar
Paul committed
805
const module* program::get_module(const std::string& name) const { return &impl->modules.at(name); }
Shucai Xiao's avatar
Shucai Xiao committed
806
807
808

module* program::create_module(const std::string& name)
{
809
    assert(not contains(impl->modules, name));
810
811
    auto r = impl->modules.emplace(name, name);
    return &(r.first->second);
Shucai Xiao's avatar
Shucai Xiao committed
812
813
}

Paul's avatar
Paul committed
814
module* program::get_module(const std::string& name) { return &impl->modules.at(name); }
Shucai Xiao's avatar
Shucai Xiao committed
815
816
817
818
819

module* program::get_main_module() { return get_module("main"); }

const module* program::get_main_module() const { return get_module("main"); }

Paul's avatar
Paul committed
820
template <class T>
821
std::vector<T*> generic_get_modules(T* mm)
Shucai Xiao's avatar
Shucai Xiao committed
822
{
823
    std::vector<T*> vec_modules;
Shucai Xiao's avatar
Shucai Xiao committed
824
825
826
827
828
    vec_modules.push_back(mm);
    auto sub_modules = mm->get_sub_modules();
    vec_modules.insert(vec_modules.end(), sub_modules.begin(), sub_modules.end());
    return vec_modules;
}
Shucai Xiao's avatar
Shucai Xiao committed
829

Paul's avatar
Paul committed
830
template <class Map, class T, class OutputIterator>
831
void generic_get_unused_modules(Map& m, const std::vector<T*>& mods, OutputIterator out)
Shucai Xiao's avatar
Shucai Xiao committed
832
{
833
834
835
836
    std::unordered_set<std::string> used;
    std::transform(mods.begin(), mods.end(), std::inserter(used, used.end()), [](auto&& mod) {
        return mod->name();
    });
Paul's avatar
Paul committed
837
838
839
840
841
    transform_if(m.begin(),
                 m.end(),
                 out,
                 [&](auto&& pp) { return not contains(used, pp.first); },
                 [](auto&& pp) { return &pp.second; });
842
}
Shucai Xiao's avatar
Shucai Xiao committed
843

844
845
846
847
848
849
850
851
852
853
854
855
std::vector<const module*> program::get_modules() const
{
    auto result = generic_get_modules(this->get_main_module());
    generic_get_unused_modules(impl->modules, result, std::back_inserter(result));
    return result;
}

std::vector<module*> program::get_modules()
{
    auto result = generic_get_modules(this->get_main_module());
    generic_get_unused_modules(impl->modules, result, std::back_inserter(result));
    return result;
Shucai Xiao's avatar
Shucai Xiao committed
856
857
}

858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
template <class Map, class T>
bool is_unused_module(Map& m, const std::vector<T*>& mods, const std::string& name)
{
    bool is_unused = false;
    generic_get_unused_modules(m, mods, make_function_output_iterator([&](auto* mod) {
                                   if(mod->name() == name)
                                       is_unused = true;
                               }));
    return is_unused;
}

template <class Map>
bool references_instruction(Map& m, const instruction& ins, const std::string& name)
{
    return std::any_of(m.begin(), m.end(), [&](auto&& p) {
        if(p.first == name)
            return false;
        return std::any_of(p.second.begin(), p.second.end(), [&](auto&& i) {
            return std::any_of(i.inputs().begin(), i.inputs().end(), [&](auto&& j) {
                return std::addressof(*j) == std::addressof(ins);
            });
        });
    });
}

void program::remove_module(const std::string& name)
{
    // cppcheck-suppress assertWithSideEffect
    assert(is_unused_module(impl->modules, generic_get_modules(this->get_main_module()), name) &&
           "Module used in program");
    assert(std::none_of(
               impl->modules.at(name).begin(),
               impl->modules.at(name).end(),
               [&](auto&& ins) { return references_instruction(impl->modules, ins, name); }) &&
           "Instruction referenced in another module");
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908

    // if an instruction has an input out side of the current module, need to remove
    // the instruction from its input's outputs
    auto& mod = impl->modules.at(name);
    for(auto ins : iterator_for(mod))
    {
        auto inputs = ins->inputs();
        for(auto in : inputs)
        {
            if(not mod.has_instruction(in))
            {
                in->remove_output(ins);
            }
        }
    }

909
910
911
912
913
914
915
916
917
918
919
920
    impl->modules.erase(name);
}

void program::remove_unused_modules()
{
    std::vector<module*> unused;
    generic_get_unused_modules(
        impl->modules, generic_get_modules(this->get_main_module()), std::back_inserter(unused));
    for(auto* m : unused)
        this->remove_module(m->name());
}

921
922
program& program::sort()
{
923
    for(auto& pp : this->impl->modules)
Shucai Xiao's avatar
Shucai Xiao committed
924
    {
925
        pp.second.sort();
Shucai Xiao's avatar
Shucai Xiao committed
926
927
    }

928
929
930
    return *this;
}

Paul's avatar
Paul committed
931
bool operator==(const program& x, const program& y) { return to_string(x) == to_string(y); }
Paul's avatar
Paul committed
932

Paul's avatar
Paul committed
933
std::ostream& operator<<(std::ostream& os, const program& p)
Paul's avatar
Paul committed
934
{
Shucai Xiao's avatar
Shucai Xiao committed
935
936
937
    auto vec_modules = p.get_modules();
    std::unordered_map<instruction_ref, std::string> names;
    for(auto& mod : vec_modules)
Shucai Xiao's avatar
Shucai Xiao committed
938
    {
Shucai Xiao's avatar
Shucai Xiao committed
939
940
941
942
943
944
945
        os << "module: \"" << mod->name() << "\"" << std::endl;
        names = mod->print(
            [&](auto ins, auto ins_names) {
                instruction::print(os, ins, ins_names);
                os << std::endl;
            },
            names);
946
        os << std::endl;
Shucai Xiao's avatar
Shucai Xiao committed
947
948
    }

Paul's avatar
Paul committed
949
    return os;
Paul's avatar
Paul committed
950
}
Paul's avatar
Paul committed
951

Paul's avatar
Paul committed
952
} // namespace MIGRAPHX_INLINE_NS
Paul's avatar
Paul committed
953
} // namespace migraphx