program.cpp 34.1 KB
Newer Older
1
#include <functional>
Paul's avatar
Paul committed
2
3
4
#include <migraphx/program.hpp>
#include <migraphx/stringutils.hpp>
#include <migraphx/instruction.hpp>
5
#include <migraphx/op/identity.hpp>
Paul's avatar
Paul committed
6
#include <migraphx/target.hpp>
Paul's avatar
Paul committed
7
8
9
#include <migraphx/env.hpp>
#include <migraphx/ranges.hpp>
#include <migraphx/time.hpp>
10
#include <migraphx/pass_manager.hpp>
11
#include <migraphx/register_target.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
12
#include <migraphx/iterator_for.hpp>
13
#include <migraphx/iterator.hpp>
14
#include <migraphx/algorithm.hpp>
15
#include <migraphx/output_iterator.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
16
#include <migraphx/make_op.hpp>
17
#include <migraphx/marker.hpp>
Paul's avatar
Paul committed
18
#include <iostream>
19
#include <numeric>
Paul's avatar
Paul committed
20
#include <sstream>
Paul's avatar
Paul committed
21
#include <algorithm>
22
#include <set>
Paul's avatar
Paul committed
23
#include <utility>
24
#include <iomanip>
25

26
#include <unordered_set>
Shucai Xiao's avatar
Shucai Xiao committed
27
28
#include <map>
#include <cassert>
Paul's avatar
Paul committed
29

Paul's avatar
Paul committed
30
namespace migraphx {
Paul's avatar
Paul committed
31
inline namespace MIGRAPHX_INLINE_NS {
Paul's avatar
Paul committed
32

33
34
using milliseconds = std::chrono::duration<double, std::milli>;

Paul's avatar
Paul committed
35
36
struct program_impl
{
Shucai Xiao's avatar
Shucai Xiao committed
37
    // A map is used to keep references to modules of the program
38
    std::unordered_map<std::string, module> modules;
Paul's avatar
Paul committed
39
    context ctx;
40
    std::string target_name;
Paul's avatar
Paul committed
41
42
};

43
program::program() : impl(std::make_unique<program_impl>()) { this->create_module("main"); }
Paul's avatar
Paul committed
44

Paul's avatar
Paul committed
45
program::program(program&&) noexcept = default;
Shucai Xiao's avatar
Shucai Xiao committed
46
program::~program() noexcept         = default;
Paul's avatar
Paul committed
47

48
// copy constructor
Shucai Xiao's avatar
Shucai Xiao committed
49
program::program(const program& p) { assign(p); }
50
51

// copy assignment operator
Shucai Xiao's avatar
Shucai Xiao committed
52
program& program::operator=(program p)
53
{
Shucai Xiao's avatar
Shucai Xiao committed
54
    std::swap(p.impl, this->impl);
55
56
57
    return *this;
}

Shucai Xiao's avatar
Shucai Xiao committed
58
void program::assign(const program& p)
59
{
Shucai Xiao's avatar
Shucai Xiao committed
60
    if(!impl)
61
62
63
    {
        impl = std::make_unique<program_impl>();
    }
Shucai Xiao's avatar
Shucai Xiao committed
64
    else if(!impl->modules.empty())
65
    {
Shucai Xiao's avatar
Shucai Xiao committed
66
        impl->modules.clear();
67
    }
Shucai Xiao's avatar
Shucai Xiao committed
68

69
    impl->ctx         = p.impl->ctx;
Shucai Xiao's avatar
Shucai Xiao committed
70
71
    impl->target_name = p.impl->target_name;
    impl->modules     = p.impl->modules;
Shucai Xiao's avatar
Shucai Xiao committed
72
73
74
75

    // build a map from old ins to new ins
    // Build a map from old module to new module
    std::unordered_map<module_ref, module_ref> mod_map;
Paul's avatar
Paul committed
76
77
78
79
80
    std::transform(
        impl->modules.begin(),
        impl->modules.end(),
        std::inserter(mod_map, mod_map.begin()),
        [&](auto&& xp) { return std::make_pair(&p.impl->modules.at(xp.first), &xp.second); });
Shucai Xiao's avatar
Shucai Xiao committed
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

    std::unordered_map<instruction_ref, instruction_ref> ins_map;
    for(auto&& pp : mod_map)
    {
        auto old_ins = iterator_for(*pp.first);
        auto new_ins = iterator_for(*pp.second);
        std::transform(old_ins.begin(),
                       old_ins.end(),
                       new_ins.begin(),
                       std::inserter(ins_map, ins_map.begin()),
                       [](auto x, auto y) { return std::make_pair(x, y); });
    }

    // Update all references from all modules
    for(auto&& mp : impl->modules)
    {
97
        for(auto ins : iterator_for(mp.second))
Shucai Xiao's avatar
Shucai Xiao committed
98
99
            instruction::replace_refs(ins, ins_map, mod_map);
    }
100
101
}

Paul's avatar
Paul committed
102
shape program::get_parameter_shape(std::string name) const
Paul's avatar
Paul committed
103
{
Shucai Xiao's avatar
Shucai Xiao committed
104
105
    const auto* mm = this->get_main_module();
    return mm->get_parameter_shape(std::move(name));
Paul's avatar
Paul committed
106
107
}

108
109
std::vector<std::string> program::get_parameter_names() const
{
Shucai Xiao's avatar
Shucai Xiao committed
110
111
    const auto* mm = this->get_main_module();
    return mm->get_parameter_names();
112
113
}

mei-ye's avatar
mei-ye committed
114
115
instruction_ref program::get_parameter(std::string name) const
{
Shucai Xiao's avatar
Shucai Xiao committed
116
117
    const auto* mm = this->get_main_module();
    return mm->get_parameter(std::move(name));
mei-ye's avatar
mei-ye committed
118
119
}

Paul's avatar
Paul committed
120
121
std::unordered_map<std::string, shape> program::get_parameter_shapes() const
{
Shucai Xiao's avatar
Shucai Xiao committed
122
123
    const auto* mm = this->get_main_module();
    return mm->get_parameter_shapes();
Paul's avatar
Paul committed
124
125
}

Shucai Xiao's avatar
Shucai Xiao committed
126
std::size_t program::size() const { return impl->modules.size(); }
127

128
129
std::vector<shape> program::get_output_shapes() const
{
Shucai Xiao's avatar
Shucai Xiao committed
130
131
    const auto* mm = this->get_main_module();
    return mm->get_output_shapes();
132
}
Paul's avatar
Paul committed
133

Paul's avatar
Paul committed
134
135
context& program::get_context() const { return impl->ctx; }

Paul's avatar
Paul committed
136
137
instruction_ref program::validate() const
{
Shucai Xiao's avatar
Shucai Xiao committed
138
139
    const auto* mm = this->get_main_module();
    return mm->validate();
Paul's avatar
Paul committed
140
141
}

142
143
bool program::is_compiled() const { return not this->impl->target_name.empty(); }

144
void program::compile(const target& t, compile_options options)
Paul's avatar
Paul committed
145
{
146
147
148
    assert(not this->is_compiled());
    this->impl->target_name = t.name();
    this->impl->ctx         = t.get_context();
Paul's avatar
Paul committed
149
    if(enabled(MIGRAPHX_TRACE_COMPILE{}))
150
        options.trace = tracer{std::cout};
Shucai Xiao's avatar
Shucai Xiao committed
151

152
153
    options.trace(*this);
    options.trace();
Shucai Xiao's avatar
Shucai Xiao committed
154

Shucai Xiao's avatar
Shucai Xiao committed
155
    auto&& passes = t.get_passes(this->impl->ctx, options);
156
157
158
    run_passes(*this, passes, options.trace);

    auto mods = this->get_modules();
Shucai Xiao's avatar
Shucai Xiao committed
159

160
161
    // Validate and finalize
    for(const auto& mod : reverse(mods))
Paul's avatar
Paul committed
162
    {
Shucai Xiao's avatar
Shucai Xiao committed
163
164
165
166
167
168
        auto invalid = mod->validate();
        if(invalid != mod->end())
        {
            MIGRAPHX_THROW("Invalid module " + mod->name() + " from compilation at instruction " +
                           std::to_string(std::distance(mod->begin(), invalid)));
        }
169
170
171
172
173
174
175
        auto dangling = mod->find_dangling_reference();
        if(dangling != mod->end())
        {
            auto index = std::distance(mod->begin(), dangling);
            MIGRAPHX_THROW("Dangling reference in module " + mod->name() + " from instruction " +
                           std::to_string(index));
        }
Shucai Xiao's avatar
Shucai Xiao committed
176
        mod->finalize(this->impl->ctx);
Paul's avatar
Paul committed
177
    }
Paul's avatar
Paul committed
178
179
180
181
}

void program::finalize()
{
Shucai Xiao's avatar
Shucai Xiao committed
182
183
    auto* mm = this->get_main_module();
    mm->finalize(this->impl->ctx);
Paul's avatar
Paul committed
184
185
}

186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
template <class T>
std::string classify(T x)
{
    switch(std::fpclassify(x))
    {
    case FP_INFINITE: return "inf";
    case FP_NAN: return "nan";
    case FP_NORMAL: return "normal";
    case FP_SUBNORMAL: return "subnormal";
    case FP_ZERO: return "zero";
    default: return "unknown";
    }
}

std::unordered_set<std::string> classify_argument(const argument& a)
{
    std::unordered_set<std::string> result;
    a.visit(
        [&](auto t) {
            for(const auto& x : t)
                result.insert(classify(x));
        },
        [&](const auto& xs) {
            for(const auto& x : xs)
            {
                auto r = classify_argument(x);
                result.insert(r.begin(), r.end());
            }
        });
    return result;
}

void preview_argument(std::ostream& os, const argument& a)
{
    a.visit(
        [&](auto t) {
            if(t.size() <= 10)
            {
                os << t;
            }
            else
            {
                os << to_string_range(t.begin(), t.begin() + 5);
                os << ", ..., ";
                os << to_string_range(t.end() - 5, t.end());
            }
        },
        [&](const auto& xs) {
            for(const auto& x : xs)
            {
                os << '{';
                preview_argument(os, x);
                os << '}';
            }
        });
}

Paul's avatar
Paul committed
243
template <class F>
Shucai Xiao's avatar
Shucai Xiao committed
244
std::vector<argument> generic_eval(const module* mod,
245
246
                                   context& ctx,
                                   std::unordered_map<std::string, argument> params,
Shucai Xiao's avatar
Shucai Xiao committed
247
                                   std::unordered_map<instruction_ref, argument> results,
248
                                   F make_trace)
Paul's avatar
Paul committed
249
{
Shucai Xiao's avatar
Shucai Xiao committed
250
251
    assert(mod->validate() == mod->end());
    results.reserve(mod->size() * 2);
Paul's avatar
Paul committed
252
253
    std::vector<argument> values;
    values.reserve(16);
254
    auto trace = make_trace(mod);
Shucai Xiao's avatar
Shucai Xiao committed
255
    for(auto ins : iterator_for(*mod))
Paul's avatar
Paul committed
256
    {
257
        assert(results.find(ins) == results.end());
258
259
        const auto& name = ins->name();
        if(name == "@literal")
Paul's avatar
Paul committed
260
        {
Paul's avatar
Paul committed
261
            results.emplace(ins, trace(ins, [&] { return ins->get_literal().get_argument(); }));
Paul's avatar
Paul committed
262
        }
263
        else if(name == "@param")
Paul's avatar
Paul committed
264
        {
Paul's avatar
Paul committed
265
266
267
268
269
            results.emplace(
                ins, trace(ins, [&] {
                    auto param_name = any_cast<builtin::param>(ins->get_operator()).parameter;
                    if(not contains(params, param_name))
                        MIGRAPHX_THROW("Parameter not found: " + param_name);
270
                    auto param = params[param_name];
Paul's avatar
Paul committed
271
272
273
274
275
                    if(param.get_shape() != ins->get_shape())
                        MIGRAPHX_THROW("Incorrect shape {" + to_string(param.get_shape()) +
                                       "} for parameter: " + param_name);
                    return param;
                }));
Paul's avatar
Paul committed
276
        }
277
        else if(name == "@outline")
Paul's avatar
Paul committed
278
        {
Paul's avatar
Paul committed
279
            results.emplace(ins, trace(ins, [&] { return argument{ins->get_shape(), nullptr}; }));
Paul's avatar
Paul committed
280
        }
281
282
283
284
285
286
287
288
289
290
291
292
293
        else if(name == "@return")
        {
            std::vector<argument> prog_outputs;
            std::transform(ins->inputs().begin(),
                           ins->inputs().end(),
                           std::back_inserter(prog_outputs),
                           [&](instruction_ref i) {
                               assert(results.find(i) != results.end());
                               return results[i];
                           });

            return prog_outputs;
        }
Paul's avatar
Paul committed
294
295
        else
        {
Paul's avatar
Paul committed
296
            values.resize(ins->inputs().size());
Paul's avatar
Paul committed
297
298
299
300
301
            std::transform(
                ins->inputs().begin(), ins->inputs().end(), values.begin(), [&](instruction_ref i) {
                    assert(results.find(i) != results.end());
                    return results[i];
                });
Shucai Xiao's avatar
Shucai Xiao committed
302
303
304
305

            const auto& mod_args = ins->module_inputs();
            auto module_eval     = [&](module_ref smod,
                                   const std::unordered_map<std::string, argument>& inputs) {
Shucai Xiao's avatar
Shucai Xiao committed
306
307
                auto ssctx = ctx;
                return generic_eval(smod, ssctx, inputs, results, make_trace);
Shucai Xiao's avatar
Shucai Xiao committed
308
309
            };

Shucai Xiao's avatar
Shucai Xiao committed
310
311
312
313
            results.emplace(ins, trace(ins, [&] {
                                return ins->normalized_operator().compute(
                                    ctx, ins->get_shape(), values, mod_args, module_eval);
                            }));
Paul's avatar
Paul committed
314
        }
315
        assert(results.find(ins) != results.end());
316
        assert(results.at(ins).get_shape() == ins->get_shape());
Paul's avatar
Paul committed
317
    }
Shucai Xiao's avatar
Shucai Xiao committed
318
    return {results.at(std::prev(mod->end()))};
Paul's avatar
Paul committed
319
320
}

Shucai Xiao's avatar
Shucai Xiao committed
321
322
323
324
template <class F>
std::vector<argument> generic_eval(const program& p,
                                   context& ctx,
                                   std::unordered_map<std::string, argument> params,
325
                                   F make_trace)
Shucai Xiao's avatar
Shucai Xiao committed
326
{
Shucai Xiao's avatar
Shucai Xiao committed
327
    const module* mm = p.get_main_module();
328
    return generic_eval(mm, ctx, params, {}, make_trace);
Shucai Xiao's avatar
Shucai Xiao committed
329
330
}

331
std::vector<argument> program::eval(parameter_map params) const
Paul's avatar
Paul committed
332
{
Paul's avatar
Paul committed
333
334
    auto& ctx = this->impl->ctx;
#ifndef NDEBUG
335
336
337
338
339
340
341
342
343
344
345
    auto with_check_context = [&](auto f) {
        return [=, &ctx](auto&&) {
            auto sctx          = std::make_shared<context>(ctx);
            auto check_context = [=, &ctx](auto g) {
                assert(is_shared(ctx, *sctx));
                auto x = g();
                *sctx  = ctx;
                return x;
            };
            return [=](auto&&... xs) { return f(xs..., check_context); };
        };
Paul's avatar
Paul committed
346
347
    };
#else
348
349
350
351
352
    auto with_check_context = [](auto f) {
        return [=](auto&&) {
            return [=](auto&&... xs) { return f(xs..., [](auto g) { return g(); }); };
        };
    };
Paul's avatar
Paul committed
353
#endif
Paul's avatar
Paul committed
354
355
356
357

    auto trace_level = value_of(MIGRAPHX_TRACE_EVAL{});

    if(trace_level > 0)
Paul's avatar
Paul committed
358
    {
359
360
361
362
363
364
365
366
367
368
369
370
371
372
        return generic_eval(*this,
                            ctx,
                            std::move(params),
                            with_check_context([&](auto& ins, auto f, auto&& check_context) {
                                ctx.finish();
                                std::cout << "Run instruction: ";
                                this->debug_print(ins);
                                timer t{};
                                auto result = check_context(f);
                                double t1   = t.record<milliseconds>();
                                ctx.finish();
                                double t2 = t.record<milliseconds>();
                                std::cout << "Time: " << t1 << "ms, " << t2 << "ms" << std::endl;
                                if(trace_level > 1 and ins->name().front() != '@' and
Shucai Xiao's avatar
Shucai Xiao committed
373
374
                                   ins->name() != "load" and not result.empty())
                                {
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
                                    target tgt  = make_target(this->impl->target_name);
                                    auto buffer = tgt.copy_from(result);
                                    if(trace_level == 2)
                                    {
                                        std::cout << "Output has "
                                                  << to_string_range(classify_argument(buffer))
                                                  << std::endl;
                                        std::cout << "Output: ";
                                        preview_argument(std::cout, buffer);
                                        std::cout << std::endl;
                                    }
                                    else
                                    {
                                        std::cout << "Output: " << buffer << std::endl;
                                    }
Shucai Xiao's avatar
Shucai Xiao committed
390
                                }
391
392
                                return result;
                            }));
Paul's avatar
Paul committed
393
394
395
    }
    else
    {
396
397
398
399
400
401
        return generic_eval(*this,
                            ctx,
                            std::move(params),
                            with_check_context([&](auto&, auto f, auto&& check_context) {
                                return check_context(f);
                            }));
Paul's avatar
Paul committed
402
    }
Paul's avatar
Paul committed
403
404
}

405
const int program_file_version = 5;
406
407
408
409
410
411
412
413

value program::to_value() const
{
    value result;
    result["version"] = program_file_version;
    result["target"]  = this->impl->target_name;
    if(not this->impl->target_name.empty())
        result["context"] = this->impl->ctx.to_value();
Shucai Xiao's avatar
Shucai Xiao committed
414

415
    value module_vals = value::object{};
Shucai Xiao's avatar
Shucai Xiao committed
416
    std::unordered_map<instruction_ref, std::string> names;
417
    for(auto& mod : this->get_modules())
Shucai Xiao's avatar
Shucai Xiao committed
418
    {
Shucai Xiao's avatar
Shucai Xiao committed
419
420
        value mod_val;
        value nodes;
421
422
        mod_val["name"] = mod->name();
        names           = mod->print(
Shucai Xiao's avatar
Shucai Xiao committed
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
            [&](auto ins, auto ins_names) {
                value node;
                node["output"]     = ins_names.at(ins);
                node["name"]       = ins->name();
                node["shape"]      = migraphx::to_value(ins->get_shape());
                node["normalized"] = ins->is_normalized();
                if(ins->name() == "@literal")
                    node["literal"] = migraphx::to_value(ins->get_literal());
                node["operator"] = ins->get_operator().to_value();
                std::vector<std::string> inputs;
                std::transform(ins->inputs().begin(),
                               ins->inputs().end(),
                               std::back_inserter(inputs),
                               [&](auto i) {
                                   assert(contains(ins_names, i));
                                   return ins_names.at(i);
                               });
                node["inputs"]   = inputs;
                auto module_args = ins->module_inputs();
                if(not module_args.empty())
                {
                    std::vector<std::string> module_inputs;
                    std::transform(module_args.begin(),
                                   module_args.end(),
                                   std::back_inserter(module_inputs),
                                   [&](auto mod_ref) { return mod_ref->name(); });
                    node["module_inputs"] = module_inputs;
                }

                nodes.push_back(node);
            },
            names);
        mod_val["nodes"] = nodes;

457
        module_vals[mod->name()] = mod_val;
Shucai Xiao's avatar
Shucai Xiao committed
458
    }
Shucai Xiao's avatar
Shucai Xiao committed
459
460
461

    result["modules"] = module_vals;

462
463
    return result;
}
Shucai Xiao's avatar
Shucai Xiao committed
464

Shucai Xiao's avatar
Shucai Xiao committed
465
466
467
468
469
static void mod_from_val(module_ref mod,
                         const value& v,
                         std::unordered_map<std::string, instruction_ref>& instructions,
                         const std::unordered_map<std::string, module_ref>& map_mods)
{
470
    const auto& module_val = v.at(mod->name());
Shucai Xiao's avatar
Shucai Xiao committed
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
    for(const value& node : module_val.at("nodes"))
    {
        instruction_ref output;
        auto name       = node.at("name").to<std::string>();
        auto fields     = node.at("operator");
        auto normalized = node.at("normalized").to<bool>();

        if(name == "@param")
        {
            output = mod->add_parameter(fields["parameter"].to<std::string>(),
                                        migraphx::from_value<shape>(node.at("shape")));
        }
        else if(name == "@literal")
        {
            output = mod->add_literal(migraphx::from_value<literal>(node.at("literal")));
        }
        else
        {
            auto op = make_op(name, fields);
            std::vector<instruction_ref> inputs;
            std::transform(node.at("inputs").begin(),
                           node.at("inputs").end(),
                           std::back_inserter(inputs),
                           [&](const value& i) {
                               auto i_name = i.to<std::string>();
                               assert(contains(instructions, i_name));
                               return instructions.at(i_name);
                           });

            std::vector<module_ref> module_inputs;
            if(node.contains("module_inputs"))
            {
                std::transform(node.at("module_inputs").begin(),
                               node.at("module_inputs").end(),
                               std::back_inserter(module_inputs),
                               [&](const value& i) { return map_mods.at(i.to<std::string>()); });

                for(auto& smod : module_inputs)
                {
                    mod_from_val(smod, v, instructions, map_mods);
                }
            }

            if(name == "@return")
            {
                output = mod->add_return(inputs);
            }
            else if(module_inputs.empty())
            {
                output = mod->add_instruction(op, inputs);
            }
            else
            {
                output = mod->add_instruction(op, inputs, module_inputs);
            }
        }
        output->set_normalized(normalized);
        instructions[node.at("output").to<std::string>()] = output;
    }
}

532
533
534
535
void program::from_value(const value& v)
{
    auto version = v.at("version").to<int>();
    if(version != program_file_version)
Shucai Xiao's avatar
Shucai Xiao committed
536
537
538
539
    {
        MIGRAPHX_THROW("Warning: Program version mismatch");
    }

540
541
542
543
544
545
546
547
    this->impl->target_name = v.at("target").to<std::string>();
    if(not this->impl->target_name.empty())
    {
        target t        = make_target(this->impl->target_name);
        this->impl->ctx = t.get_context();
        this->impl->ctx.from_value(v.at("context"));
    }

Shucai Xiao's avatar
Shucai Xiao committed
548
549
    auto module_vals = v.at("modules");
    for(const auto& vv : module_vals)
550
    {
551
        const auto& name = vv.get_key();
Shucai Xiao's avatar
Shucai Xiao committed
552
553
        if(name == "main")
            continue;
554
        impl->modules.emplace(name, name);
555
    }
556
    std::unordered_map<std::string, module_ref> map_mods;
Paul's avatar
Paul committed
557
558
559
560
    std::transform(impl->modules.begin(),
                   impl->modules.end(),
                   std::inserter(map_mods, map_mods.end()),
                   [&](auto&& pp) { return std::make_pair(pp.first, &pp.second); });
Shucai Xiao's avatar
Shucai Xiao committed
561
562
563
564
565

    std::unordered_map<std::string, instruction_ref> map_insts;
    auto* mm = get_main_module();
    mod_from_val(mm, module_vals, map_insts, map_mods);

566
567
568
    this->finalize();
}

Paul's avatar
Paul committed
569
570
571
double common_average(const std::vector<double>& v)
{
    std::size_t n = v.size() / 4;
Paul's avatar
Paul committed
572
573
    double total  = std::accumulate(v.begin() + n, v.end() - n, 0.0);
    return total / std::distance(v.begin() + n, v.end() - n);
Paul's avatar
Paul committed
574
575
}

Paul Fultz II's avatar
Paul Fultz II committed
576
577
578
579
580
581
582
583
std::string perf_group(const operation& op)
{
    auto attr = op.attributes();
    if(attr.contains("group"))
        return attr.at("group").to<std::string>();
    return op.name();
}

584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
void program::mark(const parameter_map& params, marker&& m)
{
    auto& ctx = this->impl->ctx;
    // Run once by itself
    eval(params);
    ctx.finish();
    // Start marking
    m.mark_start(*this);
    generic_eval(*this, ctx, params, always([&](auto ins, auto f) {
        argument result;
        m.mark_start(ins);
        result = f();
        m.mark_stop(ins);
        return result;
    }));
    m.mark_stop(*this);
}

602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
static void print_space(std::ostream& os, int n)
{
    for(int i = 0; i < n; ++i)
    {
        os << ' ';
    }
}

using op_flops = std::function<double(const std::vector<shape>& vec_ss)>;

auto get_flops_funcs()
{
    static std::unordered_map<std::string, op_flops> op_funcs;
    op_funcs.emplace("gemm", [&](const std::vector<shape>& vec_ss) {
        assert(vec_ss.size() >= 2);
Shucai Xiao's avatar
Shucai Xiao committed
617
618
619
        auto sa     = vec_ss.front();
        auto sb     = vec_ss.at(1);
        auto batch  = 1;
620
        auto lens_a = sa.lens();
Shucai Xiao's avatar
Shucai Xiao committed
621
622
623
624
        batch =
            std::accumulate(lens_a.rbegin() + 2, lens_a.rend(), 1, std::multiplies<std::size_t>{});
        auto m      = lens_a[lens_a.size() - 2];
        auto k      = lens_a.back();
625
626
627
628
629
630
631
632
633
634
        auto lens_b = sb.lens();
        assert(k == lens_b[lens_b.size() - 2]);
        auto n = lens_b.back();

        return 2.0 * m * n * k * batch;
    });

    return op_funcs;
}

635
636
637
638
void program::perf_report(std::ostream& os,
                          std::size_t n,
                          parameter_map params,
                          std::size_t batch) const
Paul's avatar
Paul committed
639
{
640
    auto& ctx = this->impl->ctx;
Paul's avatar
Paul committed
641
642
    // Run once by itself
    eval(params);
Paul's avatar
Paul committed
643
    ctx.finish();
Paul's avatar
Paul committed
644
    // Run and time entire program
Paul's avatar
Paul committed
645
646
    std::vector<double> total_vec;
    total_vec.reserve(n);
Paul's avatar
Paul committed
647
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
648
    {
Paul's avatar
Paul committed
649
650
651
652
        total_vec.push_back(time<milliseconds>([&] {
            eval(params);
            ctx.finish();
        }));
Paul's avatar
Paul committed
653
    }
Paul's avatar
Paul committed
654
655
    std::sort(total_vec.begin(), total_vec.end());
    std::unordered_map<instruction_ref, std::vector<double>> ins_vec;
Paul's avatar
Paul committed
656
    // Fill the map
657
    generic_eval(*this, ctx, params, always([&](auto ins, auto) {
Paul's avatar
Paul committed
658
        ins_vec[ins].reserve(n);
659
        return argument{ins->get_shape(), nullptr};
660
    }));
661

Paul's avatar
Paul committed
662
    // Run and time each instruction
Paul's avatar
Paul committed
663
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
664
    {
665
        generic_eval(*this, ctx, params, always([&](auto ins, auto f) {
666
            argument result;
Paul's avatar
Paul committed
667
668
669
670
            ins_vec[ins].push_back(time<milliseconds>([&] {
                result = f();
                ctx.finish();
            }));
671
            return result;
672
        }));
Paul's avatar
Paul committed
673
    }
Paul's avatar
Paul committed
674
675
    for(auto&& p : ins_vec)
        std::sort(p.second.begin(), p.second.end());
Paul's avatar
Paul committed
676
    // Run and time implicit overhead
Paul's avatar
Paul committed
677
678
    std::vector<double> overhead_vec;
    overhead_vec.reserve(n);
Paul's avatar
Paul committed
679
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
680
    {
Paul's avatar
Paul committed
681
        overhead_vec.push_back(time<milliseconds>([&] { dry_run(params); }));
Paul's avatar
Paul committed
682
683
    }

Paul's avatar
Paul committed
684
    double total_time             = common_average(total_vec);
Paul's avatar
Paul committed
685
    double rate                   = 1000.0 / total_time;
Paul's avatar
Paul committed
686
    double overhead_time          = common_average(overhead_vec);
Paul's avatar
Paul committed
687
    double overhead_percent       = overhead_time * 100.0 / total_time;
Paul's avatar
Paul committed
688
    double total_instruction_time = 0.0;
Paul's avatar
Paul committed
689
    std::unordered_map<std::string, double> op_times;
Paul's avatar
Paul committed
690
    for(auto&& p : ins_vec)
Paul's avatar
Paul committed
691
692
    {
        double avg = common_average(p.second);
Paul Fultz II's avatar
Paul Fultz II committed
693
        op_times[perf_group(p.first->get_operator())] += avg;
Paul's avatar
Paul committed
694
695
        total_instruction_time += avg;
    }
Paul's avatar
Paul committed
696
697
    double calculate_overhead_time    = total_time - total_instruction_time;
    double calculate_overhead_percent = calculate_overhead_time * 100.0 / total_time;
Paul's avatar
Paul committed
698

Shucai Xiao's avatar
Shucai Xiao committed
699
    std::unordered_map<instruction_ref, std::string> names;
700
701
702
703
704
705

    // count max instruction length
    int max_ins_len = 0;
    this->print(names, [&](auto ins, auto ins_names) {
        std::stringstream ss;
        instruction::print(ss, ins, ins_names);
Shucai Xiao's avatar
Shucai Xiao committed
706
        if(max_ins_len < ss.str().length())
707
708
709
710
711
712
713
714
715
        {
            max_ins_len = ss.str().length();
        }

        // skip return instruction
        if(ins->name() == "@return")
            return;
    });

Shucai Xiao's avatar
Shucai Xiao committed
716
    int print_ins_len = max_ins_len + 1;
Shucai Xiao's avatar
Shucai Xiao committed
717
    std::string str   = "Instructions";
Shucai Xiao's avatar
Shucai Xiao committed
718
719
720
721
722
723
724
    str.append(print_ins_len - str.length(), ' ');
    str.append("\t");
    std::string time_str    = "Time(ms)    \t";
    std::string percent_str = "Percentage  \t";
    std::string size_str    = "(b, m, n, k)              \t";
    std::string flops_str   = "Flops(TFlops/s)";
    os << str << time_str << percent_str << size_str << flops_str << std::endl;
725
726

    auto flops_funcs = get_flops_funcs();
Shucai Xiao's avatar
Shucai Xiao committed
727
    this->print(names, [&](auto ins, auto ins_names) {
728
729
730
        std::stringstream ss;
        instruction::print(ss, ins, ins_names);
        os << ss.str();
731
732
733
734
735

        // skip return instruction
        if(ins->name() == "@return")
            return;

736
        print_space(os, print_ins_len - ss.str().length());
Shucai Xiao's avatar
Shucai Xiao committed
737
        os << "\t";
738

Shucai Xiao's avatar
Shucai Xiao committed
739
        double avg      = common_average(ins_vec[ins]);
740
741
        std::string tms = std::to_string(avg);
        tms.append(time_str.length() - tms.length(), ' ');
Shucai Xiao's avatar
Shucai Xiao committed
742
        tms.append(1, '\t');
Shucai Xiao's avatar
Shucai Xiao committed
743
        double percent   = 100.0 * avg / total_instruction_time;
744
        std::string pers = std::to_string(percent);
Shucai Xiao's avatar
Shucai Xiao committed
745
746
        auto loc         = pers.find('.');
        if(loc != std::string::npos)
747
748
749
750
        {
            pers.erase(pers.begin() + loc + 6, pers.end());
        }
        pers.append(percent_str.length() - pers.length(), ' ');
Shucai Xiao's avatar
Shucai Xiao committed
751
        pers.append(1, '\t');
752
753

        // calculate flops
Shucai Xiao's avatar
Shucai Xiao committed
754
        std::string szs;
755
756
        std::string flps;
        std::string op_name = ins->name();
Shucai Xiao's avatar
Shucai Xiao committed
757
        auto nloc           = op_name.find("::");
758
        op_name.erase(op_name.begin(), op_name.begin() + nloc + 2);
Shucai Xiao's avatar
Shucai Xiao committed
759
        if(contains(flops_funcs, op_name))
760
        {
Shucai Xiao's avatar
Shucai Xiao committed
761
            // print size
Shucai Xiao's avatar
Shucai Xiao committed
762
            auto inss         = to_shapes(ins->inputs());
Shucai Xiao's avatar
Shucai Xiao committed
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
            auto alens = inss.front().lens();
            auto blens = inss.at(1).lens();
            auto mb = std::accumulate(alens.rbegin() + 2, alens.rend(), 1, std::multiplies<std::size_t>{});
            int mm = alens[alens.size() - 2];
            int mk = alens.back();
            int mn = blens.back();

            szs = "{";
            szs.append(std::to_string(mb));
            szs.append(1, ',');
            szs.append(std::to_string(mm));
            szs.append(1, ',');
            szs.append(std::to_string(mk));
            szs.append(1, ',');
            szs.append(std::to_string(mn));
            szs.append("}");
            szs.append(1, '\t');

            auto op_flop_func = flops_funcs.at(op_name);
Shucai Xiao's avatar
Shucai Xiao committed
782
            double flops      = op_flop_func(inss);
783
784
            flops /= avg;
            // convert to GFlops
Shucai Xiao's avatar
Shucai Xiao committed
785
            flops /= 1.0e9;
Shucai Xiao's avatar
Shucai Xiao committed
786
            flps      = std::to_string(flops);
787
            auto floc = flps.find('.');
Shucai Xiao's avatar
Shucai Xiao committed
788
            if(floc != std::string::npos)
789
790
791
792
            {
                flps.erase(flps.begin() + floc + 4, flps.end());
            }
        }
Shucai Xiao's avatar
Shucai Xiao committed
793
794
        szs.append(size_str.length() - szs.length(), ' ');
        os << tms << pers << szs << flps << std::endl;
Paul's avatar
Paul committed
795
    });
Paul's avatar
Paul committed
796
797
798

    os << std::endl;
    os << "Summary:" << std::endl;
799
800
801
802
803
804
805
    std::vector<std::pair<double, std::string>> op_times_sorted;
    std::transform(op_times.begin(),
                   op_times.end(),
                   std::back_inserter(op_times_sorted),
                   [](auto p) { return std::make_pair(p.second, p.first); });
    std::sort(op_times_sorted.begin(), op_times_sorted.end(), std::greater<>{});
    for(auto&& p : op_times_sorted)
Paul's avatar
Paul committed
806
    {
807
808
        auto&& name    = p.second;
        double avg     = p.first;
Paul's avatar
Paul committed
809
810
811
812
813
        double percent = std::ceil(100.0 * avg / total_instruction_time);
        os << name << ": " << avg << "ms, " << percent << "%" << std::endl;
    }

    os << std::endl;
Paul's avatar
Paul committed
814

815
816
    os << "Batch size: " << batch << std::endl;
    os << "Rate: " << rate * batch << "/sec" << std::endl;
Paul's avatar
Paul committed
817
818
    os << "Total time: " << total_time << "ms" << std::endl;
    os << "Total instructions time: " << total_instruction_time << "ms" << std::endl;
Paul's avatar
Paul committed
819
820
821
822
    os << "Overhead time: " << overhead_time << "ms"
       << ", " << calculate_overhead_time << "ms" << std::endl;
    os << "Overhead: " << std::round(overhead_percent) << "%"
       << ", " << std::round(calculate_overhead_percent) << "%" << std::endl;
Paul's avatar
Paul committed
823
824
}

Paul's avatar
Paul committed
825
826
void program::debug_print() const { std::cout << *this << std::endl; }
void program::debug_print(instruction_ref ins) const
Paul's avatar
Paul committed
827
{
Shucai Xiao's avatar
Shucai Xiao committed
828
    std::unordered_map<instruction_ref, std::string> names;
829
    if(std::any_of(this->impl->modules.begin(), this->impl->modules.end(), [&](const auto& pp) {
830
           return is_end(pp.second.end(), ins);
Shucai Xiao's avatar
Shucai Xiao committed
831
       }))
Paul's avatar
Paul committed
832
833
834
835
    {
        std::cout << "End instruction" << std::endl;
        return;
    }
836
837
    else if(std::none_of(this->impl->modules.begin(),
                         this->impl->modules.end(),
838
                         [&](const auto& pp) { return pp.second.has_instruction(ins); }))
Paul's avatar
Paul committed
839
840
841
842
    {
        std::cout << "Instruction not part of program" << std::endl;
        return;
    }
Shucai Xiao's avatar
Shucai Xiao committed
843

Paul's avatar
Paul committed
844
    std::stringstream ss;
Shucai Xiao's avatar
Shucai Xiao committed
845
    this->print(names, [&](auto x, auto ins_names) {
Paul's avatar
Paul committed
846
        if(x == ins)
Paul's avatar
Paul committed
847
        {
Shucai Xiao's avatar
Shucai Xiao committed
848
            instruction::print(std::cout, x, ins_names);
Paul's avatar
Paul committed
849
850
851
852
853
            std::cout << std::endl;
        }
    });
}

Shucai Xiao's avatar
Shucai Xiao committed
854
855
856
857
void program::print(
    std::unordered_map<instruction_ref, std::string>& names,
    const std::function<void(instruction_ref, std::unordered_map<instruction_ref, std::string>)>&
        print_func) const
858
{
859
    for(const auto& pp : this->impl->modules)
860
    {
861
        names = pp.second.print(print_func, names);
862
863
864
    }
}

Shucai Xiao's avatar
Shucai Xiao committed
865
void program::print_graph(std::ostream& os, bool brief) const
866
{
Shucai Xiao's avatar
Shucai Xiao committed
867
868
    const auto* mm = this->get_main_module();
    mm->print_graph(os, brief);
869
870
871
872
}

void program::print_cpp(std::ostream& os) const
{
Shucai Xiao's avatar
Shucai Xiao committed
873
874
875
876
877
878
879
880
    auto vec_modules = this->get_modules();
    std::unordered_map<instruction_ref, std::string> names;
    for(auto& mod : vec_modules)
    {
        os << "module: \"" << mod->name() << "\"" << std::endl;
        names = mod->print_cpp(os, names);
        os << std::endl;
    }
881
882
}

Paul's avatar
Paul committed
883
884
void program::dry_run(std::unordered_map<std::string, argument> params) const
{
Paul's avatar
Paul committed
885
    auto& ctx = this->impl->ctx;
886
887
888
    generic_eval(*this, ctx, std::move(params), always([](auto ins, auto&&...) {
        return argument{ins->get_shape(), nullptr};
    }));
Paul's avatar
Paul committed
889
890
}

Shucai Xiao's avatar
Shucai Xiao committed
891
void program::annotate(std::ostream& os, const std::function<void(instruction_ref)>& a) const
Paul's avatar
Paul committed
892
{
893
    for(auto& pp : this->impl->modules)
Shucai Xiao's avatar
Shucai Xiao committed
894
    {
895
896
        std::cout << pp.first << ":" << std::endl;
        pp.second.annotate(os, a);
Shucai Xiao's avatar
Shucai Xiao committed
897
    }
Paul's avatar
Paul committed
898
899
}

Paul's avatar
Paul committed
900
const module* program::get_module(const std::string& name) const { return &impl->modules.at(name); }
Shucai Xiao's avatar
Shucai Xiao committed
901
902
903

module* program::create_module(const std::string& name)
{
904
    assert(not contains(impl->modules, name));
905
906
    auto r = impl->modules.emplace(name, name);
    return &(r.first->second);
Shucai Xiao's avatar
Shucai Xiao committed
907
908
}

Paul's avatar
Paul committed
909
module* program::get_module(const std::string& name) { return &impl->modules.at(name); }
Shucai Xiao's avatar
Shucai Xiao committed
910
911
912
913
914

module* program::get_main_module() { return get_module("main"); }

const module* program::get_main_module() const { return get_module("main"); }

Paul's avatar
Paul committed
915
template <class T>
916
std::vector<T*> generic_get_modules(T* mm)
Shucai Xiao's avatar
Shucai Xiao committed
917
{
918
    std::vector<T*> vec_modules;
Shucai Xiao's avatar
Shucai Xiao committed
919
920
921
922
923
    vec_modules.push_back(mm);
    auto sub_modules = mm->get_sub_modules();
    vec_modules.insert(vec_modules.end(), sub_modules.begin(), sub_modules.end());
    return vec_modules;
}
Shucai Xiao's avatar
Shucai Xiao committed
924

Paul's avatar
Paul committed
925
template <class Map, class T, class OutputIterator>
926
void generic_get_unused_modules(Map& m, const std::vector<T*>& mods, OutputIterator out)
Shucai Xiao's avatar
Shucai Xiao committed
927
{
928
929
930
931
    std::unordered_set<std::string> used;
    std::transform(mods.begin(), mods.end(), std::inserter(used, used.end()), [](auto&& mod) {
        return mod->name();
    });
Paul's avatar
Paul committed
932
933
934
935
936
    transform_if(m.begin(),
                 m.end(),
                 out,
                 [&](auto&& pp) { return not contains(used, pp.first); },
                 [](auto&& pp) { return &pp.second; });
937
}
Shucai Xiao's avatar
Shucai Xiao committed
938

939
940
941
942
943
944
945
946
947
948
949
950
std::vector<const module*> program::get_modules() const
{
    auto result = generic_get_modules(this->get_main_module());
    generic_get_unused_modules(impl->modules, result, std::back_inserter(result));
    return result;
}

std::vector<module*> program::get_modules()
{
    auto result = generic_get_modules(this->get_main_module());
    generic_get_unused_modules(impl->modules, result, std::back_inserter(result));
    return result;
Shucai Xiao's avatar
Shucai Xiao committed
951
952
}

953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
template <class Map, class T>
bool is_unused_module(Map& m, const std::vector<T*>& mods, const std::string& name)
{
    bool is_unused = false;
    generic_get_unused_modules(m, mods, make_function_output_iterator([&](auto* mod) {
                                   if(mod->name() == name)
                                       is_unused = true;
                               }));
    return is_unused;
}

template <class Map>
bool references_instruction(Map& m, const instruction& ins, const std::string& name)
{
    return std::any_of(m.begin(), m.end(), [&](auto&& p) {
        if(p.first == name)
            return false;
        return std::any_of(p.second.begin(), p.second.end(), [&](auto&& i) {
            return std::any_of(i.inputs().begin(), i.inputs().end(), [&](auto&& j) {
                return std::addressof(*j) == std::addressof(ins);
            });
        });
    });
}

void program::remove_module(const std::string& name)
{
    // cppcheck-suppress assertWithSideEffect
    assert(is_unused_module(impl->modules, generic_get_modules(this->get_main_module()), name) &&
           "Module used in program");
    assert(std::none_of(
               impl->modules.at(name).begin(),
               impl->modules.at(name).end(),
               [&](auto&& ins) { return references_instruction(impl->modules, ins, name); }) &&
           "Instruction referenced in another module");
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003

    // if an instruction has an input out side of the current module, need to remove
    // the instruction from its input's outputs
    auto& mod = impl->modules.at(name);
    for(auto ins : iterator_for(mod))
    {
        auto inputs = ins->inputs();
        for(auto in : inputs)
        {
            if(not mod.has_instruction(in))
            {
                in->remove_output(ins);
            }
        }
    }

1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
    impl->modules.erase(name);
}

void program::remove_unused_modules()
{
    std::vector<module*> unused;
    generic_get_unused_modules(
        impl->modules, generic_get_modules(this->get_main_module()), std::back_inserter(unused));
    for(auto* m : unused)
        this->remove_module(m->name());
}

1016
1017
program& program::sort()
{
1018
    for(auto& pp : this->impl->modules)
Shucai Xiao's avatar
Shucai Xiao committed
1019
    {
1020
        pp.second.sort();
Shucai Xiao's avatar
Shucai Xiao committed
1021
1022
    }

1023
1024
1025
    return *this;
}

Paul's avatar
Paul committed
1026
bool operator==(const program& x, const program& y) { return to_string(x) == to_string(y); }
Paul's avatar
Paul committed
1027

Paul's avatar
Paul committed
1028
std::ostream& operator<<(std::ostream& os, const program& p)
Paul's avatar
Paul committed
1029
{
Shucai Xiao's avatar
Shucai Xiao committed
1030
1031
1032
    auto vec_modules = p.get_modules();
    std::unordered_map<instruction_ref, std::string> names;
    for(auto& mod : vec_modules)
Shucai Xiao's avatar
Shucai Xiao committed
1033
    {
Shucai Xiao's avatar
Shucai Xiao committed
1034
1035
1036
1037
1038
1039
1040
        os << "module: \"" << mod->name() << "\"" << std::endl;
        names = mod->print(
            [&](auto ins, auto ins_names) {
                instruction::print(os, ins, ins_names);
                os << std::endl;
            },
            names);
1041
        os << std::endl;
Shucai Xiao's avatar
Shucai Xiao committed
1042
1043
    }

Paul's avatar
Paul committed
1044
    return os;
Paul's avatar
Paul committed
1045
}
Paul's avatar
Paul committed
1046

Paul's avatar
Paul committed
1047
} // namespace MIGRAPHX_INLINE_NS
Paul's avatar
Paul committed
1048
} // namespace migraphx