program.cpp 32 KB
Newer Older
Paul's avatar
Paul committed
1
2
3
#include <migraphx/program.hpp>
#include <migraphx/stringutils.hpp>
#include <migraphx/instruction.hpp>
4
#include <migraphx/op/identity.hpp>
Paul's avatar
Paul committed
5
#include <migraphx/target.hpp>
Paul's avatar
Paul committed
6
7
8
#include <migraphx/env.hpp>
#include <migraphx/ranges.hpp>
#include <migraphx/time.hpp>
9
#include <migraphx/pass_manager.hpp>
10
#include <migraphx/register_target.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
11
#include <migraphx/iterator_for.hpp>
12
#include <migraphx/iterator.hpp>
13
#include <migraphx/algorithm.hpp>
14
#include <migraphx/output_iterator.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
15
#include <migraphx/make_op.hpp>
16
#include <migraphx/marker.hpp>
Paul's avatar
Paul committed
17
#include <iostream>
Paul's avatar
Paul committed
18
#include <sstream>
Paul's avatar
Paul committed
19
#include <algorithm>
20
#include <set>
Paul's avatar
Paul committed
21
#include <utility>
22

23
#include <unordered_set>
Shucai Xiao's avatar
Shucai Xiao committed
24
25
#include <map>
#include <cassert>
Paul's avatar
Paul committed
26

Paul's avatar
Paul committed
27
namespace migraphx {
Paul's avatar
Paul committed
28
inline namespace MIGRAPHX_INLINE_NS {
Paul's avatar
Paul committed
29

30
31
using milliseconds = std::chrono::duration<double, std::milli>;

Paul's avatar
Paul committed
32
33
struct program_impl
{
Shucai Xiao's avatar
Shucai Xiao committed
34
    // A map is used to keep references to modules of the program
35
    std::unordered_map<std::string, module> modules;
Paul's avatar
Paul committed
36
    context ctx;
37
    std::string target_name;
Paul's avatar
Paul committed
38
39
};

40
program::program() : impl(std::make_unique<program_impl>()) { this->create_module("main"); }
Paul's avatar
Paul committed
41

Paul's avatar
Paul committed
42
program::program(program&&) noexcept = default;
Shucai Xiao's avatar
Shucai Xiao committed
43
program::~program() noexcept         = default;
Paul's avatar
Paul committed
44

45
// copy constructor
Shucai Xiao's avatar
Shucai Xiao committed
46
program::program(const program& p) { assign(p); }
47
48

// copy assignment operator
Shucai Xiao's avatar
Shucai Xiao committed
49
program& program::operator=(program p)
50
{
Shucai Xiao's avatar
Shucai Xiao committed
51
    std::swap(p.impl, this->impl);
52
53
54
    return *this;
}

Shucai Xiao's avatar
Shucai Xiao committed
55
void program::assign(const program& p)
56
{
Shucai Xiao's avatar
Shucai Xiao committed
57
    if(!impl)
58
59
60
    {
        impl = std::make_unique<program_impl>();
    }
Shucai Xiao's avatar
Shucai Xiao committed
61
    else if(!impl->modules.empty())
62
    {
Shucai Xiao's avatar
Shucai Xiao committed
63
        impl->modules.clear();
64
    }
Shucai Xiao's avatar
Shucai Xiao committed
65

66
    impl->ctx         = p.impl->ctx;
Shucai Xiao's avatar
Shucai Xiao committed
67
68
    impl->target_name = p.impl->target_name;
    impl->modules     = p.impl->modules;
Shucai Xiao's avatar
Shucai Xiao committed
69
70
71
72

    // build a map from old ins to new ins
    // Build a map from old module to new module
    std::unordered_map<module_ref, module_ref> mod_map;
Paul's avatar
Paul committed
73
74
75
76
77
    std::transform(
        impl->modules.begin(),
        impl->modules.end(),
        std::inserter(mod_map, mod_map.begin()),
        [&](auto&& xp) { return std::make_pair(&p.impl->modules.at(xp.first), &xp.second); });
Shucai Xiao's avatar
Shucai Xiao committed
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

    std::unordered_map<instruction_ref, instruction_ref> ins_map;
    for(auto&& pp : mod_map)
    {
        auto old_ins = iterator_for(*pp.first);
        auto new_ins = iterator_for(*pp.second);
        std::transform(old_ins.begin(),
                       old_ins.end(),
                       new_ins.begin(),
                       std::inserter(ins_map, ins_map.begin()),
                       [](auto x, auto y) { return std::make_pair(x, y); });
    }

    // Update all references from all modules
    for(auto&& mp : impl->modules)
    {
94
        for(auto ins : iterator_for(mp.second))
Shucai Xiao's avatar
Shucai Xiao committed
95
96
            instruction::replace_refs(ins, ins_map, mod_map);
    }
97
98
}

Paul's avatar
Paul committed
99
shape program::get_parameter_shape(std::string name) const
Paul's avatar
Paul committed
100
{
Shucai Xiao's avatar
Shucai Xiao committed
101
102
    const auto* mm = this->get_main_module();
    return mm->get_parameter_shape(std::move(name));
Paul's avatar
Paul committed
103
104
}

105
106
std::vector<std::string> program::get_parameter_names() const
{
Shucai Xiao's avatar
Shucai Xiao committed
107
108
    const auto* mm = this->get_main_module();
    return mm->get_parameter_names();
109
110
}

mei-ye's avatar
mei-ye committed
111
112
instruction_ref program::get_parameter(std::string name) const
{
Shucai Xiao's avatar
Shucai Xiao committed
113
114
    const auto* mm = this->get_main_module();
    return mm->get_parameter(std::move(name));
mei-ye's avatar
mei-ye committed
115
116
}

Paul's avatar
Paul committed
117
118
std::unordered_map<std::string, shape> program::get_parameter_shapes() const
{
Shucai Xiao's avatar
Shucai Xiao committed
119
120
    const auto* mm = this->get_main_module();
    return mm->get_parameter_shapes();
Paul's avatar
Paul committed
121
122
}

Shucai Xiao's avatar
Shucai Xiao committed
123
std::size_t program::size() const { return impl->modules.size(); }
124

125
126
std::vector<shape> program::get_output_shapes() const
{
Shucai Xiao's avatar
Shucai Xiao committed
127
128
    const auto* mm = this->get_main_module();
    return mm->get_output_shapes();
129
}
Paul's avatar
Paul committed
130

Paul's avatar
Paul committed
131
132
context& program::get_context() const { return impl->ctx; }

Paul's avatar
Paul committed
133
134
instruction_ref program::validate() const
{
Shucai Xiao's avatar
Shucai Xiao committed
135
136
    const auto* mm = this->get_main_module();
    return mm->validate();
Paul's avatar
Paul committed
137
138
}

139
140
bool program::is_compiled() const { return not this->impl->target_name.empty(); }

141
void program::compile(const target& t, compile_options options)
Paul's avatar
Paul committed
142
{
143
144
145
    assert(not this->is_compiled());
    this->impl->target_name = t.name();
    this->impl->ctx         = t.get_context();
Paul's avatar
Paul committed
146
    if(enabled(MIGRAPHX_TRACE_COMPILE{}))
147
        options.trace = tracer{std::cout};
Shucai Xiao's avatar
Shucai Xiao committed
148

149
150
    options.trace(*this);
    options.trace();
Shucai Xiao's avatar
Shucai Xiao committed
151

Shucai Xiao's avatar
Shucai Xiao committed
152
    auto&& passes = t.get_passes(this->impl->ctx, options);
153
154
155
    run_passes(*this, passes, options.trace);

    auto mods = this->get_modules();
Shucai Xiao's avatar
Shucai Xiao committed
156

157
158
    // Validate and finalize
    for(const auto& mod : reverse(mods))
Paul's avatar
Paul committed
159
    {
Shucai Xiao's avatar
Shucai Xiao committed
160
161
162
163
164
165
        auto invalid = mod->validate();
        if(invalid != mod->end())
        {
            MIGRAPHX_THROW("Invalid module " + mod->name() + " from compilation at instruction " +
                           std::to_string(std::distance(mod->begin(), invalid)));
        }
166
167
168
169
170
171
172
        auto dangling = mod->find_dangling_reference();
        if(dangling != mod->end())
        {
            auto index = std::distance(mod->begin(), dangling);
            MIGRAPHX_THROW("Dangling reference in module " + mod->name() + " from instruction " +
                           std::to_string(index));
        }
Shucai Xiao's avatar
Shucai Xiao committed
173
        mod->finalize(this->impl->ctx);
Paul's avatar
Paul committed
174
    }
Paul's avatar
Paul committed
175
176
177
178
}

void program::finalize()
{
Shucai Xiao's avatar
Shucai Xiao committed
179
180
    auto* mm = this->get_main_module();
    mm->finalize(this->impl->ctx);
Paul's avatar
Paul committed
181
182
}

183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
template <class T>
std::string classify(T x)
{
    switch(std::fpclassify(x))
    {
    case FP_INFINITE: return "inf";
    case FP_NAN: return "nan";
    case FP_NORMAL: return "normal";
    case FP_SUBNORMAL: return "subnormal";
    case FP_ZERO: return "zero";
    default: return "unknown";
    }
}

std::unordered_set<std::string> classify_argument(const argument& a)
{
    std::unordered_set<std::string> result;
    a.visit(
        [&](auto t) {
            for(const auto& x : t)
                result.insert(classify(x));
        },
        [&](const auto& xs) {
            for(const auto& x : xs)
            {
                auto r = classify_argument(x);
                result.insert(r.begin(), r.end());
            }
        });
    return result;
}

void preview_argument(std::ostream& os, const argument& a)
{
    a.visit(
        [&](auto t) {
            if(t.size() <= 10)
            {
                os << t;
            }
            else
            {
                os << to_string_range(t.begin(), t.begin() + 5);
                os << ", ..., ";
                os << to_string_range(t.end() - 5, t.end());
            }
        },
        [&](const auto& xs) {
            for(const auto& x : xs)
            {
                os << '{';
                preview_argument(os, x);
                os << '}';
            }
        });
}

Paul's avatar
Paul committed
240
template <class F>
Shucai Xiao's avatar
Shucai Xiao committed
241
std::vector<argument> generic_eval(const module* mod,
242
243
                                   context& ctx,
                                   std::unordered_map<std::string, argument> params,
Shucai Xiao's avatar
Shucai Xiao committed
244
                                   std::unordered_map<instruction_ref, argument> results,
245
                                   F make_trace)
Paul's avatar
Paul committed
246
{
Shucai Xiao's avatar
Shucai Xiao committed
247
248
    assert(mod->validate() == mod->end());
    results.reserve(mod->size() * 2);
Paul's avatar
Paul committed
249
250
    std::vector<argument> values;
    values.reserve(16);
251
    auto trace = make_trace(mod);
Shucai Xiao's avatar
Shucai Xiao committed
252
    for(auto ins : iterator_for(*mod))
Paul's avatar
Paul committed
253
    {
254
        assert(results.find(ins) == results.end());
255
256
        const auto& name = ins->name();
        if(name == "@literal")
Paul's avatar
Paul committed
257
        {
Paul's avatar
Paul committed
258
            results.emplace(ins, trace(ins, [&] { return ins->get_literal().get_argument(); }));
Paul's avatar
Paul committed
259
        }
260
        else if(name == "@param")
Paul's avatar
Paul committed
261
        {
Paul's avatar
Paul committed
262
263
264
265
266
            results.emplace(
                ins, trace(ins, [&] {
                    auto param_name = any_cast<builtin::param>(ins->get_operator()).parameter;
                    if(not contains(params, param_name))
                        MIGRAPHX_THROW("Parameter not found: " + param_name);
charlie's avatar
charlie committed
267

268
                    auto param = params[param_name];
charlie's avatar
charlie committed
269
270
                    // TODO: may want to check correct number of dimensions and/or was within bounds
                    if(not ins->get_shape().dynamic() and param.get_shape() != ins->get_shape())
Paul's avatar
Paul committed
271
272
273
274
                        MIGRAPHX_THROW("Incorrect shape {" + to_string(param.get_shape()) +
                                       "} for parameter: " + param_name);
                    return param;
                }));
Paul's avatar
Paul committed
275
        }
276
        else if(name == "@outline")
Paul's avatar
Paul committed
277
        {
Paul's avatar
Paul committed
278
            results.emplace(ins, trace(ins, [&] { return argument{ins->get_shape(), nullptr}; }));
Paul's avatar
Paul committed
279
        }
280
281
282
283
284
285
286
287
288
289
290
291
292
        else if(name == "@return")
        {
            std::vector<argument> prog_outputs;
            std::transform(ins->inputs().begin(),
                           ins->inputs().end(),
                           std::back_inserter(prog_outputs),
                           [&](instruction_ref i) {
                               assert(results.find(i) != results.end());
                               return results[i];
                           });

            return prog_outputs;
        }
Paul's avatar
Paul committed
293
294
        else
        {
Paul's avatar
Paul committed
295
            values.resize(ins->inputs().size());
Paul's avatar
Paul committed
296
297
298
299
300
            std::transform(
                ins->inputs().begin(), ins->inputs().end(), values.begin(), [&](instruction_ref i) {
                    assert(results.find(i) != results.end());
                    return results[i];
                });
Shucai Xiao's avatar
Shucai Xiao committed
301

charlie's avatar
charlie committed
302
303
304
305
306
307
308
309
310
311
312
313
            shape output_shape;
            auto ins_shape = ins->get_shape();
            if(ins_shape.dynamic())
            {
                // Make into a std::vector<instruction_ref> of inputs
                auto to_shapes = [](std::vector<argument> args) {
                    std::vector<shape> shapes(args.size());
                    std::transform(args.begin(), args.end(), shapes.begin(), [](argument i) {
                        return i.get_shape();
                    });
                    return shapes;
                };
charlie's avatar
charlie committed
314
                // TODO: Consider how this will be handled when memoized.
charlie's avatar
charlie committed
315
316
317
                // Could memoize these output shapes into a map so not recalculating
                // TODO: Issue with incompatible input tensor to kernel and needing to set
                // padding/strides
charlie's avatar
charlie committed
318
319
320
321
322
323
324
                output_shape = ins->get_operator().compute_shape(to_shapes(values));
            }
            else
            {
                output_shape = ins_shape;
            }

Shucai Xiao's avatar
Shucai Xiao committed
325
326
327
            const auto& mod_args = ins->module_inputs();
            auto module_eval     = [&](module_ref smod,
                                   const std::unordered_map<std::string, argument>& inputs) {
Shucai Xiao's avatar
Shucai Xiao committed
328
329
                auto ssctx = ctx;
                return generic_eval(smod, ssctx, inputs, results, make_trace);
Shucai Xiao's avatar
Shucai Xiao committed
330
331
            };

Shucai Xiao's avatar
Shucai Xiao committed
332
333
            results.emplace(ins, trace(ins, [&] {
                                return ins->normalized_operator().compute(
charlie's avatar
charlie committed
334
                                    ctx, output_shape, values, mod_args, module_eval);
Shucai Xiao's avatar
Shucai Xiao committed
335
                            }));
Paul's avatar
Paul committed
336
        }
337
        assert(results.find(ins) != results.end());
charlie's avatar
charlie committed
338
339
340
341
        if(not ins->get_shape().dynamic())
        {
            assert(results.at(ins).get_shape() == ins->get_shape());
        }
Paul's avatar
Paul committed
342
    }
Shucai Xiao's avatar
Shucai Xiao committed
343
    return {results.at(std::prev(mod->end()))};
Paul's avatar
Paul committed
344
345
}

Shucai Xiao's avatar
Shucai Xiao committed
346
347
348
349
template <class F>
std::vector<argument> generic_eval(const program& p,
                                   context& ctx,
                                   std::unordered_map<std::string, argument> params,
350
                                   F make_trace)
Shucai Xiao's avatar
Shucai Xiao committed
351
{
Shucai Xiao's avatar
Shucai Xiao committed
352
    const module* mm = p.get_main_module();
353
    return generic_eval(mm, ctx, params, {}, make_trace);
Shucai Xiao's avatar
Shucai Xiao committed
354
355
}

356
std::vector<argument> program::eval(parameter_map params) const
Paul's avatar
Paul committed
357
{
Paul's avatar
Paul committed
358
359
    auto& ctx = this->impl->ctx;
#ifndef NDEBUG
360
361
362
363
364
365
366
367
368
369
370
    auto with_check_context = [&](auto f) {
        return [=, &ctx](auto&&) {
            auto sctx          = std::make_shared<context>(ctx);
            auto check_context = [=, &ctx](auto g) {
                assert(is_shared(ctx, *sctx));
                auto x = g();
                *sctx  = ctx;
                return x;
            };
            return [=](auto&&... xs) { return f(xs..., check_context); };
        };
Paul's avatar
Paul committed
371
372
    };
#else
373
374
375
376
377
    auto with_check_context = [](auto f) {
        return [=](auto&&) {
            return [=](auto&&... xs) { return f(xs..., [](auto g) { return g(); }); };
        };
    };
Paul's avatar
Paul committed
378
#endif
Paul's avatar
Paul committed
379
380
381
382

    auto trace_level = value_of(MIGRAPHX_TRACE_EVAL{});

    if(trace_level > 0)
Paul's avatar
Paul committed
383
    {
Shucai Xiao's avatar
Shucai Xiao committed
384
385
386
387
388
389
390
391
        std::unordered_map<instruction_ref, std::string> ins_out;
        // get instruction names
        this->print([&](auto x, auto ins_names) {
            std::stringstream ss;
            instruction::print(ss, x, ins_names);
            ins_out[x] = ss.str();
        });

392
393
394
395
396
        return generic_eval(*this,
                            ctx,
                            std::move(params),
                            with_check_context([&](auto& ins, auto f, auto&& check_context) {
                                ctx.finish();
Shucai Xiao's avatar
Shucai Xiao committed
397
                                std::cout << "Run instruction: " << ins_out.at(ins) << std::endl;
398
399
400
401
402
403
404
                                timer t{};
                                auto result = check_context(f);
                                double t1   = t.record<milliseconds>();
                                ctx.finish();
                                double t2 = t.record<milliseconds>();
                                std::cout << "Time: " << t1 << "ms, " << t2 << "ms" << std::endl;
                                if(trace_level > 1 and ins->name().front() != '@' and
Shucai Xiao's avatar
Shucai Xiao committed
405
406
                                   ins->name() != "load" and not result.empty())
                                {
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
                                    target tgt  = make_target(this->impl->target_name);
                                    auto buffer = tgt.copy_from(result);
                                    if(trace_level == 2)
                                    {
                                        std::cout << "Output has "
                                                  << to_string_range(classify_argument(buffer))
                                                  << std::endl;
                                        std::cout << "Output: ";
                                        preview_argument(std::cout, buffer);
                                        std::cout << std::endl;
                                    }
                                    else
                                    {
                                        std::cout << "Output: " << buffer << std::endl;
                                    }
Shucai Xiao's avatar
Shucai Xiao committed
422
                                }
423
424
                                return result;
                            }));
Paul's avatar
Paul committed
425
426
427
    }
    else
    {
428
429
430
431
432
433
        return generic_eval(*this,
                            ctx,
                            std::move(params),
                            with_check_context([&](auto&, auto f, auto&& check_context) {
                                return check_context(f);
                            }));
Paul's avatar
Paul committed
434
    }
Paul's avatar
Paul committed
435
436
}

437
const int program_file_version = 5;
438
439
440
441
442
443
444
445

value program::to_value() const
{
    value result;
    result["version"] = program_file_version;
    result["target"]  = this->impl->target_name;
    if(not this->impl->target_name.empty())
        result["context"] = this->impl->ctx.to_value();
Shucai Xiao's avatar
Shucai Xiao committed
446

447
    value module_vals = value::object{};
Shucai Xiao's avatar
Shucai Xiao committed
448
    std::unordered_map<instruction_ref, std::string> names;
449
    for(auto& mod : this->get_modules())
Shucai Xiao's avatar
Shucai Xiao committed
450
    {
Shucai Xiao's avatar
Shucai Xiao committed
451
452
        value mod_val;
        value nodes;
453
454
        mod_val["name"] = mod->name();
        names           = mod->print(
Shucai Xiao's avatar
Shucai Xiao committed
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
            [&](auto ins, auto ins_names) {
                value node;
                node["output"]     = ins_names.at(ins);
                node["name"]       = ins->name();
                node["shape"]      = migraphx::to_value(ins->get_shape());
                node["normalized"] = ins->is_normalized();
                if(ins->name() == "@literal")
                    node["literal"] = migraphx::to_value(ins->get_literal());
                node["operator"] = ins->get_operator().to_value();
                std::vector<std::string> inputs;
                std::transform(ins->inputs().begin(),
                               ins->inputs().end(),
                               std::back_inserter(inputs),
                               [&](auto i) {
                                   assert(contains(ins_names, i));
                                   return ins_names.at(i);
                               });
                node["inputs"]   = inputs;
                auto module_args = ins->module_inputs();
                if(not module_args.empty())
                {
                    std::vector<std::string> module_inputs;
                    std::transform(module_args.begin(),
                                   module_args.end(),
                                   std::back_inserter(module_inputs),
                                   [&](auto mod_ref) { return mod_ref->name(); });
                    node["module_inputs"] = module_inputs;
                }

                nodes.push_back(node);
            },
            names);
        mod_val["nodes"] = nodes;

489
        module_vals[mod->name()] = mod_val;
Shucai Xiao's avatar
Shucai Xiao committed
490
    }
Shucai Xiao's avatar
Shucai Xiao committed
491
492
493

    result["modules"] = module_vals;

494
495
    return result;
}
Shucai Xiao's avatar
Shucai Xiao committed
496

Shucai Xiao's avatar
Shucai Xiao committed
497
498
499
500
501
static void mod_from_val(module_ref mod,
                         const value& v,
                         std::unordered_map<std::string, instruction_ref>& instructions,
                         const std::unordered_map<std::string, module_ref>& map_mods)
{
502
    const auto& module_val = v.at(mod->name());
Shucai Xiao's avatar
Shucai Xiao committed
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
    for(const value& node : module_val.at("nodes"))
    {
        instruction_ref output;
        auto name       = node.at("name").to<std::string>();
        auto fields     = node.at("operator");
        auto normalized = node.at("normalized").to<bool>();

        if(name == "@param")
        {
            output = mod->add_parameter(fields["parameter"].to<std::string>(),
                                        migraphx::from_value<shape>(node.at("shape")));
        }
        else if(name == "@literal")
        {
            output = mod->add_literal(migraphx::from_value<literal>(node.at("literal")));
        }
        else
        {
            auto op = make_op(name, fields);
            std::vector<instruction_ref> inputs;
            std::transform(node.at("inputs").begin(),
                           node.at("inputs").end(),
                           std::back_inserter(inputs),
                           [&](const value& i) {
                               auto i_name = i.to<std::string>();
                               assert(contains(instructions, i_name));
                               return instructions.at(i_name);
                           });

            std::vector<module_ref> module_inputs;
            if(node.contains("module_inputs"))
            {
                std::transform(node.at("module_inputs").begin(),
                               node.at("module_inputs").end(),
                               std::back_inserter(module_inputs),
                               [&](const value& i) { return map_mods.at(i.to<std::string>()); });

                for(auto& smod : module_inputs)
                {
                    mod_from_val(smod, v, instructions, map_mods);
                }
            }

            if(name == "@return")
            {
                output = mod->add_return(inputs);
            }
            else if(module_inputs.empty())
            {
                output = mod->add_instruction(op, inputs);
            }
            else
            {
                output = mod->add_instruction(op, inputs, module_inputs);
            }
        }
        output->set_normalized(normalized);
        instructions[node.at("output").to<std::string>()] = output;
    }
}

564
565
566
567
void program::from_value(const value& v)
{
    auto version = v.at("version").to<int>();
    if(version != program_file_version)
Shucai Xiao's avatar
Shucai Xiao committed
568
569
570
571
    {
        MIGRAPHX_THROW("Warning: Program version mismatch");
    }

572
573
574
575
576
577
578
579
    this->impl->target_name = v.at("target").to<std::string>();
    if(not this->impl->target_name.empty())
    {
        target t        = make_target(this->impl->target_name);
        this->impl->ctx = t.get_context();
        this->impl->ctx.from_value(v.at("context"));
    }

Shucai Xiao's avatar
Shucai Xiao committed
580
581
    auto module_vals = v.at("modules");
    for(const auto& vv : module_vals)
582
    {
583
        const auto& name = vv.get_key();
Shucai Xiao's avatar
Shucai Xiao committed
584
585
        if(name == "main")
            continue;
586
        impl->modules.emplace(name, name);
587
    }
588
    std::unordered_map<std::string, module_ref> map_mods;
Paul's avatar
Paul committed
589
590
591
592
    std::transform(impl->modules.begin(),
                   impl->modules.end(),
                   std::inserter(map_mods, map_mods.end()),
                   [&](auto&& pp) { return std::make_pair(pp.first, &pp.second); });
Shucai Xiao's avatar
Shucai Xiao committed
593
594
595
596
597

    std::unordered_map<std::string, instruction_ref> map_insts;
    auto* mm = get_main_module();
    mod_from_val(mm, module_vals, map_insts, map_mods);

598
599
600
    this->finalize();
}

Paul's avatar
Paul committed
601
602
603
double common_average(const std::vector<double>& v)
{
    std::size_t n = v.size() / 4;
Paul's avatar
Paul committed
604
605
    double total  = std::accumulate(v.begin() + n, v.end() - n, 0.0);
    return total / std::distance(v.begin() + n, v.end() - n);
Paul's avatar
Paul committed
606
607
}

Paul Fultz II's avatar
Paul Fultz II committed
608
609
610
611
612
613
614
615
std::string perf_group(const operation& op)
{
    auto attr = op.attributes();
    if(attr.contains("group"))
        return attr.at("group").to<std::string>();
    return op.name();
}

616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
void program::mark(const parameter_map& params, marker&& m)
{
    auto& ctx = this->impl->ctx;
    // Run once by itself
    eval(params);
    ctx.finish();
    // Start marking
    m.mark_start(*this);
    generic_eval(*this, ctx, params, always([&](auto ins, auto f) {
        argument result;
        m.mark_start(ins);
        result = f();
        m.mark_stop(ins);
        return result;
    }));
    m.mark_stop(*this);
}

634
635
636
637
void program::perf_report(std::ostream& os,
                          std::size_t n,
                          parameter_map params,
                          std::size_t batch) const
Paul's avatar
Paul committed
638
{
639
    auto& ctx = this->impl->ctx;
Paul's avatar
Paul committed
640
641
    // Run once by itself
    eval(params);
Paul's avatar
Paul committed
642
    ctx.finish();
Paul's avatar
Paul committed
643
    // Run and time entire program
Paul's avatar
Paul committed
644
645
    std::vector<double> total_vec;
    total_vec.reserve(n);
Paul's avatar
Paul committed
646
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
647
    {
Paul's avatar
Paul committed
648
649
650
651
        total_vec.push_back(time<milliseconds>([&] {
            eval(params);
            ctx.finish();
        }));
Paul's avatar
Paul committed
652
    }
Paul's avatar
Paul committed
653
654
    std::sort(total_vec.begin(), total_vec.end());
    std::unordered_map<instruction_ref, std::vector<double>> ins_vec;
Paul's avatar
Paul committed
655
    // Fill the map
656
    generic_eval(*this, ctx, params, always([&](auto ins, auto) {
Paul's avatar
Paul committed
657
        ins_vec[ins].reserve(n);
658
        return argument{ins->get_shape(), nullptr};
659
    }));
660

Paul's avatar
Paul committed
661
    // Run and time each instruction
Paul's avatar
Paul committed
662
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
663
    {
664
        generic_eval(*this, ctx, params, always([&](auto ins, auto f) {
665
            argument result;
Paul's avatar
Paul committed
666
667
668
669
            ins_vec[ins].push_back(time<milliseconds>([&] {
                result = f();
                ctx.finish();
            }));
670
            return result;
671
        }));
Paul's avatar
Paul committed
672
    }
Paul's avatar
Paul committed
673
674
    for(auto&& p : ins_vec)
        std::sort(p.second.begin(), p.second.end());
Paul's avatar
Paul committed
675
    // Run and time implicit overhead
Paul's avatar
Paul committed
676
677
    std::vector<double> overhead_vec;
    overhead_vec.reserve(n);
Paul's avatar
Paul committed
678
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
679
    {
Paul's avatar
Paul committed
680
        overhead_vec.push_back(time<milliseconds>([&] { dry_run(params); }));
Paul's avatar
Paul committed
681
682
    }

Paul's avatar
Paul committed
683
    double total_time             = common_average(total_vec);
Paul's avatar
Paul committed
684
    double rate                   = 1000.0 / total_time;
Paul's avatar
Paul committed
685
    double overhead_time          = common_average(overhead_vec);
Paul's avatar
Paul committed
686
    double overhead_percent       = overhead_time * 100.0 / total_time;
Paul's avatar
Paul committed
687
    double total_instruction_time = 0.0;
Paul's avatar
Paul committed
688
    std::unordered_map<std::string, double> op_times;
Paul's avatar
Paul committed
689
    for(auto&& p : ins_vec)
Paul's avatar
Paul committed
690
691
    {
        double avg = common_average(p.second);
Paul Fultz II's avatar
Paul Fultz II committed
692
        op_times[perf_group(p.first->get_operator())] += avg;
Paul's avatar
Paul committed
693
694
        total_instruction_time += avg;
    }
Paul's avatar
Paul committed
695
696
    double calculate_overhead_time    = total_time - total_instruction_time;
    double calculate_overhead_percent = calculate_overhead_time * 100.0 / total_time;
Paul's avatar
Paul committed
697

Shucai Xiao's avatar
Shucai Xiao committed
698
699
700
    std::unordered_map<instruction_ref, std::string> names;
    this->print(names, [&](auto ins, auto ins_names) {
        instruction::print(std::cout, ins, ins_names);
701
702
703
704
705

        // skip return instruction
        if(ins->name() == "@return")
            return;

Paul's avatar
Paul committed
706
707
708
        double avg     = common_average(ins_vec[ins]);
        double percent = std::ceil(100.0 * avg / total_instruction_time);
        os << ": " << avg << "ms, " << percent << "%";
709
        os << std::endl;
Paul's avatar
Paul committed
710
    });
Paul's avatar
Paul committed
711
712
713

    os << std::endl;
    os << "Summary:" << std::endl;
714
715
716
717
718
719
720
    std::vector<std::pair<double, std::string>> op_times_sorted;
    std::transform(op_times.begin(),
                   op_times.end(),
                   std::back_inserter(op_times_sorted),
                   [](auto p) { return std::make_pair(p.second, p.first); });
    std::sort(op_times_sorted.begin(), op_times_sorted.end(), std::greater<>{});
    for(auto&& p : op_times_sorted)
Paul's avatar
Paul committed
721
    {
722
723
        auto&& name    = p.second;
        double avg     = p.first;
Paul's avatar
Paul committed
724
725
726
727
728
        double percent = std::ceil(100.0 * avg / total_instruction_time);
        os << name << ": " << avg << "ms, " << percent << "%" << std::endl;
    }

    os << std::endl;
Paul's avatar
Paul committed
729

730
731
    os << "Batch size: " << batch << std::endl;
    os << "Rate: " << rate * batch << "/sec" << std::endl;
Paul's avatar
Paul committed
732
733
    os << "Total time: " << total_time << "ms" << std::endl;
    os << "Total instructions time: " << total_instruction_time << "ms" << std::endl;
Paul's avatar
Paul committed
734
735
736
737
    os << "Overhead time: " << overhead_time << "ms"
       << ", " << calculate_overhead_time << "ms" << std::endl;
    os << "Overhead: " << std::round(overhead_percent) << "%"
       << ", " << std::round(calculate_overhead_percent) << "%" << std::endl;
Paul's avatar
Paul committed
738
739
}

Paul's avatar
Paul committed
740
741
void program::debug_print() const { std::cout << *this << std::endl; }
void program::debug_print(instruction_ref ins) const
Paul's avatar
Paul committed
742
{
Shucai Xiao's avatar
Shucai Xiao committed
743
    std::unordered_map<instruction_ref, std::string> names;
744
    if(std::any_of(this->impl->modules.begin(), this->impl->modules.end(), [&](const auto& pp) {
745
           return is_end(pp.second.end(), ins);
Shucai Xiao's avatar
Shucai Xiao committed
746
       }))
Paul's avatar
Paul committed
747
748
749
750
    {
        std::cout << "End instruction" << std::endl;
        return;
    }
751
752
    else if(std::none_of(this->impl->modules.begin(),
                         this->impl->modules.end(),
753
                         [&](const auto& pp) { return pp.second.has_instruction(ins); }))
Paul's avatar
Paul committed
754
755
756
757
    {
        std::cout << "Instruction not part of program" << std::endl;
        return;
    }
Shucai Xiao's avatar
Shucai Xiao committed
758

Paul's avatar
Paul committed
759
    std::stringstream ss;
Shucai Xiao's avatar
Shucai Xiao committed
760
    this->print(names, [&](auto x, auto ins_names) {
Paul's avatar
Paul committed
761
        if(x == ins)
Paul's avatar
Paul committed
762
        {
Shucai Xiao's avatar
Shucai Xiao committed
763
            instruction::print(std::cout, x, ins_names);
Paul's avatar
Paul committed
764
765
766
767
768
            std::cout << std::endl;
        }
    });
}

Shucai Xiao's avatar
Shucai Xiao committed
769
770
771
772
void program::print(
    std::unordered_map<instruction_ref, std::string>& names,
    const std::function<void(instruction_ref, std::unordered_map<instruction_ref, std::string>)>&
        print_func) const
773
{
774
    for(const auto& pp : this->impl->modules)
775
    {
776
        names = pp.second.print(print_func, names);
777
778
779
    }
}

Shucai Xiao's avatar
Shucai Xiao committed
780
781
782
783
784
785
786
787
void program::print(
    const std::function<void(instruction_ref ins,
                             std::unordered_map<instruction_ref, std::string>)>& print_func) const
{
    std::unordered_map<instruction_ref, std::string> names;
    this->print(names, print_func);
}

Shucai Xiao's avatar
Shucai Xiao committed
788
void program::print_graph(std::ostream& os, bool brief) const
789
{
Shucai Xiao's avatar
Shucai Xiao committed
790
791
    const auto* mm = this->get_main_module();
    mm->print_graph(os, brief);
792
793
794
795
}

void program::print_cpp(std::ostream& os) const
{
Shucai Xiao's avatar
Shucai Xiao committed
796
797
798
799
800
801
802
803
    auto vec_modules = this->get_modules();
    std::unordered_map<instruction_ref, std::string> names;
    for(auto& mod : vec_modules)
    {
        os << "module: \"" << mod->name() << "\"" << std::endl;
        names = mod->print_cpp(os, names);
        os << std::endl;
    }
804
805
}

Paul's avatar
Paul committed
806
807
void program::dry_run(std::unordered_map<std::string, argument> params) const
{
Paul's avatar
Paul committed
808
    auto& ctx = this->impl->ctx;
809
810
811
    generic_eval(*this, ctx, std::move(params), always([](auto ins, auto&&...) {
        return argument{ins->get_shape(), nullptr};
    }));
Paul's avatar
Paul committed
812
813
}

Shucai Xiao's avatar
Shucai Xiao committed
814
void program::annotate(std::ostream& os, const std::function<void(instruction_ref)>& a) const
Paul's avatar
Paul committed
815
{
816
    for(auto& pp : this->impl->modules)
Shucai Xiao's avatar
Shucai Xiao committed
817
    {
818
819
        std::cout << pp.first << ":" << std::endl;
        pp.second.annotate(os, a);
Shucai Xiao's avatar
Shucai Xiao committed
820
    }
Paul's avatar
Paul committed
821
822
}

Paul's avatar
Paul committed
823
const module* program::get_module(const std::string& name) const { return &impl->modules.at(name); }
Shucai Xiao's avatar
Shucai Xiao committed
824
825
826

module* program::create_module(const std::string& name)
{
827
    assert(not contains(impl->modules, name));
828
829
    auto r = impl->modules.emplace(name, name);
    return &(r.first->second);
Shucai Xiao's avatar
Shucai Xiao committed
830
831
}

Paul's avatar
Paul committed
832
module* program::get_module(const std::string& name) { return &impl->modules.at(name); }
Shucai Xiao's avatar
Shucai Xiao committed
833
834
835
836
837

module* program::get_main_module() { return get_module("main"); }

const module* program::get_main_module() const { return get_module("main"); }

Paul's avatar
Paul committed
838
template <class T>
839
std::vector<T*> generic_get_modules(T* mm)
Shucai Xiao's avatar
Shucai Xiao committed
840
{
841
    std::vector<T*> vec_modules;
Shucai Xiao's avatar
Shucai Xiao committed
842
843
844
845
846
    vec_modules.push_back(mm);
    auto sub_modules = mm->get_sub_modules();
    vec_modules.insert(vec_modules.end(), sub_modules.begin(), sub_modules.end());
    return vec_modules;
}
Shucai Xiao's avatar
Shucai Xiao committed
847

Paul's avatar
Paul committed
848
template <class Map, class T, class OutputIterator>
849
void generic_get_unused_modules(Map& m, const std::vector<T*>& mods, OutputIterator out)
Shucai Xiao's avatar
Shucai Xiao committed
850
{
851
852
853
854
    std::unordered_set<std::string> used;
    std::transform(mods.begin(), mods.end(), std::inserter(used, used.end()), [](auto&& mod) {
        return mod->name();
    });
bpickrel's avatar
bpickrel committed
855
856
857
858
859
860
    transform_if(
        m.begin(),
        m.end(),
        out,
        [&](auto&& pp) { return not contains(used, pp.first); },
        [](auto&& pp) { return &pp.second; });
861
}
Shucai Xiao's avatar
Shucai Xiao committed
862

863
864
865
866
867
868
869
870
871
872
873
874
std::vector<const module*> program::get_modules() const
{
    auto result = generic_get_modules(this->get_main_module());
    generic_get_unused_modules(impl->modules, result, std::back_inserter(result));
    return result;
}

std::vector<module*> program::get_modules()
{
    auto result = generic_get_modules(this->get_main_module());
    generic_get_unused_modules(impl->modules, result, std::back_inserter(result));
    return result;
Shucai Xiao's avatar
Shucai Xiao committed
875
876
}

877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
template <class Map, class T>
bool is_unused_module(Map& m, const std::vector<T*>& mods, const std::string& name)
{
    bool is_unused = false;
    generic_get_unused_modules(m, mods, make_function_output_iterator([&](auto* mod) {
                                   if(mod->name() == name)
                                       is_unused = true;
                               }));
    return is_unused;
}

template <class Map>
bool references_instruction(Map& m, const instruction& ins, const std::string& name)
{
    return std::any_of(m.begin(), m.end(), [&](auto&& p) {
        if(p.first == name)
            return false;
        return std::any_of(p.second.begin(), p.second.end(), [&](auto&& i) {
            return std::any_of(i.inputs().begin(), i.inputs().end(), [&](auto&& j) {
                return std::addressof(*j) == std::addressof(ins);
            });
        });
    });
}

void program::remove_module(const std::string& name)
{
    // cppcheck-suppress assertWithSideEffect
    assert(is_unused_module(impl->modules, generic_get_modules(this->get_main_module()), name) &&
           "Module used in program");
    assert(std::none_of(
               impl->modules.at(name).begin(),
               impl->modules.at(name).end(),
               [&](auto&& ins) { return references_instruction(impl->modules, ins, name); }) &&
           "Instruction referenced in another module");
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927

    // if an instruction has an input out side of the current module, need to remove
    // the instruction from its input's outputs
    auto& mod = impl->modules.at(name);
    for(auto ins : iterator_for(mod))
    {
        auto inputs = ins->inputs();
        for(auto in : inputs)
        {
            if(not mod.has_instruction(in))
            {
                in->remove_output(ins);
            }
        }
    }

928
929
930
931
932
933
934
935
936
937
938
939
    impl->modules.erase(name);
}

void program::remove_unused_modules()
{
    std::vector<module*> unused;
    generic_get_unused_modules(
        impl->modules, generic_get_modules(this->get_main_module()), std::back_inserter(unused));
    for(auto* m : unused)
        this->remove_module(m->name());
}

940
941
program& program::sort()
{
942
    for(auto& pp : this->impl->modules)
Shucai Xiao's avatar
Shucai Xiao committed
943
    {
944
        pp.second.sort();
Shucai Xiao's avatar
Shucai Xiao committed
945
946
    }

947
948
949
    return *this;
}

Paul's avatar
Paul committed
950
bool operator==(const program& x, const program& y) { return to_string(x) == to_string(y); }
Paul's avatar
Paul committed
951

Paul's avatar
Paul committed
952
std::ostream& operator<<(std::ostream& os, const program& p)
Paul's avatar
Paul committed
953
{
Shucai Xiao's avatar
Shucai Xiao committed
954
955
956
    auto vec_modules = p.get_modules();
    std::unordered_map<instruction_ref, std::string> names;
    for(auto& mod : vec_modules)
Shucai Xiao's avatar
Shucai Xiao committed
957
    {
Shucai Xiao's avatar
Shucai Xiao committed
958
959
960
961
962
963
964
        os << "module: \"" << mod->name() << "\"" << std::endl;
        names = mod->print(
            [&](auto ins, auto ins_names) {
                instruction::print(os, ins, ins_names);
                os << std::endl;
            },
            names);
965
        os << std::endl;
Shucai Xiao's avatar
Shucai Xiao committed
966
967
    }

Paul's avatar
Paul committed
968
    return os;
Paul's avatar
Paul committed
969
}
Paul's avatar
Paul committed
970

Paul's avatar
Paul committed
971
} // namespace MIGRAPHX_INLINE_NS
Paul's avatar
Paul committed
972
} // namespace migraphx