program.cpp 33.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/*
 * The MIT License (MIT)
 *
 * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
Paul's avatar
Paul committed
24
25
26
#include <migraphx/program.hpp>
#include <migraphx/stringutils.hpp>
#include <migraphx/instruction.hpp>
27
#include <migraphx/op/identity.hpp>
Paul's avatar
Paul committed
28
#include <migraphx/target.hpp>
Paul's avatar
Paul committed
29
30
31
#include <migraphx/env.hpp>
#include <migraphx/ranges.hpp>
#include <migraphx/time.hpp>
32
#include <migraphx/pass_manager.hpp>
33
#include <migraphx/register_target.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
34
#include <migraphx/iterator_for.hpp>
35
#include <migraphx/iterator.hpp>
36
#include <migraphx/algorithm.hpp>
37
#include <migraphx/output_iterator.hpp>
Shucai Xiao's avatar
Shucai Xiao committed
38
#include <migraphx/make_op.hpp>
39
#include <migraphx/marker.hpp>
Paul's avatar
Paul committed
40
#include <iostream>
Paul's avatar
Paul committed
41
#include <sstream>
Paul's avatar
Paul committed
42
#include <algorithm>
43
#include <set>
Paul's avatar
Paul committed
44
#include <utility>
45

46
#include <unordered_set>
Shucai Xiao's avatar
Shucai Xiao committed
47
48
#include <map>
#include <cassert>
Paul's avatar
Paul committed
49

Paul's avatar
Paul committed
50
namespace migraphx {
Paul's avatar
Paul committed
51
inline namespace MIGRAPHX_INLINE_NS {
Paul's avatar
Paul committed
52

53
54
using milliseconds = std::chrono::duration<double, std::milli>;

Paul's avatar
Paul committed
55
56
struct program_impl
{
Shucai Xiao's avatar
Shucai Xiao committed
57
    // A map is used to keep references to modules of the program
58
    std::unordered_map<std::string, module> modules;
Paul's avatar
Paul committed
59
    context ctx;
60
    std::string target_name;
Paul's avatar
Paul committed
61
62
};

63
program::program() : impl(std::make_unique<program_impl>()) { this->create_module("main"); }
Paul's avatar
Paul committed
64

Paul's avatar
Paul committed
65
program::program(program&&) noexcept = default;
Shucai Xiao's avatar
Shucai Xiao committed
66
program::~program() noexcept         = default;
Paul's avatar
Paul committed
67

68
// copy constructor
Shucai Xiao's avatar
Shucai Xiao committed
69
program::program(const program& p) { assign(p); }
70
71

// copy assignment operator
Shucai Xiao's avatar
Shucai Xiao committed
72
program& program::operator=(program p)
73
{
Shucai Xiao's avatar
Shucai Xiao committed
74
    std::swap(p.impl, this->impl);
75
76
77
    return *this;
}

Shucai Xiao's avatar
Shucai Xiao committed
78
void program::assign(const program& p)
79
{
Shucai Xiao's avatar
Shucai Xiao committed
80
    if(!impl)
81
82
83
    {
        impl = std::make_unique<program_impl>();
    }
Shucai Xiao's avatar
Shucai Xiao committed
84
    else if(!impl->modules.empty())
85
    {
Shucai Xiao's avatar
Shucai Xiao committed
86
        impl->modules.clear();
87
    }
Shucai Xiao's avatar
Shucai Xiao committed
88

89
    impl->ctx         = p.impl->ctx;
Shucai Xiao's avatar
Shucai Xiao committed
90
91
    impl->target_name = p.impl->target_name;
    impl->modules     = p.impl->modules;
Shucai Xiao's avatar
Shucai Xiao committed
92
93
94
95

    // build a map from old ins to new ins
    // Build a map from old module to new module
    std::unordered_map<module_ref, module_ref> mod_map;
Paul's avatar
Paul committed
96
97
98
99
100
    std::transform(
        impl->modules.begin(),
        impl->modules.end(),
        std::inserter(mod_map, mod_map.begin()),
        [&](auto&& xp) { return std::make_pair(&p.impl->modules.at(xp.first), &xp.second); });
Shucai Xiao's avatar
Shucai Xiao committed
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

    std::unordered_map<instruction_ref, instruction_ref> ins_map;
    for(auto&& pp : mod_map)
    {
        auto old_ins = iterator_for(*pp.first);
        auto new_ins = iterator_for(*pp.second);
        std::transform(old_ins.begin(),
                       old_ins.end(),
                       new_ins.begin(),
                       std::inserter(ins_map, ins_map.begin()),
                       [](auto x, auto y) { return std::make_pair(x, y); });
    }

    // Update all references from all modules
    for(auto&& mp : impl->modules)
    {
117
        for(auto ins : iterator_for(mp.second))
Shucai Xiao's avatar
Shucai Xiao committed
118
119
            instruction::replace_refs(ins, ins_map, mod_map);
    }
120
121
}

Paul's avatar
Paul committed
122
shape program::get_parameter_shape(std::string name) const
Paul's avatar
Paul committed
123
{
Shucai Xiao's avatar
Shucai Xiao committed
124
125
    const auto* mm = this->get_main_module();
    return mm->get_parameter_shape(std::move(name));
Paul's avatar
Paul committed
126
127
}

128
129
std::vector<std::string> program::get_parameter_names() const
{
Shucai Xiao's avatar
Shucai Xiao committed
130
131
    const auto* mm = this->get_main_module();
    return mm->get_parameter_names();
132
133
}

mei-ye's avatar
mei-ye committed
134
135
instruction_ref program::get_parameter(std::string name) const
{
Shucai Xiao's avatar
Shucai Xiao committed
136
137
    const auto* mm = this->get_main_module();
    return mm->get_parameter(std::move(name));
mei-ye's avatar
mei-ye committed
138
139
}

Paul's avatar
Paul committed
140
141
std::unordered_map<std::string, shape> program::get_parameter_shapes() const
{
Shucai Xiao's avatar
Shucai Xiao committed
142
143
    const auto* mm = this->get_main_module();
    return mm->get_parameter_shapes();
Paul's avatar
Paul committed
144
145
}

Shucai Xiao's avatar
Shucai Xiao committed
146
std::size_t program::size() const { return impl->modules.size(); }
147

148
149
std::vector<shape> program::get_output_shapes() const
{
Shucai Xiao's avatar
Shucai Xiao committed
150
151
    const auto* mm = this->get_main_module();
    return mm->get_output_shapes();
152
}
Paul's avatar
Paul committed
153

Paul's avatar
Paul committed
154
155
context& program::get_context() const { return impl->ctx; }

Paul's avatar
Paul committed
156
157
instruction_ref program::validate() const
{
Shucai Xiao's avatar
Shucai Xiao committed
158
159
    const auto* mm = this->get_main_module();
    return mm->validate();
Paul's avatar
Paul committed
160
161
}

162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
target_assignments program::get_target_assignments(const std::vector<target>& targets,
                                                   assignment_options options)
{
    const auto m = options.metric;

    target_assignments p;

    const auto* mod = get_main_module();
    for(auto it : iterator_for(*mod))
    {
        auto t = std::max_element(
            targets.begin(), targets.end(), [it, m](const target& lhs, const target& rhs) {
                return lhs.is_supported(it, m) < rhs.is_supported(it, m);
            });
        p.add_assignment(it, t->name());
    }
    return p;
}

181
182
bool program::is_compiled() const { return not this->impl->target_name.empty(); }

183
void program::compile(const target& t, compile_options options)
Paul's avatar
Paul committed
184
{
185
186
187
    assert(not this->is_compiled());
    this->impl->target_name = t.name();
    this->impl->ctx         = t.get_context();
Paul's avatar
Paul committed
188
    if(enabled(MIGRAPHX_TRACE_COMPILE{}))
189
        options.trace = tracer{std::cout};
Shucai Xiao's avatar
Shucai Xiao committed
190

191
192
    options.trace(*this);
    options.trace();
Shucai Xiao's avatar
Shucai Xiao committed
193

Shucai Xiao's avatar
Shucai Xiao committed
194
    auto&& passes = t.get_passes(this->impl->ctx, options);
195
196
197
    run_passes(*this, passes, options.trace);

    auto mods = this->get_modules();
Shucai Xiao's avatar
Shucai Xiao committed
198

199
200
    // Validate and finalize
    for(const auto& mod : reverse(mods))
Paul's avatar
Paul committed
201
    {
Shucai Xiao's avatar
Shucai Xiao committed
202
203
204
205
206
207
        auto invalid = mod->validate();
        if(invalid != mod->end())
        {
            MIGRAPHX_THROW("Invalid module " + mod->name() + " from compilation at instruction " +
                           std::to_string(std::distance(mod->begin(), invalid)));
        }
208

209
210
211
212
213
214
215
        auto dangling = mod->find_dangling_reference();
        if(dangling != mod->end())
        {
            auto index = std::distance(mod->begin(), dangling);
            MIGRAPHX_THROW("Dangling reference in module " + mod->name() + " from instruction " +
                           std::to_string(index));
        }
216

217
        auto divide_by_zero = mod->find_division_by_zero();
218
219
        if(divide_by_zero != mod->end())
        {
220
221
222
            auto index = std::distance(mod->begin(), divide_by_zero);
            MIGRAPHX_THROW("Division by zero in module " + mod->name() + "from instruction" +
                           std::to_string(index));
223
224
        }

Shucai Xiao's avatar
Shucai Xiao committed
225
        mod->finalize(this->impl->ctx);
Paul's avatar
Paul committed
226
    }
Paul's avatar
Paul committed
227
228
229
230
}

void program::finalize()
{
Shucai Xiao's avatar
Shucai Xiao committed
231
232
    auto* mm = this->get_main_module();
    mm->finalize(this->impl->ctx);
Paul's avatar
Paul committed
233
234
}

235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
template <class T>
std::string classify(T x)
{
    switch(std::fpclassify(x))
    {
    case FP_INFINITE: return "inf";
    case FP_NAN: return "nan";
    case FP_NORMAL: return "normal";
    case FP_SUBNORMAL: return "subnormal";
    case FP_ZERO: return "zero";
    default: return "unknown";
    }
}

std::unordered_set<std::string> classify_argument(const argument& a)
{
    std::unordered_set<std::string> result;
    a.visit(
        [&](auto t) {
            for(const auto& x : t)
                result.insert(classify(x));
        },
        [&](const auto& xs) {
            for(const auto& x : xs)
            {
                auto r = classify_argument(x);
                result.insert(r.begin(), r.end());
            }
        });
    return result;
}

void preview_argument(std::ostream& os, const argument& a)
{
    a.visit(
        [&](auto t) {
            if(t.size() <= 10)
            {
                os << t;
            }
            else
            {
                os << to_string_range(t.begin(), t.begin() + 5);
                os << ", ..., ";
                os << to_string_range(t.end() - 5, t.end());
            }
        },
        [&](const auto& xs) {
            for(const auto& x : xs)
            {
                os << '{';
                preview_argument(os, x);
                os << '}';
            }
        });
}

Paul's avatar
Paul committed
292
template <class F>
Shucai Xiao's avatar
Shucai Xiao committed
293
std::vector<argument> generic_eval(const module* mod,
294
295
                                   context& ctx,
                                   std::unordered_map<std::string, argument> params,
Shucai Xiao's avatar
Shucai Xiao committed
296
                                   std::unordered_map<instruction_ref, argument> results,
297
                                   F make_trace)
Paul's avatar
Paul committed
298
{
Shucai Xiao's avatar
Shucai Xiao committed
299
300
    assert(mod->validate() == mod->end());
    results.reserve(mod->size() * 2);
Paul's avatar
Paul committed
301
302
    std::vector<argument> values;
    values.reserve(16);
303
    auto trace = make_trace(mod);
Shucai Xiao's avatar
Shucai Xiao committed
304
    for(auto ins : iterator_for(*mod))
Paul's avatar
Paul committed
305
    {
306
        assert(results.find(ins) == results.end());
307
308
        const auto& name = ins->name();
        if(name == "@literal")
Paul's avatar
Paul committed
309
        {
Paul's avatar
Paul committed
310
            results.emplace(ins, trace(ins, [&] { return ins->get_literal().get_argument(); }));
Paul's avatar
Paul committed
311
        }
312
        else if(name == "@param")
Paul's avatar
Paul committed
313
        {
Paul's avatar
Paul committed
314
315
316
317
318
            results.emplace(
                ins, trace(ins, [&] {
                    auto param_name = any_cast<builtin::param>(ins->get_operator()).parameter;
                    if(not contains(params, param_name))
                        MIGRAPHX_THROW("Parameter not found: " + param_name);
319
                    auto param = params[param_name];
Paul's avatar
Paul committed
320
321
322
323
324
                    if(param.get_shape() != ins->get_shape())
                        MIGRAPHX_THROW("Incorrect shape {" + to_string(param.get_shape()) +
                                       "} for parameter: " + param_name);
                    return param;
                }));
Paul's avatar
Paul committed
325
        }
326
        else if(name == "@outline")
Paul's avatar
Paul committed
327
        {
Paul's avatar
Paul committed
328
            results.emplace(ins, trace(ins, [&] { return argument{ins->get_shape(), nullptr}; }));
Paul's avatar
Paul committed
329
        }
330
331
332
333
334
335
336
337
338
339
340
341
342
        else if(name == "@return")
        {
            std::vector<argument> prog_outputs;
            std::transform(ins->inputs().begin(),
                           ins->inputs().end(),
                           std::back_inserter(prog_outputs),
                           [&](instruction_ref i) {
                               assert(results.find(i) != results.end());
                               return results[i];
                           });

            return prog_outputs;
        }
Paul's avatar
Paul committed
343
344
        else
        {
Paul's avatar
Paul committed
345
            values.resize(ins->inputs().size());
Paul's avatar
Paul committed
346
347
348
349
350
            std::transform(
                ins->inputs().begin(), ins->inputs().end(), values.begin(), [&](instruction_ref i) {
                    assert(results.find(i) != results.end());
                    return results[i];
                });
Shucai Xiao's avatar
Shucai Xiao committed
351
352
353
354

            const auto& mod_args = ins->module_inputs();
            auto module_eval     = [&](module_ref smod,
                                   const std::unordered_map<std::string, argument>& inputs) {
Shucai Xiao's avatar
Shucai Xiao committed
355
356
                auto ssctx = ctx;
                return generic_eval(smod, ssctx, inputs, results, make_trace);
Shucai Xiao's avatar
Shucai Xiao committed
357
358
            };

Shucai Xiao's avatar
Shucai Xiao committed
359
360
361
362
            results.emplace(ins, trace(ins, [&] {
                                return ins->normalized_operator().compute(
                                    ctx, ins->get_shape(), values, mod_args, module_eval);
                            }));
Paul's avatar
Paul committed
363
        }
364
        assert(results.find(ins) != results.end());
365
        assert(results.at(ins).get_shape() == ins->get_shape());
Paul's avatar
Paul committed
366
    }
Shucai Xiao's avatar
Shucai Xiao committed
367
    return {results.at(std::prev(mod->end()))};
Paul's avatar
Paul committed
368
369
}

Shucai Xiao's avatar
Shucai Xiao committed
370
371
372
373
template <class F>
std::vector<argument> generic_eval(const program& p,
                                   context& ctx,
                                   std::unordered_map<std::string, argument> params,
374
                                   F make_trace)
Shucai Xiao's avatar
Shucai Xiao committed
375
{
Shucai Xiao's avatar
Shucai Xiao committed
376
    const module* mm = p.get_main_module();
377
    return generic_eval(mm, ctx, params, {}, make_trace);
Shucai Xiao's avatar
Shucai Xiao committed
378
379
}

380
std::vector<argument> program::eval(parameter_map params) const
Paul's avatar
Paul committed
381
{
Paul's avatar
Paul committed
382
383
    auto& ctx = this->impl->ctx;
#ifndef NDEBUG
384
385
386
387
388
389
390
391
392
393
394
    auto with_check_context = [&](auto f) {
        return [=, &ctx](auto&&) {
            auto sctx          = std::make_shared<context>(ctx);
            auto check_context = [=, &ctx](auto g) {
                assert(is_shared(ctx, *sctx));
                auto x = g();
                *sctx  = ctx;
                return x;
            };
            return [=](auto&&... xs) { return f(xs..., check_context); };
        };
Paul's avatar
Paul committed
395
396
    };
#else
397
398
399
400
401
    auto with_check_context = [](auto f) {
        return [=](auto&&) {
            return [=](auto&&... xs) { return f(xs..., [](auto g) { return g(); }); };
        };
    };
Paul's avatar
Paul committed
402
#endif
Paul's avatar
Paul committed
403
404
405
406

    auto trace_level = value_of(MIGRAPHX_TRACE_EVAL{});

    if(trace_level > 0)
Paul's avatar
Paul committed
407
    {
Shucai Xiao's avatar
Shucai Xiao committed
408
409
410
411
412
413
414
415
        std::unordered_map<instruction_ref, std::string> ins_out;
        // get instruction names
        this->print([&](auto x, auto ins_names) {
            std::stringstream ss;
            instruction::print(ss, x, ins_names);
            ins_out[x] = ss.str();
        });

416
417
418
419
420
        return generic_eval(*this,
                            ctx,
                            std::move(params),
                            with_check_context([&](auto& ins, auto f, auto&& check_context) {
                                ctx.finish();
Shucai Xiao's avatar
Shucai Xiao committed
421
                                std::cout << "Run instruction: " << ins_out.at(ins) << std::endl;
422
423
424
425
426
427
428
                                timer t{};
                                auto result = check_context(f);
                                double t1   = t.record<milliseconds>();
                                ctx.finish();
                                double t2 = t.record<milliseconds>();
                                std::cout << "Time: " << t1 << "ms, " << t2 << "ms" << std::endl;
                                if(trace_level > 1 and ins->name().front() != '@' and
Shucai Xiao's avatar
Shucai Xiao committed
429
430
                                   ins->name() != "load" and not result.empty())
                                {
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
                                    target tgt  = make_target(this->impl->target_name);
                                    auto buffer = tgt.copy_from(result);
                                    if(trace_level == 2)
                                    {
                                        std::cout << "Output has "
                                                  << to_string_range(classify_argument(buffer))
                                                  << std::endl;
                                        std::cout << "Output: ";
                                        preview_argument(std::cout, buffer);
                                        std::cout << std::endl;
                                    }
                                    else
                                    {
                                        std::cout << "Output: " << buffer << std::endl;
                                    }
Shucai Xiao's avatar
Shucai Xiao committed
446
                                }
447
448
                                return result;
                            }));
Paul's avatar
Paul committed
449
450
451
    }
    else
    {
452
453
454
455
456
457
        return generic_eval(*this,
                            ctx,
                            std::move(params),
                            with_check_context([&](auto&, auto f, auto&& check_context) {
                                return check_context(f);
                            }));
Paul's avatar
Paul committed
458
    }
Paul's avatar
Paul committed
459
460
}

461
const int program_file_version = 5;
462
463
464
465
466
467
468
469

value program::to_value() const
{
    value result;
    result["version"] = program_file_version;
    result["target"]  = this->impl->target_name;
    if(not this->impl->target_name.empty())
        result["context"] = this->impl->ctx.to_value();
Shucai Xiao's avatar
Shucai Xiao committed
470

471
    value module_vals = value::object{};
Shucai Xiao's avatar
Shucai Xiao committed
472
    std::unordered_map<instruction_ref, std::string> names;
473
    for(auto& mod : this->get_modules())
Shucai Xiao's avatar
Shucai Xiao committed
474
    {
Shucai Xiao's avatar
Shucai Xiao committed
475
476
        value mod_val;
        value nodes;
477
478
        mod_val["name"] = mod->name();
        names           = mod->print(
Shucai Xiao's avatar
Shucai Xiao committed
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
            [&](auto ins, auto ins_names) {
                value node;
                node["output"]     = ins_names.at(ins);
                node["name"]       = ins->name();
                node["shape"]      = migraphx::to_value(ins->get_shape());
                node["normalized"] = ins->is_normalized();
                if(ins->name() == "@literal")
                    node["literal"] = migraphx::to_value(ins->get_literal());
                node["operator"] = ins->get_operator().to_value();
                std::vector<std::string> inputs;
                std::transform(ins->inputs().begin(),
                               ins->inputs().end(),
                               std::back_inserter(inputs),
                               [&](auto i) {
                                   assert(contains(ins_names, i));
                                   return ins_names.at(i);
                               });
                node["inputs"]   = inputs;
                auto module_args = ins->module_inputs();
                if(not module_args.empty())
                {
                    std::vector<std::string> module_inputs;
                    std::transform(module_args.begin(),
                                   module_args.end(),
                                   std::back_inserter(module_inputs),
                                   [&](auto mod_ref) { return mod_ref->name(); });
                    node["module_inputs"] = module_inputs;
                }

                nodes.push_back(node);
            },
            names);
        mod_val["nodes"] = nodes;

513
        module_vals[mod->name()] = mod_val;
Shucai Xiao's avatar
Shucai Xiao committed
514
    }
Shucai Xiao's avatar
Shucai Xiao committed
515
516
517

    result["modules"] = module_vals;

518
519
    return result;
}
Shucai Xiao's avatar
Shucai Xiao committed
520

Shucai Xiao's avatar
Shucai Xiao committed
521
522
523
524
525
static void mod_from_val(module_ref mod,
                         const value& v,
                         std::unordered_map<std::string, instruction_ref>& instructions,
                         const std::unordered_map<std::string, module_ref>& map_mods)
{
526
    const auto& module_val = v.at(mod->name());
Shucai Xiao's avatar
Shucai Xiao committed
527
528
529
530
531
532
533
534
535
    for(const value& node : module_val.at("nodes"))
    {
        instruction_ref output;
        auto name       = node.at("name").to<std::string>();
        auto fields     = node.at("operator");
        auto normalized = node.at("normalized").to<bool>();

        if(name == "@param")
        {
Paul Fultz II's avatar
Paul Fultz II committed
536
537
538
            output = mod->insert_parameter(mod->end(),
                                           fields["parameter"].to<std::string>(),
                                           migraphx::from_value<shape>(node.at("shape")));
Shucai Xiao's avatar
Shucai Xiao committed
539
540
541
        }
        else if(name == "@literal")
        {
Paul Fultz II's avatar
Paul Fultz II committed
542
543
            output =
                mod->insert_literal(mod->end(), migraphx::from_value<literal>(node.at("literal")));
Shucai Xiao's avatar
Shucai Xiao committed
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
        }
        else
        {
            auto op = make_op(name, fields);
            std::vector<instruction_ref> inputs;
            std::transform(node.at("inputs").begin(),
                           node.at("inputs").end(),
                           std::back_inserter(inputs),
                           [&](const value& i) {
                               auto i_name = i.to<std::string>();
                               assert(contains(instructions, i_name));
                               return instructions.at(i_name);
                           });

            std::vector<module_ref> module_inputs;
            if(node.contains("module_inputs"))
            {
                std::transform(node.at("module_inputs").begin(),
                               node.at("module_inputs").end(),
                               std::back_inserter(module_inputs),
                               [&](const value& i) { return map_mods.at(i.to<std::string>()); });

                for(auto& smod : module_inputs)
                {
                    mod_from_val(smod, v, instructions, map_mods);
                }
            }

            if(name == "@return")
            {
                output = mod->add_return(inputs);
            }
            else if(module_inputs.empty())
            {
Paul Fultz II's avatar
Paul Fultz II committed
578
                output = mod->insert_instruction(mod->end(), op, inputs);
Shucai Xiao's avatar
Shucai Xiao committed
579
580
581
            }
            else
            {
Paul Fultz II's avatar
Paul Fultz II committed
582
                output = mod->insert_instruction(mod->end(), op, inputs, module_inputs);
Shucai Xiao's avatar
Shucai Xiao committed
583
584
585
586
587
588
589
            }
        }
        output->set_normalized(normalized);
        instructions[node.at("output").to<std::string>()] = output;
    }
}

590
591
592
593
void program::from_value(const value& v)
{
    auto version = v.at("version").to<int>();
    if(version != program_file_version)
Shucai Xiao's avatar
Shucai Xiao committed
594
595
596
597
    {
        MIGRAPHX_THROW("Warning: Program version mismatch");
    }

598
599
600
601
602
603
604
605
    this->impl->target_name = v.at("target").to<std::string>();
    if(not this->impl->target_name.empty())
    {
        target t        = make_target(this->impl->target_name);
        this->impl->ctx = t.get_context();
        this->impl->ctx.from_value(v.at("context"));
    }

Shucai Xiao's avatar
Shucai Xiao committed
606
607
    auto module_vals = v.at("modules");
    for(const auto& vv : module_vals)
608
    {
609
        const auto& name = vv.get_key();
Shucai Xiao's avatar
Shucai Xiao committed
610
611
        if(name == "main")
            continue;
612
        impl->modules.emplace(name, name);
613
    }
614
    std::unordered_map<std::string, module_ref> map_mods;
Paul's avatar
Paul committed
615
616
617
618
    std::transform(impl->modules.begin(),
                   impl->modules.end(),
                   std::inserter(map_mods, map_mods.end()),
                   [&](auto&& pp) { return std::make_pair(pp.first, &pp.second); });
Shucai Xiao's avatar
Shucai Xiao committed
619
620
621
622
623

    std::unordered_map<std::string, instruction_ref> map_insts;
    auto* mm = get_main_module();
    mod_from_val(mm, module_vals, map_insts, map_mods);

624
625
626
    this->finalize();
}

Paul's avatar
Paul committed
627
628
629
double common_average(const std::vector<double>& v)
{
    std::size_t n = v.size() / 4;
Paul's avatar
Paul committed
630
631
    double total  = std::accumulate(v.begin() + n, v.end() - n, 0.0);
    return total / std::distance(v.begin() + n, v.end() - n);
Paul's avatar
Paul committed
632
633
}

Paul Fultz II's avatar
Paul Fultz II committed
634
635
636
637
638
639
640
641
std::string perf_group(const operation& op)
{
    auto attr = op.attributes();
    if(attr.contains("group"))
        return attr.at("group").to<std::string>();
    return op.name();
}

642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
void program::mark(const parameter_map& params, marker&& m)
{
    auto& ctx = this->impl->ctx;
    // Run once by itself
    eval(params);
    ctx.finish();
    // Start marking
    m.mark_start(*this);
    generic_eval(*this, ctx, params, always([&](auto ins, auto f) {
        argument result;
        m.mark_start(ins);
        result = f();
        m.mark_stop(ins);
        return result;
    }));
    m.mark_stop(*this);
}

660
661
662
663
void program::perf_report(std::ostream& os,
                          std::size_t n,
                          parameter_map params,
                          std::size_t batch) const
Paul's avatar
Paul committed
664
{
665
    auto& ctx = this->impl->ctx;
Paul's avatar
Paul committed
666
667
    // Run once by itself
    eval(params);
Paul's avatar
Paul committed
668
    ctx.finish();
Paul's avatar
Paul committed
669
    // Run and time entire program
Paul's avatar
Paul committed
670
671
    std::vector<double> total_vec;
    total_vec.reserve(n);
Paul's avatar
Paul committed
672
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
673
    {
Paul's avatar
Paul committed
674
675
676
677
        total_vec.push_back(time<milliseconds>([&] {
            eval(params);
            ctx.finish();
        }));
Paul's avatar
Paul committed
678
    }
Paul's avatar
Paul committed
679
680
    std::sort(total_vec.begin(), total_vec.end());
    std::unordered_map<instruction_ref, std::vector<double>> ins_vec;
Paul's avatar
Paul committed
681
    // Fill the map
682
    generic_eval(*this, ctx, params, always([&](auto ins, auto) {
Paul's avatar
Paul committed
683
        ins_vec[ins].reserve(n);
684
        return argument{ins->get_shape(), nullptr};
685
    }));
686

Paul's avatar
Paul committed
687
    // Run and time each instruction
Paul's avatar
Paul committed
688
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
689
    {
690
        generic_eval(*this, ctx, params, always([&](auto ins, auto f) {
691
            argument result;
Paul's avatar
Paul committed
692
693
694
695
            ins_vec[ins].push_back(time<milliseconds>([&] {
                result = f();
                ctx.finish();
            }));
696
            return result;
697
        }));
Paul's avatar
Paul committed
698
    }
Paul's avatar
Paul committed
699
700
    for(auto&& p : ins_vec)
        std::sort(p.second.begin(), p.second.end());
Paul's avatar
Paul committed
701
    // Run and time implicit overhead
Paul's avatar
Paul committed
702
703
    std::vector<double> overhead_vec;
    overhead_vec.reserve(n);
Paul's avatar
Paul committed
704
    for(std::size_t i = 0; i < n; i++)
Paul's avatar
Paul committed
705
    {
Paul's avatar
Paul committed
706
        overhead_vec.push_back(time<milliseconds>([&] { dry_run(params); }));
Paul's avatar
Paul committed
707
708
    }

Paul's avatar
Paul committed
709
    double total_time             = common_average(total_vec);
Paul's avatar
Paul committed
710
    double rate                   = 1000.0 / total_time;
Paul's avatar
Paul committed
711
    double overhead_time          = common_average(overhead_vec);
Paul's avatar
Paul committed
712
    double overhead_percent       = overhead_time * 100.0 / total_time;
Paul's avatar
Paul committed
713
    double total_instruction_time = 0.0;
Paul's avatar
Paul committed
714
    std::unordered_map<std::string, double> op_times;
715
    std::unordered_map<std::string, std::size_t> op_n;
Paul's avatar
Paul committed
716
    for(auto&& p : ins_vec)
Paul's avatar
Paul committed
717
718
    {
        double avg = common_average(p.second);
Paul Fultz II's avatar
Paul Fultz II committed
719
        op_times[perf_group(p.first->get_operator())] += avg;
Paul's avatar
Paul committed
720
        total_instruction_time += avg;
721
        op_n[perf_group(p.first->get_operator())]++;
Paul's avatar
Paul committed
722
    }
Paul's avatar
Paul committed
723
724
    double calculate_overhead_time    = total_time - total_instruction_time;
    double calculate_overhead_percent = calculate_overhead_time * 100.0 / total_time;
Paul's avatar
Paul committed
725

Shucai Xiao's avatar
Shucai Xiao committed
726
727
728
    std::unordered_map<instruction_ref, std::string> names;
    this->print(names, [&](auto ins, auto ins_names) {
        instruction::print(std::cout, ins, ins_names);
729
730
731
732
733

        // skip return instruction
        if(ins->name() == "@return")
            return;

Paul's avatar
Paul committed
734
735
736
        double avg     = common_average(ins_vec[ins]);
        double percent = std::ceil(100.0 * avg / total_instruction_time);
        os << ": " << avg << "ms, " << percent << "%";
737
        os << std::endl;
Paul's avatar
Paul committed
738
    });
Paul's avatar
Paul committed
739
740
741

    os << std::endl;
    os << "Summary:" << std::endl;
742
743
744
745
746
747
    std::vector<std::tuple<double, std::size_t, std::string>> op_times_sorted;
    std::transform(
        op_times.begin(), op_times.end(), std::back_inserter(op_times_sorted), [&](auto p) {
            auto&& name = p.first;
            return std::make_tuple(p.second, op_n.at(name), name);
        });
748
    std::sort(op_times_sorted.begin(), op_times_sorted.end(), std::greater<>{});
749
    for(auto&& [avg, nn, name] : op_times_sorted)
Paul's avatar
Paul committed
750
751
    {
        double percent = std::ceil(100.0 * avg / total_instruction_time);
752
753
754
        double per_ins = avg / nn;
        os << name << ": " << avg << "ms / " << nn << " = " << per_ins << "ms, " << percent << "%"
           << std::endl;
Paul's avatar
Paul committed
755
756
757
    }

    os << std::endl;
Paul's avatar
Paul committed
758

759
760
    os << "Batch size: " << batch << std::endl;
    os << "Rate: " << rate * batch << "/sec" << std::endl;
Paul's avatar
Paul committed
761
762
    os << "Total time: " << total_time << "ms" << std::endl;
    os << "Total instructions time: " << total_instruction_time << "ms" << std::endl;
Paul's avatar
Paul committed
763
764
765
766
    os << "Overhead time: " << overhead_time << "ms"
       << ", " << calculate_overhead_time << "ms" << std::endl;
    os << "Overhead: " << std::round(overhead_percent) << "%"
       << ", " << std::round(calculate_overhead_percent) << "%" << std::endl;
Paul's avatar
Paul committed
767
768
}

Paul's avatar
Paul committed
769
770
void program::debug_print() const { std::cout << *this << std::endl; }
void program::debug_print(instruction_ref ins) const
Paul's avatar
Paul committed
771
{
Shucai Xiao's avatar
Shucai Xiao committed
772
    std::unordered_map<instruction_ref, std::string> names;
773
    if(std::any_of(this->impl->modules.begin(), this->impl->modules.end(), [&](const auto& pp) {
774
           return is_end(pp.second.end(), ins);
Shucai Xiao's avatar
Shucai Xiao committed
775
       }))
Paul's avatar
Paul committed
776
777
778
779
    {
        std::cout << "End instruction" << std::endl;
        return;
    }
780
781
    else if(std::none_of(this->impl->modules.begin(),
                         this->impl->modules.end(),
782
                         [&](const auto& pp) { return pp.second.has_instruction(ins); }))
Paul's avatar
Paul committed
783
784
785
786
    {
        std::cout << "Instruction not part of program" << std::endl;
        return;
    }
Shucai Xiao's avatar
Shucai Xiao committed
787

Paul's avatar
Paul committed
788
    std::stringstream ss;
Shucai Xiao's avatar
Shucai Xiao committed
789
    this->print(names, [&](auto x, auto ins_names) {
Paul's avatar
Paul committed
790
        if(x == ins)
Paul's avatar
Paul committed
791
        {
Shucai Xiao's avatar
Shucai Xiao committed
792
            instruction::print(std::cout, x, ins_names);
Paul's avatar
Paul committed
793
794
795
796
797
            std::cout << std::endl;
        }
    });
}

Shucai Xiao's avatar
Shucai Xiao committed
798
799
800
801
void program::print(
    std::unordered_map<instruction_ref, std::string>& names,
    const std::function<void(instruction_ref, std::unordered_map<instruction_ref, std::string>)>&
        print_func) const
802
{
803
    for(const auto& pp : this->impl->modules)
804
    {
805
        names = pp.second.print(print_func, names);
806
807
808
    }
}

Shucai Xiao's avatar
Shucai Xiao committed
809
810
811
812
813
814
815
816
void program::print(
    const std::function<void(instruction_ref ins,
                             std::unordered_map<instruction_ref, std::string>)>& print_func) const
{
    std::unordered_map<instruction_ref, std::string> names;
    this->print(names, print_func);
}

Shucai Xiao's avatar
Shucai Xiao committed
817
void program::print_graph(std::ostream& os, bool brief) const
818
{
Shucai Xiao's avatar
Shucai Xiao committed
819
820
    const auto* mm = this->get_main_module();
    mm->print_graph(os, brief);
821
822
823
824
}

void program::print_cpp(std::ostream& os) const
{
Shucai Xiao's avatar
Shucai Xiao committed
825
826
    auto vec_modules = this->get_modules();
    std::unordered_map<instruction_ref, std::string> names;
827
    os << "migraphx::program p;\n";
Shucai Xiao's avatar
Shucai Xiao committed
828
829
    for(auto& mod : vec_modules)
    {
830
831
832
833
834
835
836
837
        std::string var_name = "m" + mod->name();
        os << "migraphx::module_ref " << var_name << " = ";
        if(mod->name() == "main")
            os << "p.get_main_module();";
        else
            os << "p.create_module(\"" << mod->name() << "\");";
        os << std::endl;
        names = mod->print_cpp(os, var_name, names);
Shucai Xiao's avatar
Shucai Xiao committed
838
839
        os << std::endl;
    }
840
841
}

Paul's avatar
Paul committed
842
843
void program::dry_run(std::unordered_map<std::string, argument> params) const
{
Paul's avatar
Paul committed
844
    auto& ctx = this->impl->ctx;
845
846
847
    generic_eval(*this, ctx, std::move(params), always([](auto ins, auto&&...) {
        return argument{ins->get_shape(), nullptr};
    }));
Paul's avatar
Paul committed
848
849
}

Shucai Xiao's avatar
Shucai Xiao committed
850
void program::annotate(std::ostream& os, const std::function<void(instruction_ref)>& a) const
Paul's avatar
Paul committed
851
{
852
    for(auto& pp : this->impl->modules)
Shucai Xiao's avatar
Shucai Xiao committed
853
    {
854
855
        std::cout << pp.first << ":" << std::endl;
        pp.second.annotate(os, a);
Shucai Xiao's avatar
Shucai Xiao committed
856
    }
Paul's avatar
Paul committed
857
858
}

Paul's avatar
Paul committed
859
const module* program::get_module(const std::string& name) const { return &impl->modules.at(name); }
Shucai Xiao's avatar
Shucai Xiao committed
860
861
862

module* program::create_module(const std::string& name)
{
863
    assert(not contains(impl->modules, name));
864
865
    auto r = impl->modules.emplace(name, name);
    return &(r.first->second);
Shucai Xiao's avatar
Shucai Xiao committed
866
867
}

Paul's avatar
Paul committed
868
module* program::get_module(const std::string& name) { return &impl->modules.at(name); }
Shucai Xiao's avatar
Shucai Xiao committed
869
870
871
872
873

module* program::get_main_module() { return get_module("main"); }

const module* program::get_main_module() const { return get_module("main"); }

Paul's avatar
Paul committed
874
template <class T>
875
std::vector<T*> generic_get_modules(T* mm)
Shucai Xiao's avatar
Shucai Xiao committed
876
{
877
    std::vector<T*> vec_modules;
Shucai Xiao's avatar
Shucai Xiao committed
878
879
880
881
882
    vec_modules.push_back(mm);
    auto sub_modules = mm->get_sub_modules();
    vec_modules.insert(vec_modules.end(), sub_modules.begin(), sub_modules.end());
    return vec_modules;
}
Shucai Xiao's avatar
Shucai Xiao committed
883

Paul's avatar
Paul committed
884
template <class Map, class T, class OutputIterator>
885
void generic_get_unused_modules(Map& m, const std::vector<T*>& mods, OutputIterator out)
Shucai Xiao's avatar
Shucai Xiao committed
886
{
887
888
889
890
    std::unordered_set<std::string> used;
    std::transform(mods.begin(), mods.end(), std::inserter(used, used.end()), [](auto&& mod) {
        return mod->name();
    });
bpickrel's avatar
bpickrel committed
891
892
893
894
895
896
    transform_if(
        m.begin(),
        m.end(),
        out,
        [&](auto&& pp) { return not contains(used, pp.first); },
        [](auto&& pp) { return &pp.second; });
897
}
Shucai Xiao's avatar
Shucai Xiao committed
898

899
900
901
902
903
904
905
906
907
908
909
910
std::vector<const module*> program::get_modules() const
{
    auto result = generic_get_modules(this->get_main_module());
    generic_get_unused_modules(impl->modules, result, std::back_inserter(result));
    return result;
}

std::vector<module*> program::get_modules()
{
    auto result = generic_get_modules(this->get_main_module());
    generic_get_unused_modules(impl->modules, result, std::back_inserter(result));
    return result;
Shucai Xiao's avatar
Shucai Xiao committed
911
912
}

913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
template <class Module, class Map>
void generic_insert_module_tree(Module* pm, Map& m)
{
    for(auto* sm : pm->get_sub_modules(true))
    {
        m.insert(std::make_pair(sm, pm));
        generic_insert_module_tree(sm, m);
    }
}

std::unordered_multimap<module_ref, module_ref> program::get_module_tree()
{
    std::unordered_multimap<module_ref, module_ref> result;
    generic_insert_module_tree(this->get_main_module(), result);
    return result;
}

930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
template <class Map, class T>
bool is_unused_module(Map& m, const std::vector<T*>& mods, const std::string& name)
{
    bool is_unused = false;
    generic_get_unused_modules(m, mods, make_function_output_iterator([&](auto* mod) {
                                   if(mod->name() == name)
                                       is_unused = true;
                               }));
    return is_unused;
}

template <class Map>
bool references_instruction(Map& m, const instruction& ins, const std::string& name)
{
    return std::any_of(m.begin(), m.end(), [&](auto&& p) {
        if(p.first == name)
            return false;
        return std::any_of(p.second.begin(), p.second.end(), [&](auto&& i) {
            return std::any_of(i.inputs().begin(), i.inputs().end(), [&](auto&& j) {
                return std::addressof(*j) == std::addressof(ins);
            });
        });
    });
}

void program::remove_module(const std::string& name)
{
    // cppcheck-suppress assertWithSideEffect
    assert(is_unused_module(impl->modules, generic_get_modules(this->get_main_module()), name) &&
           "Module used in program");
    assert(std::none_of(
               impl->modules.at(name).begin(),
               impl->modules.at(name).end(),
               [&](auto&& ins) { return references_instruction(impl->modules, ins, name); }) &&
           "Instruction referenced in another module");
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980

    // if an instruction has an input out side of the current module, need to remove
    // the instruction from its input's outputs
    auto& mod = impl->modules.at(name);
    for(auto ins : iterator_for(mod))
    {
        auto inputs = ins->inputs();
        for(auto in : inputs)
        {
            if(not mod.has_instruction(in))
            {
                in->remove_output(ins);
            }
        }
    }

981
982
983
984
985
986
987
988
989
990
991
992
    impl->modules.erase(name);
}

void program::remove_unused_modules()
{
    std::vector<module*> unused;
    generic_get_unused_modules(
        impl->modules, generic_get_modules(this->get_main_module()), std::back_inserter(unused));
    for(auto* m : unused)
        this->remove_module(m->name());
}

993
994
program& program::sort()
{
995
    for(auto& pp : this->impl->modules)
Shucai Xiao's avatar
Shucai Xiao committed
996
    {
997
        pp.second.sort();
Shucai Xiao's avatar
Shucai Xiao committed
998
999
    }

1000
1001
1002
    return *this;
}

Paul's avatar
Paul committed
1003
bool operator==(const program& x, const program& y) { return to_string(x) == to_string(y); }
Paul's avatar
Paul committed
1004

Paul's avatar
Paul committed
1005
std::ostream& operator<<(std::ostream& os, const program& p)
Paul's avatar
Paul committed
1006
{
Shucai Xiao's avatar
Shucai Xiao committed
1007
1008
1009
    auto vec_modules = p.get_modules();
    std::unordered_map<instruction_ref, std::string> names;
    for(auto& mod : vec_modules)
Shucai Xiao's avatar
Shucai Xiao committed
1010
    {
Shucai Xiao's avatar
Shucai Xiao committed
1011
1012
1013
1014
1015
1016
1017
        os << "module: \"" << mod->name() << "\"" << std::endl;
        names = mod->print(
            [&](auto ins, auto ins_names) {
                instruction::print(os, ins, ins_names);
                os << std::endl;
            },
            names);
1018
        os << std::endl;
Shucai Xiao's avatar
Shucai Xiao committed
1019
1020
    }

Paul's avatar
Paul committed
1021
    return os;
Paul's avatar
Paul committed
1022
}
Paul's avatar
Paul committed
1023

Paul's avatar
Paul committed
1024
} // namespace MIGRAPHX_INLINE_NS
Paul's avatar
Paul committed
1025
} // namespace migraphx