Merge branch 'master' into gpu_slice_test

src/include/migraph/simplify_reshapes.hpp

@@ -3,8 +3,10 @@
 
 #include <string>
 #include <migraph/instruction_ref.hpp>
+#include <migraph/config.hpp>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 struct program;
 
@@ -14,6 +16,7 @@ struct simplify_reshapes
     void apply(program& p) const;
 };
 
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 
 #endif

src/include/migraph/streamutils.hpp

@@ -4,8 +4,10 @@
 #include <ostream>
 #include <algorithm>
 #include <migraph/rank.hpp>
+#include <migraph/config.hpp>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 template <class T>
 struct stream_range_container
@@ -54,6 +56,7 @@ void stream_write_value(std::ostream& os, const T& x)
     detail::stream_write_value_impl(rank<1>{}, os, x);
 }
 
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 
 #endif

src/include/migraph/stringutils.hpp

@@ -5,8 +5,10 @@
 #include <numeric>
 #include <string>
 #include <sstream>
+#include <migraph/config.hpp>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 inline std::string
 replace_string(std::string subject, const std::string& search, const std::string& replace)
@@ -85,6 +87,7 @@ inline std::string to_string(const T& x)
     return ss.str();
 }
 
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 
 #endif

src/include/migraph/target.hpp

@@ -10,8 +10,10 @@
 #include <vector>
 #include <migraph/context.hpp>
 #include <migraph/pass.hpp>
+#include <migraph/config.hpp>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 #ifdef DOXYGEN
 
@@ -242,6 +244,7 @@ inline const ValueType& any_cast(const target& x)
 
 #endif
 
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 
 #endif

src/include/migraph/tensor_view.hpp

@@ -4,11 +4,13 @@
 #include <migraph/shape.hpp>
 #include <migraph/float_equal.hpp>
 #include <migraph/requires.hpp>
+#include <migraph/config.hpp>
 
 #include <iostream>
 #include <utility>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 template <class T>
 struct tensor_view
@@ -167,6 +169,7 @@ tensor_view<T> make_view(shape s, T* data)
     return {s, data};
 }
 
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 
 #endif

src/include/migraph/time.hpp

@@ -2,8 +2,10 @@
 #define MIGRAPH_GUARD_RTGLIB_TIME_HPP
 
 #include <chrono>
+#include <migraph/config.hpp>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 template <class Duration, class F>
 auto time(F f)
@@ -14,6 +16,7 @@ auto time(F f)
     return std::chrono::duration_cast<Duration>(finish - start).count();
 }
 
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 
 #endif

src/include/migraph/tracer.hpp

@@ -3,8 +3,10 @@
 
 #include <ostream>
 #include <migraph/functional.hpp>
+#include <migraph/config.hpp>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 struct tracer
 {
@@ -28,6 +30,7 @@ struct tracer
     std::ostream* os = nullptr;
 };
 
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 
 #endif

src/include/migraph/type_name.hpp

@@ -2,8 +2,10 @@
 #define MIGRAPH_GUARD_RTGLIB_TYPE_NAME_HPP
 
 #include <string>
+#include <migraph/config.hpp>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 template <class PrivateMigraphTypeNameProbe>
 const std::string& get_type_name()
@@ -39,6 +41,7 @@ const std::string& get_type_name(const T&)
     return migraph::get_type_name<T>();
 }
 
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 
 #endif

src/include/migraph/type_traits.hpp

@@ -10,8 +10,10 @@
 
 #include <type_traits>
 #include <migraph/half.hpp>
+#include <migraph/config.hpp>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 #define MIGRAPH_DETAIL_EXTEND_TRAIT_FOR(trait, T) \
     template <class X>                            \
@@ -28,6 +30,7 @@ MIGRAPH_DETAIL_EXTEND_TRAIT_FOR(is_floating_point, half)
 MIGRAPH_DETAIL_EXTEND_TRAIT_FOR(is_signed, half)
 MIGRAPH_DETAIL_EXTEND_TRAIT_FOR(is_arithmetic, half)
 
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 
 #endif

src/include/migraph/verify.hpp

@@ -8,8 +8,10 @@
 #include <numeric>
 
 #include <migraph/float_equal.hpp>
+#include <migraph/config.hpp>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 // Compute the value of a range
 template <class R>
@@ -170,5 +172,7 @@ bool verify_range(R1&& r1, R2&& r2, double tolerance = 80, double* out_error = n
         *out_error = error;
     return error <= threshold;
 }
+
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 #endif

src/include/migraph/verify_args.hpp

@@ -3,8 +3,10 @@
 
 #include <migraph/verify.hpp>
 #include <migraph/argument.hpp>
+#include <migraph/config.hpp>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 inline bool verify_args(const std::string& name,
                         const argument& cpu_arg,
@@ -82,6 +84,7 @@ inline bool verify_args(const std::string& name,
     return passed;
 }
 
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 
 #endif

src/instruction.cpp

@@ -3,6 +3,7 @@
 #include <migraph/erase.hpp>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 instruction::instruction(operation o, shape r, std::vector<instruction_ref> args)
     : op(std::move(o)), result(std::move(r)), arguments(std::move(args))
@@ -161,12 +162,26 @@ void instruction::replace_argument(instruction_ref old, instruction_ref new_ins)
     old->remove_output(*this);
 }
 
-shape compute_shape(const operation& op, const std::vector<instruction_ref>& args)
+std::vector<shape> compute_shapes(const std::vector<instruction_ref>& args)
 {
     std::vector<shape> shapes(args.size());
     std::transform(
         args.begin(), args.end(), shapes.begin(), [](instruction_ref i) { return i->get_shape(); });
-    return op.compute_shape(shapes);
+    return shapes;
+}
+
+instruction_ref instruction::get_output_alias(instruction_ref ins)
+{
+    auto i = ins->get_operator().output_alias(compute_shapes(ins->inputs()));
+    if(i < 0)
+        return ins;
+    return get_output_alias(ins->inputs().at(i));
+}
+
+shape compute_shape(const operation& op, const std::vector<instruction_ref>& args)
+{
+    return op.compute_shape(compute_shapes(args));
 }
 
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph

src/onnx/cifar10.cpp

@@ -6,7 +6,7 @@
 
 #include <migraph/onnx.hpp>
 
-#include <migraph/cpu/cpu_target.hpp>
+#include <migraph/cpu/target.hpp>
 #include <migraph/gpu/target.hpp>
 #include <migraph/gpu/hip.hpp>
 #include <migraph/generate.hpp>
@@ -86,7 +86,7 @@ int main(int argc, char const* argv[])
     else
     {
         // CPU target
-        prog.compile(migraph::cpu::cpu_target{});
+        prog.compile(migraph::cpu::target{});
         auto s      = migraph::shape{migraph::shape::float_type, {1, 3, 32, 32}};
         auto labels = imageset.first;
         auto input  = imageset.second;

src/onnx/onnx.cpp

@@ -14,9 +14,10 @@
 #include <migraph/operators.hpp>
 #include <migraph/ranges.hpp>
 #include <migraph/instruction.hpp>
+#include <migraph/config.hpp>
 
 namespace migraph {
-
+inline namespace MIGRAPH_INLINE_NS {
 struct unknown
 {
     std::string op;
@@ -52,7 +53,7 @@ struct onnx_parser
         add_generic_op("Div", op::div{});
         add_generic_op("MatMul", op::dot{});
         add_generic_op("Mul", op::mul{});
-        add_generic_op("Relu", op::activation{"relu"});
+        add_generic_op("Relu", op::relu{});
         add_generic_op("Sub", op::sub{});
         add_generic_op("Sum", op::add{});
 
@@ -62,6 +63,8 @@ struct onnx_parser
         add_mem_op("Conv", &onnx_parser::parse_conv);
         add_mem_op("MaxPool", &onnx_parser::parse_pooling);
         add_mem_op("AveragePool", &onnx_parser::parse_pooling);
+        add_mem_op("GlobalMaxPool", &onnx_parser::parse_pooling);
+        add_mem_op("GlobalAveragePool", &onnx_parser::parse_pooling);
         add_mem_op("Reshape", &onnx_parser::parse_reshape);
         add_mem_op("Flatten", &onnx_parser::parse_flatten);
         add_mem_op("Gemm", &onnx_parser::parse_gemm);
@@ -148,7 +151,12 @@ struct onnx_parser
                                   attribute_map attributes,
                                   std::vector<instruction_ref> args)
     {
-        op::pooling op{name == "MaxPool" ? "max" : "average"};
+        op::pooling op{ends_with(name, "MaxPool") ? "max" : "average"};
+        if(starts_with(name, "Global"))
+        {
+            auto lens  = args.front()->get_shape().lens();
+            op.lengths = {lens[2], lens[3]};
+        }
         if(contains(attributes, "pads"))
         {
             copy(attributes["pads"].ints(), op.padding.begin());
@@ -584,10 +592,15 @@ struct onnx_parser
         }
         std::vector<std::size_t> dims;
         auto&& tensor_dims = t.tensor_type().shape().dim();
-        std::transform(tensor_dims.begin(),
-                       tensor_dims.end(),
-                       std::back_inserter(dims),
-                       [](auto&& d) { return d.dim_value(); });
+        std::transform(
+            tensor_dims.begin(), tensor_dims.end(), std::back_inserter(dims), [](auto&& d) {
+                if(not d.has_dim_value())
+                {
+                    long default_batch_size = 1; // FIXME
+                    return default_batch_size;
+                }
+                return d.dim_value();
+            });
         return {shape_type, dims};
     }
 };
@@ -613,4 +626,5 @@ program parse_onnx(const std::string& name)
     return std::move(parser.prog);
 }
 
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph

src/onnx/verify_onnx.cpp

 
 #include <migraph/onnx.hpp>
 
-#include <migraph/cpu/cpu_target.hpp>
+#include <migraph/cpu/target.hpp>
 #include <migraph/gpu/target.hpp>
 #include <migraph/gpu/hip.hpp>
 #include <migraph/generate.hpp>
@@ -18,7 +18,7 @@ template <class F>
 migraph::argument run_cpu(F f)
 {
     auto p = f();
-    p.compile(migraph::cpu::cpu_target{});
+    p.compile(migraph::cpu::target{});
     migraph::program::parameter_map m;
     for(auto&& x : p.get_parameter_shapes())
     {

src/opt/common_header.hpp

@@ -6,6 +6,7 @@
 #include <migraph/operators.hpp>
 #include <migraph/iterator_for.hpp>
 #include <migraph/pass_config.hpp>
+#include <migraph/config.hpp>
 
 #include <set>
 #include <list>
@@ -13,6 +14,7 @@
 #include <queue>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 //#define MIGRAPH_DEBUG_OPT
 
@@ -21,6 +23,8 @@ namespace migraph {
 #else
 #define MIGRAPH_DEBUG(s)
 #endif // MIGRAPH_DEBUG_OPT
+
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 
 #endif // MIGRAPH_GUARD_RTGLIB_COMMON_HEADER_HPP

src/opt/memory_coloring.cpp

@@ -2,13 +2,16 @@
 #include "memory_coloring_impl.hpp"
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 void memory_coloring::apply(program& p) const
 {
     if(!enabled(MIGRAPH_DISABLE_MEMORY_COLORING{}))
     {
-        memory_coloring_impl opt(&p, allocation_op);
+        memory_coloring_impl opt(&p, allocation_op, verify);
         opt.run();
     }
 }
+
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph

src/opt/memory_coloring_impl.cpp

 #include "memory_coloring_impl.hpp"
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 void memory_coloring_impl::run()
 {
     MIGRAPH_DEBUG(dump("---Before memory coloring---"));
     MIGRAPH_DEBUG(dump_program());
-    register_operand_alias();
     build();
     if(num_of_lives != 0)
     {
@@ -19,7 +19,8 @@ void memory_coloring_impl::run()
             alloc_queue.pop();
         }
         rewrite();
-        MIGRAPH_DEBUG(verify());
+        if(enable_verify)
+            verify();
     }
 }
 
@@ -129,11 +130,8 @@ void memory_coloring_impl::build()
         {
             is_dead = true;
         }
-        int tie_ndx = get_input_tie_ndx(iter);
-        int cnt     = -1;
         for(auto&& arg : iter->inputs())
         {
-            cnt++;
             if(is_param(arg) || is_outline(arg))
             {
                 if(is_output_param(arg))
@@ -144,15 +142,8 @@ void memory_coloring_impl::build()
                 }
                 continue;
             }
-            const instruction* p_arg = &(*arg);
-            if(cnt == tie_ndx && (def_interval != nullptr))
-            {
-                // input memory is used as this instruction's output.
-                // def is considered as use. Coalesce the live intervals.
-                def_interval->add_use(cur_points);
-                instr2_live[p_arg] = def_interval;
-            }
-            else if(instr2_live.find(p_arg) == instr2_live.end())
+            const instruction* p_arg = &(*instruction::get_output_alias(arg));
+            if(instr2_live.find(p_arg) == instr2_live.end())
             {
                 // First time see a use, create a live interval.
                 int id                = num_of_lives++;
@@ -182,22 +173,6 @@ void memory_coloring_impl::build()
     } while(iter != begin);
 }
 
-void memory_coloring_impl::register_operand_alias()
-{
-    operand_alias["hip::allocate"]     = -1;
-    operand_alias["hip::load_literal"] = -1;
-    operand_alias["@outline"]          = -1;
-    operand_alias["check_context"]     = -1;
-    operand_alias["@literal"]          = -1;
-    operand_alias["@param"]            = -1;
-    operand_alias["transpose"]         = 0;
-    operand_alias["flatten"]           = 0;
-    operand_alias["broadcast"]         = 0;
-    operand_alias["reshape"]           = 0;
-    operand_alias["pass"]              = 0;
-    operand_alias["scalar"]            = 0;
-}
-
 void memory_coloring_impl::rewrite()
 {
     std::vector<std::size_t> dims;
@@ -247,37 +222,6 @@ void memory_coloring_impl::rewrite()
     MIGRAPH_DEBUG(dump_program());
 }
 
-#ifdef MIGRAPH_DEBUG_OPT
-
-void memory_coloring_impl::dump(const std::string& str) { std::cout << str << std::endl; }
-
-void memory_coloring_impl::dump_program() { std::cout << *p_program << std::endl; }
-
-void memory_coloring_impl::dump_intervals()
-{
-    if(num_of_lives > 0)
-    {
-        std::cout << "---live intervals ---" << std::endl;
-        for(int i = 0; i < num_of_lives; ++i)
-        {
-            live_interval& interval = live_intervals[i];
-            interval.dump();
-        }
-        std::cout << "---conflict table---" << std::endl;
-        for(int i = 0; i <= max_value_number; ++i)
-        {
-            std::cout << " segment:" << i;
-            std::cout << " =>";
-            std::set<int>& table = conflict_table[i];
-            for(auto& iter : table)
-            {
-                std::cout << (iter) << ",";
-            }
-        }
-        std::cout << std::endl;
-    }
-}
-
 void memory_coloring_impl::verify()
 {
     if(num_of_lives > 0)
@@ -289,7 +233,9 @@ void memory_coloring_impl::verify()
 
             if(segment.begin == invalid_offset)
             {
-                assert(interval.is_live_on_entry);
+                // TODO: This check breaks on the tests
+                // if(!interval.is_live_on_entry)
+                // MIGRAPH_THROW("interval is not live on entry");
                 continue;
             }
 
@@ -307,13 +253,44 @@ void memory_coloring_impl::verify()
                     if(range->offset == invalid_offset)
                         continue;
                     if(!is_disjoin(*range, segment))
-                        assert(false);
+                        MIGRAPH_THROW("range and segment is not disjoined");
                 }
             }
         }
     }
 }
 
+#ifdef MIGRAPH_DEBUG_OPT
+
+void memory_coloring_impl::dump(const std::string& str) { std::cout << str << std::endl; }
+
+void memory_coloring_impl::dump_program() { std::cout << *p_program << std::endl; }
+
+void memory_coloring_impl::dump_intervals()
+{
+    if(num_of_lives > 0)
+    {
+        std::cout << "---live intervals ---" << std::endl;
+        for(int i = 0; i < num_of_lives; ++i)
+        {
+            live_interval& interval = live_intervals[i];
+            interval.dump();
+        }
+        std::cout << "---conflict table---" << std::endl;
+        for(int i = 0; i <= max_value_number; ++i)
+        {
+            std::cout << " segment:" << i;
+            std::cout << " =>";
+            std::set<int>& table = conflict_table[i];
+            for(auto& iter : table)
+            {
+                std::cout << (iter) << ",";
+            }
+        }
+        std::cout << std::endl;
+    }
+}
+
 // map liveness tracking point to instruction enum.
 static int get_ins_enum(int x)
 {
@@ -356,4 +333,6 @@ void live_interval::dump()
 }
 
 #endif
+
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph

src/opt/memory_coloring_impl.hpp

 #ifndef MIGRAPH_GUARD_RTGLIB_MEMORY_COLORING_IMPL_HPP
 #define MIGRAPH_GUARD_RTGLIB_MEMORY_COLORING_IMPL_HPP
 #include "common_header.hpp"
+#include <migraph/config.hpp>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 static const int invalid_offset = -1;
 
@@ -50,16 +52,15 @@ using interval_ptr = live_interval*;
 
 struct memory_coloring_impl
 {
-    memory_coloring_impl(program* p, std::string alloc_op)
-        : p_program(p), allocation_op(std::move(alloc_op))
+    memory_coloring_impl(program* p, std::string alloc_op, bool p_verify)
+        : p_program(p), allocation_op(std::move(alloc_op)), enable_verify(p_verify)
     {
         instr2_live.clear();
         live_ranges.clear();
         conflict_table.clear();
-        num_of_lives     = 0;
-        max_value_number = -1;
-        required_bytes   = 0;
-        operand_alias.clear();
+        num_of_lives       = 0;
+        max_value_number   = -1;
+        required_bytes     = 0;
         earliest_end_point = -1;
         latest_end_point   = -1;
         unify_literals     = false;
@@ -75,7 +76,6 @@ struct memory_coloring_impl
     }
     void build();
     void run();
-    void register_operand_alias();
     void rewrite();
 
     private:
@@ -92,31 +92,6 @@ struct memory_coloring_impl
         return ins->name() == "check_context";
     }
 
-    // get operand alias info.  This is a temporary workaround.
-    int get_input_tie_ndx(const instruction_ref ins)
-    {
-        std::string name = ins->name();
-        if(operand_alias.find(name) != operand_alias.end())
-            return operand_alias[name];
-        if(is_allocate(ins))
-        {
-            // This happens to custom allocators.
-            operand_alias[name] = -1;
-            return -1;
-        }
-        int cnt           = -1;
-        int last_allocate = -1;
-        for(auto&& arg : ins->inputs())
-        {
-            cnt++;
-            if(is_allocate(arg) || is_output_param(arg))
-                last_allocate = cnt;
-        }
-        assert(last_allocate != -1);
-        operand_alias[name] = last_allocate;
-        return last_allocate;
-    }
-#ifdef MIGRAPH_DEBUG_OPT
     static bool is_disjoin(live_range& range1, live_range& range2)
     {
         if((range1.size == 0) || (range2.size == 0))
@@ -125,10 +100,11 @@ struct memory_coloring_impl
         long long end2 = range2.offset + range2.size - 1;
         return ((end1 < range2.offset) || (end2 < range1.offset));
     }
+    void verify();
+#ifdef MIGRAPH_DEBUG_OPT
     void dump(const std::string&);
     void dump_program();
     void dump_intervals();
-    void verify();
 #endif
     struct ordering
     {
@@ -164,7 +140,6 @@ struct memory_coloring_impl
     std::unordered_map<int, std::set<int>> conflict_table;
     // Priority queue for coloring.
     std::priority_queue<interval_ptr, std::vector<interval_ptr>, ordering> alloc_queue;
-    std::unordered_map<std::string, int> operand_alias;
 
     int num_of_lives;
     int max_value_number;
@@ -176,6 +151,9 @@ struct memory_coloring_impl
     // Whether to unify literals into coloring.
     bool unify_literals;
     std::string allocation_op{};
+    bool enable_verify;
 };
+
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph
 #endif

src/program.cpp

@@ -2,6 +2,7 @@
 #include <migraph/stringutils.hpp>
 #include <migraph/instruction.hpp>
 #include <migraph/env.hpp>
+#include <migraph/ranges.hpp>
 #include <migraph/time.hpp>
 #include <migraph/iterator_for.hpp>
 #include <iostream>
@@ -10,6 +11,7 @@
 #include <utility>
 
 namespace migraph {
+inline namespace MIGRAPH_INLINE_NS {
 
 MIGRAPH_DECLARE_ENV_VAR(MIGRAPH_TRACE_COMPILE)
 MIGRAPH_DECLARE_ENV_VAR(MIGRAPH_TRACE_EVAL)
@@ -280,7 +282,7 @@ void program::compile(const target& t, tracer trace)
 {
     assert(this->validate() == impl->instructions.end());
     this->impl->ctx = t.get_context();
-    if(not trace.enabled() and enabled(MIGRAPH_TRACE_COMPILE{}))
+    if(enabled(MIGRAPH_TRACE_COMPILE{}))
         trace = tracer{std::cout};
     trace(*this);
     trace();
@@ -329,8 +331,11 @@ argument generic_eval(const program& p,
         else if(ins->name() == "@param")
         {
             results.emplace(ins, trace(ins, [&] {
-                                return params.at(
-                                    any_cast<builtin::param>(ins->get_operator()).parameter);
+                                auto param_name =
+                                    any_cast<builtin::param>(ins->get_operator()).parameter;
+                                if(not contains(params, param_name))
+                                    MIGRAPH_THROW("Parameter not found: " + param_name);
+                                return params.at(param_name);
                             }));
         }
         else if(ins->name() == "@outline")
@@ -494,4 +499,6 @@ std::ostream& operator<<(std::ostream& os, const program& p)
     print_program(os, p, [](auto&&...) {});
     return os;
 }
+
+} // namespace MIGRAPH_INLINE_NS
 } // namespace migraph