Merge branch 'develop' into type-string-driver

src/file_buffer.cpp

+#include <migraphx/file_buffer.hpp>
+#include <migraphx/errors.hpp>
+#include <fstream>
+#include <iostream>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+template <class T>
+T generic_read_file(const std::string& filename)
+{
+    std::ifstream is(filename, std::ios::binary | std::ios::ate);
+    std::streamsize size = is.tellg();
+    if(size < 1)
+        MIGRAPHX_THROW("Invalid size for: " + filename);
+    is.seekg(0, std::ios::beg);
+
+    T buffer(size, 0);
+    if(!is.read(&buffer[0], size))
+        MIGRAPHX_THROW("Error reading file: " + filename);
+    return buffer;
+}
+
+std::vector<char> read_buffer(const std::string& filename)
+{
+    return generic_read_file<std::vector<char>>(filename);
+}
+
+std::string read_string(const std::string& filename)
+{
+    return generic_read_file<std::string>(filename);
+}
+
+void write_buffer(const std::string& filename, const char* buffer, std::size_t size)
+{
+    std::ofstream os(filename);
+    os.write(buffer, size);
+}
+void write_buffer(const std::string& filename, const std::vector<char>& buffer)
+{
+    write_buffer(filename, buffer.data(), buffer.size());
+}
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx

src/fuse_pointwise.cpp

+#include <migraphx/fuse_pointwise.hpp>
+#include <migraphx/pass_manager.hpp>
+#include <migraphx/dead_code_elimination.hpp>
+#include <migraphx/instruction.hpp>
+#include <migraphx/program.hpp>
+#include <migraphx/make_op.hpp>
+#include <migraphx/iterator_for.hpp>
+#include <migraphx/ranges.hpp>
+#include <iterator>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+static literal get_scalar(instruction_ref ins)
+{
+    if(ins->name() == "contiguous")
+        return get_scalar(ins->inputs().front());
+    const auto& s = ins->get_shape();
+    if(not(s.elements() == 1 or s.scalar()))
+        return {};
+    if(not ins->can_eval())
+        return {};
+    auto e = ins->eval();
+    literal r{};
+    e.visit_at([&](auto x) { r = literal{x}; });
+    return r;
+}
+
+static void create_pointwise_modules(module_pass_manager& mpm)
+{
+    std::size_t n = 0;
+    for(auto ins : iterator_for(mpm.get_module()))
+    {
+        if(not ins->get_operator().attributes().get("pointwise", false))
+            continue;
+        assert(ins->get_operator().attributes().contains("point_op"));
+        auto* pm = mpm.create_module(mpm.get_module().name() + ":pointwise" + std::to_string(n++));
+        pm->set_bypass();
+
+        std::unordered_map<instruction_ref, instruction_ref> param_map;
+        std::vector<instruction_ref> pointwise_inputs;
+        std::size_t i = 0;
+        for(auto input : ins->inputs())
+        {
+            if(contains(param_map, input))
+                continue;
+            auto scalar = get_scalar(input);
+            if(scalar.empty())
+            {
+                pointwise_inputs.push_back(input);
+                param_map[input] =
+                    pm->add_parameter("x" + std::to_string(i), shape{input->get_shape().type()});
+                i++;
+            }
+            else
+            {
+                param_map[input] = pm->add_literal(scalar);
+            }
+        }
+
+        std::vector<instruction_ref> inputs;
+        std::transform(ins->inputs().begin(),
+                       ins->inputs().end(),
+                       std::back_inserter(inputs),
+                       [&](auto input) { return param_map[input]; });
+        auto r = pm->add_instruction(ins->get_operator(), inputs);
+        pm->add_return({r});
+
+        mpm.get_module().replace_instruction(ins, make_op("pointwise"), pointwise_inputs, {pm});
+    }
+}
+
+static std::vector<instruction_ref> append_pointwise_module(instruction_ref ins,
+                                                            instruction_ref output)
+{
+    assert(contains(output->inputs(), ins));
+    module_ref pm = ins->module_inputs().at(0);
+    module_ref xm = output->module_inputs().at(0);
+
+    auto last = std::prev(pm->end());
+    assert(last->name() == "@return");
+    assert(last->inputs().size() == 1);
+
+    assert(pm->get_parameter_names().size() == ins->inputs().size());
+    assert(xm->get_parameter_names().size() == output->inputs().size());
+
+    std::vector<instruction_ref> inputs = ins->inputs();
+    std::unordered_map<instruction_ref, instruction_ref> map_ins;
+    std::unordered_map<instruction_ref, instruction_ref> input_map;
+    // Copy inputs to input_map
+    for(auto i : range(inputs.size()))
+    {
+        auto input = inputs[i];
+        auto param = pm->get_parameter("x" + std::to_string(i));
+        assert(param != pm->end());
+        input_map[input] = param;
+    }
+    // Add the new parameter and additional inputs
+    for(auto i : range(output->inputs().size()))
+    {
+        auto input = output->inputs()[i];
+        auto param = xm->get_parameter("x" + std::to_string(i));
+        assert(param != xm->end());
+        if(input == ins)
+        {
+            map_ins[param]   = last->inputs().front();
+            input_map[input] = map_ins[param];
+        }
+        // Avoid duplicate paramter inputs
+        else if(contains(input_map, input))
+        {
+            map_ins[param] = input_map[input];
+        }
+        else
+        {
+            map_ins[param] =
+                pm->add_parameter("x" + std::to_string(inputs.size()), {input->get_shape().type()});
+            inputs.push_back(input);
+            input_map[input] = map_ins[param];
+        }
+    }
+    pm->replace_return(pm->insert_module_instructions(last, xm, map_ins));
+    return inputs;
+}
+
+static bool find_pointwise_modules(module& m)
+{
+    bool changed = false;
+    auto last    = std::prev(m.end());
+    for(auto ins : iterator_for(m))
+    {
+        if(ins->name() != "pointwise")
+            continue;
+        if(ins->outputs().empty() and ins != last)
+            continue;
+        auto it = std::find_if(ins->inputs().begin(), ins->inputs().end(), [&](auto i) {
+            return i->name() == "pointwise" and i->outputs().size() == 1;
+        });
+        if(it == ins->inputs().end())
+            continue;
+        auto input = *it;
+
+        auto new_inputs = append_pointwise_module(input, ins);
+        m.replace_instruction(input, input->get_operator(), new_inputs, input->module_inputs());
+        m.replace_instruction(ins, input);
+        m.move_instruction(input, ins);
+
+        changed = true;
+    }
+    return changed;
+}
+
+void fuse_pointwise::apply(module_pass_manager& mpm) const
+{
+    create_pointwise_modules(mpm);
+    mpm.run_pass(dead_code_elimination{});
+    for(int i = 0; i < 8; i++)
+    {
+        if(not find_pointwise_modules(mpm.get_module()))
+            break;
+        mpm.run_pass(dead_code_elimination{});
+    }
+}
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx

src/generate.cpp

@@ -6,22 +6,59 @@ inline namespace MIGRAPHX_INLINE_NS {
 argument fill_argument(shape s, unsigned long value)
 {
     argument result;
-    s.visit_type([&](auto as) {
-        using type = typename decltype(as)::type;
-        auto v     = fill_tensor_data<type>(s, value);
-        result     = {s, v};
-    });
+    if(s.type() == shape::tuple_type)
+    {
+        std::vector<argument> sub_args;
+        const auto& sub_ss = s.sub_shapes();
+        std::transform(sub_ss.begin(), sub_ss.end(), std::back_inserter(sub_args), [&](auto ss) {
+            return fill_argument(ss, value);
+        });
+
+        result = argument(sub_args);
+    }
+    else
+    {
+        s.visit_type([&](auto as) {
+            using type = typename decltype(as)::type;
+            auto v     = fill_tensor_data<type>(s, value);
+            result     = {s, v};
+        });
+    }
     return result;
 }
 
 argument generate_argument(shape s, unsigned long seed)
 {
     argument result;
-    s.visit_type([&](auto as) {
-        using type = typename decltype(as)::type;
-        auto v     = generate_tensor_data<type>(s, seed);
-        result     = {s, v};
-    });
+    if(s.type() == shape::tuple_type)
+    {
+        const auto& sub_ss = s.sub_shapes();
+        std::vector<argument> sub_args;
+        std::transform(sub_ss.begin(), sub_ss.end(), std::back_inserter(sub_args), [&](auto ss) {
+            return generate_argument(ss, seed);
+        });
+
+        result = argument(sub_args);
+    }
+    else
+    {
+        s.visit_type([&](auto as) {
+            // we use char type to store bool type internally, so bool_type
+            // needs special processing to generate data
+            if(s.type() == shape::bool_type)
+            {
+                auto v = generate_tensor_data<bool>(s, seed);
+                result = {s, v};
+            }
+            else
+            {
+                using type = typename decltype(as)::type;
+                auto v     = generate_tensor_data<type>(s, seed);
+                result     = {s, v};
+            }
+        });
+    }
+
     return result;
 }
 

src/targets/gpu/include/migraphx/gpu/adjust_allocation.hpp → src/include/migraphx/adjust_allocation.hpp

 #ifndef MIGRAPHX_GUARD_RTGLIB_ADJUST_ALLOCATION_HPP
 #define MIGRAPHX_GUARD_RTGLIB_ADJUST_ALLOCATION_HPP
 
-#include <migraphx/program.hpp>
 #include <migraphx/config.hpp>
-#include <migraphx/gpu/context.hpp>
+#include <migraphx/allocation_model.hpp>
 
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 
-namespace gpu {
+struct module;
 
 struct adjust_allocation
 {
-    std::string name() const { return "gpu::adjust_allocation"; }
-    void apply(program& p) const;
+    allocation_model model;
+    std::string name() const { return "adjust_allocation"; }
+    void apply(module& m) const;
 };
 
-} // namespace gpu
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx
 

src/include/migraphx/algorithm.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_ALGORITHM_HPP
+#define MIGRAPHX_GUARD_RTGLIB_ALGORITHM_HPP
+
+#include <algorithm>
+#include <numeric>
+#include <migraphx/config.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+template <class Iterator, class Output, class Predicate, class F>
+void transform_if(Iterator start, Iterator last, Output out, Predicate pred, F f)
+{
+    while(start != last)
+    {
+        if(pred(*start))
+        {
+            *out = f(*start);
+            ++out;
+        }
+        ++start;
+    }
+}
+
+template <class Iterator, class T, class BinaryOp, class UnaryOp>
+T transform_accumulate(Iterator first, Iterator last, T init, BinaryOp binop, UnaryOp unaryop)
+{
+    return std::inner_product(
+        first, last, first, init, binop, [&](auto&& x, auto&&) { return unaryop(x); });
+}
+
+template <class Iterator, class Output, class Predicate>
+void group_by(Iterator start, Iterator last, Output out, Predicate pred)
+{
+    while(start != last)
+    {
+        auto it = std::partition(start, last, [&](auto&& x) { return pred(x, *start); });
+        out(start, it);
+        start = it;
+    }
+}
+
+template <class Iterator, class Output, class Predicate>
+void group_unique(Iterator start, Iterator last, Output out, Predicate pred)
+{
+    while(start != last)
+    {
+        auto it = std::find_if(start, last, [&](auto&& x) { return not pred(*start, x); });
+        out(start, it);
+        start = it;
+    }
+}
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/allocation_model.hpp

+#ifndef MIGRAPHX_GUARD_ALLOCATION_MODEL_HPP
+#define MIGRAPHX_GUARD_ALLOCATION_MODEL_HPP
+
+#include <cassert>
+#include <string>
+#include <functional>
+#include <memory>
+#include <type_traits>
+#include <utility>
+
+#include <migraphx/config.hpp>
+#include <migraphx/operation.hpp>
+#include <vector>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+#ifdef DOXYGEN
+
+/// An interface for target-dependent allocation
+struct allocation_model
+{
+    /// A name of the target-dependent allocate operator
+    std::string name() const;
+    /// A name of the target-dependent copy operator
+    std::string copy() const;
+    /// Create an allocation operator for the given shape
+    operation allocate(const shape& s) const;
+    /// Create a preallocated operator for the given shape
+    operation preallocate(const shape& s, const std::string& id) const;
+    /// Check if outputs are to be inserted
+    bool needs_out_params() const;
+};
+
+#else
+
+#ifdef TYPE_ERASED_DECLARATION
+
+// Type-erased interface for:
+struct allocation_model
+{
+    //
+    std::string name() const;
+    //
+    std::string copy() const;
+    //
+    operation allocate(const shape& s) const;
+    //
+    operation preallocate(const shape& s, std::string id) const;
+    //
+    bool needs_out_params() const;
+};
+
+#else
+
+struct allocation_model
+{
+    // Constructors
+    allocation_model() = default;
+
+    template <typename PrivateDetailTypeErasedT>
+    allocation_model(PrivateDetailTypeErasedT value)
+        : private_detail_te_handle_mem_var(
+              std::make_shared<private_detail_te_handle_type<
+                  typename std::remove_reference<PrivateDetailTypeErasedT>::type>>(
+                  std::forward<PrivateDetailTypeErasedT>(value)))
+    {
+    }
+
+    // Assignment
+    template <typename PrivateDetailTypeErasedT>
+    allocation_model& operator=(PrivateDetailTypeErasedT value)
+    {
+        using std::swap;
+        auto* derived = this->any_cast<PrivateDetailTypeErasedT>();
+        if(derived and private_detail_te_handle_mem_var.unique())
+        {
+            *derived = std::forward<PrivateDetailTypeErasedT>(value);
+        }
+        else
+        {
+            allocation_model rhs(value);
+            swap(private_detail_te_handle_mem_var, rhs.private_detail_te_handle_mem_var);
+        }
+        return *this;
+    }
+
+    // Cast
+    template <typename PrivateDetailTypeErasedT>
+    PrivateDetailTypeErasedT* any_cast()
+    {
+        return this->type_id() == typeid(PrivateDetailTypeErasedT)
+                   ? std::addressof(static_cast<private_detail_te_handle_type<
+                                        typename std::remove_cv<PrivateDetailTypeErasedT>::type>&>(
+                                        private_detail_te_get_handle())
+                                        .private_detail_te_value)
+                   : nullptr;
+    }
+
+    template <typename PrivateDetailTypeErasedT>
+    const typename std::remove_cv<PrivateDetailTypeErasedT>::type* any_cast() const
+    {
+        return this->type_id() == typeid(PrivateDetailTypeErasedT)
+                   ? std::addressof(static_cast<const private_detail_te_handle_type<
+                                        typename std::remove_cv<PrivateDetailTypeErasedT>::type>&>(
+                                        private_detail_te_get_handle())
+                                        .private_detail_te_value)
+                   : nullptr;
+    }
+
+    const std::type_info& type_id() const
+    {
+        if(private_detail_te_handle_empty())
+            return typeid(std::nullptr_t);
+        else
+            return private_detail_te_get_handle().type();
+    }
+
+    std::string name() const
+    {
+        assert((*this).private_detail_te_handle_mem_var);
+        return (*this).private_detail_te_get_handle().name();
+    }
+
+    std::string copy() const
+    {
+        assert((*this).private_detail_te_handle_mem_var);
+        return (*this).private_detail_te_get_handle().copy();
+    }
+
+    operation allocate(const shape& s) const
+    {
+        assert((*this).private_detail_te_handle_mem_var);
+        return (*this).private_detail_te_get_handle().allocate(s);
+    }
+
+    operation preallocate(const shape& s, std::string id) const
+    {
+        assert((*this).private_detail_te_handle_mem_var);
+        return (*this).private_detail_te_get_handle().preallocate(s, std::move(id));
+    }
+
+    bool needs_out_params() const
+    {
+        assert((*this).private_detail_te_handle_mem_var);
+        return (*this).private_detail_te_get_handle().needs_out_params();
+    }
+
+    friend bool is_shared(const allocation_model& private_detail_x,
+                          const allocation_model& private_detail_y)
+    {
+        return private_detail_x.private_detail_te_handle_mem_var ==
+               private_detail_y.private_detail_te_handle_mem_var;
+    }
+
+    private:
+    struct private_detail_te_handle_base_type
+    {
+        virtual ~private_detail_te_handle_base_type() {}
+        virtual std::shared_ptr<private_detail_te_handle_base_type> clone() const = 0;
+        virtual const std::type_info& type() const                                = 0;
+
+        virtual std::string name() const                                    = 0;
+        virtual std::string copy() const                                    = 0;
+        virtual operation allocate(const shape& s) const                    = 0;
+        virtual operation preallocate(const shape& s, std::string id) const = 0;
+        virtual bool needs_out_params() const                               = 0;
+    };
+
+    template <typename PrivateDetailTypeErasedT>
+    struct private_detail_te_handle_type : private_detail_te_handle_base_type
+    {
+        template <typename PrivateDetailTypeErasedU = PrivateDetailTypeErasedT>
+        private_detail_te_handle_type(
+            PrivateDetailTypeErasedT value,
+            typename std::enable_if<std::is_reference<PrivateDetailTypeErasedU>::value>::type* =
+                nullptr)
+            : private_detail_te_value(value)
+        {
+        }
+
+        template <typename PrivateDetailTypeErasedU = PrivateDetailTypeErasedT>
+        private_detail_te_handle_type(
+            PrivateDetailTypeErasedT value,
+            typename std::enable_if<!std::is_reference<PrivateDetailTypeErasedU>::value,
+                                    int>::type* = nullptr) noexcept
+            : private_detail_te_value(std::move(value))
+        {
+        }
+
+        std::shared_ptr<private_detail_te_handle_base_type> clone() const override
+        {
+            return std::make_shared<private_detail_te_handle_type>(private_detail_te_value);
+        }
+
+        const std::type_info& type() const override { return typeid(private_detail_te_value); }
+
+        std::string name() const override { return private_detail_te_value.name(); }
+
+        std::string copy() const override { return private_detail_te_value.copy(); }
+
+        operation allocate(const shape& s) const override
+        {
+
+            return private_detail_te_value.allocate(s);
+        }
+
+        operation preallocate(const shape& s, std::string id) const override
+        {
+
+            return private_detail_te_value.preallocate(s, std::move(id));
+        }
+
+        bool needs_out_params() const override
+        {
+
+            return private_detail_te_value.needs_out_params();
+        }
+
+        PrivateDetailTypeErasedT private_detail_te_value;
+    };
+
+    template <typename PrivateDetailTypeErasedT>
+    struct private_detail_te_handle_type<std::reference_wrapper<PrivateDetailTypeErasedT>>
+        : private_detail_te_handle_type<PrivateDetailTypeErasedT&>
+    {
+        private_detail_te_handle_type(std::reference_wrapper<PrivateDetailTypeErasedT> ref)
+            : private_detail_te_handle_type<PrivateDetailTypeErasedT&>(ref.get())
+        {
+        }
+    };
+
+    bool private_detail_te_handle_empty() const
+    {
+        return private_detail_te_handle_mem_var == nullptr;
+    }
+
+    const private_detail_te_handle_base_type& private_detail_te_get_handle() const
+    {
+        assert(private_detail_te_handle_mem_var != nullptr);
+        return *private_detail_te_handle_mem_var;
+    }
+
+    private_detail_te_handle_base_type& private_detail_te_get_handle()
+    {
+        assert(private_detail_te_handle_mem_var != nullptr);
+        if(!private_detail_te_handle_mem_var.unique())
+            private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone();
+        return *private_detail_te_handle_mem_var;
+    }
+
+    std::shared_ptr<private_detail_te_handle_base_type> private_detail_te_handle_mem_var;
+};
+
+template <typename ValueType>
+inline const ValueType* any_cast(const allocation_model* x)
+{
+    return x->any_cast<ValueType>();
+}
+
+template <typename ValueType>
+inline ValueType* any_cast(allocation_model* x)
+{
+    return x->any_cast<ValueType>();
+}
+
+template <typename ValueType>
+inline ValueType& any_cast(allocation_model& x)
+{
+    auto* y = x.any_cast<typename std::remove_reference<ValueType>::type>();
+    if(y == nullptr)
+        throw std::bad_cast();
+    return *y;
+}
+
+template <typename ValueType>
+inline const ValueType& any_cast(const allocation_model& x)
+{
+    const auto* y = x.any_cast<typename std::remove_reference<ValueType>::type>();
+    if(y == nullptr)
+        throw std::bad_cast();
+    return *y;
+}
+#endif
+
+#endif
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/analyze_streams.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_ANALYZE_STREAMS_HPP
+#define MIGRAPHX_GUARD_RTGLIB_ANALYZE_STREAMS_HPP
+
+#include <migraphx/config.hpp>
+#include <migraphx/stream_model.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+struct module;
+
+struct stream_race
+{
+    instruction_ref ins;
+    // The instruction that should before
+    instruction_ref before;
+};
+
+std::vector<stream_race> analyze_streams(const module& m, const stream_model& strmm);
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/any_ptr.hpp

+#ifndef MIGRAPHX_GUARD_MIGRAPHX_ANY_PTR_HPP
+#define MIGRAPHX_GUARD_MIGRAPHX_ANY_PTR_HPP
+
+#include <migraphx/config.hpp>
+#include <migraphx/optional.hpp>
+#include <migraphx/errors.hpp>
+#include <migraphx/type_name.hpp>
+#include <cassert>
+#include <string_view>
+#include <typeindex>
+#include <type_traits>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+struct any_ptr
+{
+    any_ptr() = default;
+    template <class T>
+    any_ptr(T* p) : ptr(p), ti(typeid(T*)), name(get_name<T*>())
+    {
+    }
+
+    any_ptr(void* p, std::string_view pname) : ptr(p), name(pname) {}
+
+    void* get(std::string_view n) const
+    {
+        if(name != n)
+            MIGRAPHX_THROW("any_ptr: type mismatch: " + std::string{name} +
+                           " != " + std::string{n});
+        return ptr;
+    }
+
+    template <class T>
+    T get() const
+    {
+        static_assert(std::is_pointer<T>{}, "Must be a pointer");
+        assert(ptr != nullptr);
+        if(ti and std::type_index{typeid(T)} != *ti)
+            MIGRAPHX_THROW("any_ptr: type mismatch: " + std::string{name} + " != " + get_name<T>());
+        else if(name != get_name<T>())
+            MIGRAPHX_THROW("any_ptr: type mismatch: " + std::string{name} + " != " + get_name<T>());
+        return reinterpret_cast<T>(ptr);
+    }
+    void* unsafe_get() const { return ptr; }
+
+    private:
+    void* ptr                    = nullptr;
+    optional<std::type_index> ti = nullopt;
+    std::string_view name        = "";
+
+    template <class T>
+    static const std::string& get_name()
+    {
+        return get_type_name<std::remove_cv_t<std::remove_pointer_t<T>>>();
+    }
+};
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+#endif // MIGRAPHX_GUARD_MIGRAPHX_ANY_PTR_HPP

src/include/migraphx/apply_alpha_beta.hpp

+#ifndef MIGRAPHX_GUARD_MIGRAPHX_APPLY_ALPHA_BETA_HPP
+#define MIGRAPHX_GUARD_MIGRAPHX_APPLY_ALPHA_BETA_HPP
+
+#include "migraphx/make_op.hpp"
+#include "migraphx/normalize_attributes.hpp"
+#include "migraphx/operation.hpp"
+#include <migraphx/instruction_ref.hpp>
+#include <migraphx/module.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+instruction_ref insert_apply_alpha_beta(module& m,
+                                        instruction_ref pos,
+                                        const std::vector<instruction_ref>& args,
+                                        const operation& op,
+                                        const literal& alpha,
+                                        const literal& beta);
+
+template <typename T = float>
+instruction_ref insert_apply_alpha_beta(module& m,
+                                        instruction_ref pos,
+                                        const std::vector<instruction_ref>& args,
+                                        const operation& op,
+                                        T alpha = 1,
+                                        T beta  = 0)
+{
+    return insert_apply_alpha_beta(m, pos, args, op, literal{T{alpha}}, literal{T{beta}});
+}
+
+template <typename T = float>
+instruction_ref add_apply_alpha_beta(module& m,
+                                     const std::vector<instruction_ref>& args,
+                                     const operation& op,
+                                     T alpha = 1,
+                                     T beta  = 0)
+{
+    return insert_apply_alpha_beta(m, m.end(), args, op, alpha, beta);
+}
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+#endif // MIGRAPHX_GUARD_APPLY_ALPHA_BETA_HPP

src/include/migraphx/argument.hpp

@@ -4,9 +4,11 @@
 #include <migraphx/shape.hpp>
 #include <migraphx/raw_data.hpp>
 #include <migraphx/config.hpp>
+#include <migraphx/make_shared_array.hpp>
 #include <functional>
 #include <utility>
 
+// clang-format off
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 
@@ -19,61 +21,74 @@ inline namespace MIGRAPHX_INLINE_NS {
  */
 struct argument : raw_data<argument>
 {
-    argument() {}
+    argument() = default;
 
-    argument(const shape& s) : m_shape(s)
-    {
-        std::vector<char> buffer(s.bytes());
-        // TODO: Move vector
-        data = [=]() mutable { return buffer.data(); };
-    }
+    argument(const shape& s);
 
     template <class F, MIGRAPHX_REQUIRES(std::is_pointer<decltype(std::declval<F>()())>{})>
     argument(shape s, F d)
-        : data([f = std::move(d)]() mutable { return reinterpret_cast<char*>(f()); }),
-          m_shape(std::move(s))
+        : m_shape(std::move(s))
+
     {
+        assign_buffer([f = std::move(d)]() mutable { return reinterpret_cast<char*>(f()); });
     }
     template <class T>
     argument(shape s, T* d)
-        : data([d] { return reinterpret_cast<char*>(d); }), m_shape(std::move(s))
+        : m_shape(std::move(s))
     {
+        assign_buffer([d] { return reinterpret_cast<char*>(d); });
     }
 
     template <class T>
     argument(shape s, std::shared_ptr<T> d)
-        : data([d] { return reinterpret_cast<char*>(d.get()); }), m_shape(std::move(s))
+        : m_shape(std::move(s))
     {
+        assign_buffer([d] { return reinterpret_cast<char*>(d.get()); });
     }
 
-    argument(shape s, std::nullptr_t) : data([] { return nullptr; }), m_shape(std::move(s)) {}
+    argument(shape s, std::nullptr_t);
+    
+    argument(const std::vector<argument>& args);
 
     /// Provides a raw pointer to the data
-    std::function<char*()> data = nullptr;
+    char* data() const;
 
     /// Whether data is available
-    bool empty() const { return not data; }
+    bool empty() const;
 
-    const shape& get_shape() const { return this->m_shape; }
+    const shape& get_shape() const;
 
-    argument reshape(const shape& s) const
-    {
-        argument self = *this;
-        return {s, [=]() mutable { return self.data(); }};
-    }
+    argument reshape(const shape& s) const;
+
+    argument copy() const;
 
     /// Make copy of the argument that is always sharing the data
-    argument share() const
-    {
-        auto self = std::make_shared<argument>(*this);
-        return {m_shape, [self]() mutable { return self->data(); }};
-    }
+    argument share() const;
+
+    std::vector<argument> get_sub_objects() const;
+
+    /// Return the ith element
+    argument element(std::size_t i) const;
 
     private:
+    void assign_buffer(std::function<char*()> d);
+    struct data_t
+    {
+        std::function<char*()> get = nullptr;
+        std::vector<data_t> sub = {};
+        data_t share() const;
+        static data_t from_args(const std::vector<argument>& args);
+    };
+    argument(const shape& s, const data_t& d);
     shape m_shape;
+    data_t m_data{};
 };
 
+void migraphx_to_value(value& v, const argument& a);
+void migraphx_from_value(const value& v, argument& a);
+
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx
+// clang-format on
 
 #endif

src/include/migraphx/assert.hpp

+#ifndef MIGRAPHX_GUARD_MIGRAPHX_ASSERT_HPP
+#define MIGRAPHX_GUARD_MIGRAPHX_ASSERT_HPP
+
+#include <migraphx/config.hpp>
+#include <cstdlib>
+#include <iostream>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+template <class F>
+auto abort_on_throw(F f) -> decltype(f())
+{
+    try
+    {
+        return f();
+    }
+    catch(const std::exception& e)
+    {
+        std::cerr << e.what() << std::endl;
+        std::abort();
+    }
+    catch(...)
+    {
+        std::cerr << "Unknown exception" << std::endl;
+        std::abort();
+    }
+}
+#ifdef NDEBUG
+#define MIGRAPHX_ASSERT_NO_THROW(...) __VA_ARGS__
+#else
+#define MIGRAPHX_ASSERT_NO_THROW(...) \
+    migraphx::abort_on_throw([&]() -> decltype(__VA_ARGS__) { return __VA_ARGS__; })
+#endif
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+#endif // MIGRAPHX_GUARD_MIGRAPHX_ASSERT_HPP

src/include/migraphx/auto_contiguous.hpp

@@ -8,12 +8,12 @@
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 
-struct program;
+struct module;
 
 struct auto_contiguous
 {
     std::string name() const { return "auto_contiguous"; }
-    void apply(program& p) const;
+    void apply(module& m) const;
 };
 
 } // namespace MIGRAPHX_INLINE_NS

src/include/migraphx/auto_register.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_AUTO_REGISTER_HPP
+#define MIGRAPHX_GUARD_RTGLIB_AUTO_REGISTER_HPP
+
+#include <migraphx/config.hpp>
+#include <type_traits>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+template <class Action, class T>
+int auto_register_action()
+{
+    Action::template apply<T>();
+    return 0;
+}
+
+template <class Action, class T>
+struct auto_register
+{
+    const static int static_register;
+    // This typedef ensures that the static member will be instantiated if
+    // the class itself is instantiated
+    using static_register_type =
+        std::integral_constant<decltype(&static_register), &static_register>;
+};
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wglobal-constructors"
+#endif
+
+template <class Action, class T>
+const int auto_register<Action, T>::static_register = auto_register_action<Action, T>(); // NOLINT
+
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
+#define MIGRAPHX_AUTO_REGISTER_NAME_DETAIL(x) migraphx_auto_register_##x
+#define MIGRAPHX_AUTO_REGISTER_NAME(x) MIGRAPHX_AUTO_REGISTER_NAME_DETAIL(x)
+// NOLINTNEXTLINE
+#define MIGRAPHX_AUTO_REGISTER(...)                                                        \
+    void MIGRAPHX_AUTO_REGISTER_NAME(__LINE__)(migraphx::auto_register<__VA_ARGS__> x =    \
+                                                   migraphx::auto_register<__VA_ARGS__>{}) \
+        __attribute__((unused));
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/builtin.hpp

@@ -43,6 +43,7 @@ struct outline
 struct param
 {
     std::string parameter;
+    uint32_t order = 0;
 
     template <class Self, class F>
     static auto reflect(Self& self, F f)
@@ -63,6 +64,16 @@ struct param
     }
 };
 
+struct returns
+{
+    std::string name() const { return "@return"; }
+    shape compute_shape(const std::vector<shape>&) const { return {}; }
+    argument compute(context&, const shape&, const std::vector<argument>&) const
+    {
+        MIGRAPHX_THROW("builtin");
+    }
+};
+
 } // namespace builtin
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx

src/include/migraphx/check_context.hpp

@@ -3,6 +3,7 @@
 
 #include <migraphx/program.hpp>
 #include <migraphx/config.hpp>
+#include <migraphx/register_op.hpp>
 
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
@@ -10,9 +11,9 @@ inline namespace MIGRAPHX_INLINE_NS {
 template <class T>
 struct check_context
 {
-    struct op
+    struct op : auto_register_op<op>
     {
-        std::string name() const { return "check_context"; }
+        std::string name() const { return "check_context::" + get_type_name<T>(); }
         shape compute_shape(const std::vector<shape>&) const { return {}; }
         argument compute(context& ctx, const shape&, const std::vector<argument>&) const
         {
@@ -32,7 +33,7 @@ struct check_context
     };
 
     std::string name() const { return "check_context"; }
-    void apply(program& p) const { p.insert_instruction(p.begin(), op{}); }
+    void apply(module& m) const { m.insert_instruction(m.begin(), op{}); }
 };
 
 } // namespace MIGRAPHX_INLINE_NS

src/include/migraphx/check_shapes.hpp

@@ -25,8 +25,6 @@ struct check_shapes
     {
     }
 
-    check_shapes(const std::vector<shape>& s) : begin(s.data()), end(s.data() + s.size()) {}
-
     template <class Op>
     check_shapes(const std::vector<shape>& s, const Op& op)
         : begin(s.data()), end(s.data() + s.size()), name(op.name())
@@ -59,6 +57,13 @@ struct check_shapes
         return *this;
     }
 
+    const check_shapes& nelements(std::size_t n) const
+    {
+        if(!this->all_of([&](const shape& s) { return s.elements() == n; }))
+            MIGRAPHX_THROW(prefix() + "Shapes must have only " + std::to_string(n) + " elements");
+        return *this;
+    }
+
     const check_shapes& only_dims(std::size_t n) const
     {
         assert(begin != nullptr);
@@ -71,6 +76,32 @@ struct check_shapes
         return *this;
     }
 
+    const check_shapes& max_ndims(std::size_t n) const
+    {
+        assert(begin != nullptr);
+        assert(end != nullptr);
+        if(begin != end)
+        {
+            if(begin->lens().size() > n)
+                MIGRAPHX_THROW(prefix() + "Shape must have at most " + std::to_string(n) +
+                               " dimensions");
+        }
+        return *this;
+    }
+
+    const check_shapes& min_ndims(std::size_t n) const
+    {
+        assert(begin != nullptr);
+        assert(end != nullptr);
+        if(begin != end)
+        {
+            if(begin->lens().size() < n)
+                MIGRAPHX_THROW(prefix() + "Shape must have at least " + std::to_string(n) +
+                               " dimensions");
+        }
+        return *this;
+    }
+
     const check_shapes& same_shape() const
     {
         if(!this->same([](const shape& s) { return s; }))
@@ -120,6 +151,20 @@ struct check_shapes
         return *this;
     }
 
+    const check_shapes& packed_or_broadcasted() const
+    {
+        if(!this->all_of([](const shape& s) { return s.packed() or s.broadcasted(); }))
+            MIGRAPHX_THROW(prefix() + "Shapes are not packed nor broadcasted");
+        return *this;
+    }
+
+    const check_shapes& tuple_type() const
+    {
+        if(!this->all_of([](const shape& s) { return s.type() == shape::tuple_type; }))
+            MIGRAPHX_THROW(prefix() + "Shapes are not tuple!");
+        return *this;
+    }
+
     const check_shapes& not_transposed() const
     {
         if(!this->all_of([](const shape& s) { return not s.transposed(); }))
@@ -141,6 +186,13 @@ struct check_shapes
         return *this;
     }
 
+    const check_shapes& batch_not_transposed() const
+    {
+        if(!this->all_of([&](const shape& s) { return batch_not_transposed_strides(s.strides()); }))
+            MIGRAPHX_THROW(prefix() + "Batch size is transposed");
+        return *this;
+    }
+
     template <class F>
     bool same(F f) const
     {
@@ -162,7 +214,7 @@ struct check_shapes
         return std::all_of(begin, end, p);
     }
 
-    const shape* get(long i)
+    const shape* get(long i) const
     {
         if(i >= size())
             MIGRAPHX_THROW(prefix() + "Accessing shape out of bounds");
@@ -173,9 +225,31 @@ struct check_shapes
         return begin + i;
     }
 
-    check_shapes slice(long start) { return {get(start), end, name}; }
+    check_shapes slice(long start) const { return {get(start), end, name}; }
+
+    check_shapes slice(long start, long last) const { return {get(start), get(last), name}; }
 
-    check_shapes slice(long start, long last) { return {get(start), get(last), name}; }
+    private:
+    static bool batch_not_transposed_strides(const std::vector<std::size_t>& strides)
+    {
+        if(strides.size() <= 2)
+            return true;
+        auto dim_0       = strides.size() - 2;
+        auto matrix_size = std::max(strides[dim_0], strides[dim_0 + 1]);
+        std::vector<std::size_t> batch(strides.begin(), strides.begin() + dim_0);
+        if(std::all_of(batch.begin(), batch.end(), [&](auto i) { return (i < matrix_size); }))
+        {
+            return false;
+        }
+
+        if(std::adjacent_find(batch.begin(), batch.end(), [&](auto i, auto j) {
+               return (i < j or i < matrix_size or j < matrix_size);
+           }) != batch.end())
+        {
+            return false;
+        }
+        return true;
+    }
 };
 
 } // namespace MIGRAPHX_INLINE_NS

src/include/migraphx/clamp.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_CLAMP_HPP
+#define MIGRAPHX_GUARD_RTGLIB_CLAMP_HPP
+
+#include <migraphx/config.hpp>
+#include <migraphx/float_equal.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+template <class U, class T>
+U pad_clamp(T x)
+{
+    if(float_equal(x, std::numeric_limits<T>::lowest()))
+        return std::numeric_limits<U>::lowest();
+    if(float_equal(x, std::numeric_limits<T>::max()))
+        return std::numeric_limits<U>::max();
+    return (x < std::numeric_limits<U>::lowest())
+               ? std::numeric_limits<U>::lowest()
+               : (std::numeric_limits<U>::max() < x) ? std::numeric_limits<U>::max() : U(x);
+}
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/cloneable.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_CLONEABLE_HPP
+#define MIGRAPHX_GUARD_RTGLIB_CLONEABLE_HPP
+
+#include <migraphx/config.hpp>
+#include <memory>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+template <typename Base>
+struct cloneable
+{
+    friend Base;
+
+    virtual std::shared_ptr<Base> clone() = 0;
+
+    template <typename Derived>
+    struct derive : Base
+    {
+        friend Derived;
+
+        std::shared_ptr<Base> clone() override
+        {
+            return std::make_shared<Derived>(static_cast<const Derived&>(*this));
+        }
+        template <typename... Args>
+        derive(Args&&... args) : Base(std::forward<Args>(args)...)
+        {
+        }
+    };
+
+    struct share : Base, std::enable_shared_from_this<Base>
+    {
+        std::shared_ptr<Base> clone() override { return this->shared_from_this(); }
+        template <typename... Args>
+        share(Args&&... args) : Base(std::forward<Args>(args)...)
+        {
+        }
+    };
+    cloneable()                 = default;
+    cloneable(const cloneable&) = default;
+    cloneable& operator=(const cloneable&) = default;
+    virtual ~cloneable() {}
+};
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/common.hpp

+#ifndef MIGRAPHX_GUARD_MIGRAPHX_COMMON_HPP
+#define MIGRAPHX_GUARD_MIGRAPHX_COMMON_HPP
+
+#include <migraphx/config.hpp>
+#include <migraphx/shape.hpp>
+#include <migraphx/instruction_ref.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+struct module;
+struct operation;
+
+std::vector<std::size_t> compute_broadcasted_lens(std::vector<std::size_t> s0,
+                                                  std::vector<std::size_t> s1);
+shape common_shape(const std::vector<shape>& shapes);
+
+instruction_ref insert_common_op(module& m,
+                                 instruction_ref ins,
+                                 const operation& op,
+                                 std::vector<instruction_ref> inputs);
+instruction_ref add_common_op(module& m, const operation& op, std::vector<instruction_ref> inputs);
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+#endif // MIGRAPHX_GUARD_MIGRAPHX_COMMON_HPP

src/include/migraphx/compile_options.hpp

@@ -10,6 +10,7 @@ inline namespace MIGRAPHX_INLINE_NS {
 struct compile_options
 {
     bool offload_copy = false;
+    bool fast_math    = true;
     tracer trace{};
 };