Merge branch 'develop' into mlir-c

.github/workflows/ci.yaml

@@ -80,10 +80,10 @@ jobs:
       uses: pat-s/always-upload-cache@v2.1.3
       with:
         path: cppcheck-cache
-        key: cppcheck-cache-${{ steps.cache_timestamp.outputs.timestamp }}
+        key: cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-${{ steps.cache_timestamp.outputs.timestamp }}
         restore-keys: |
-            cppcheck-cache-${{ steps.cache_timestamp.outputs.timestamp }}
-            cppcheck-cache-
+            cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-${{ steps.cache_timestamp.outputs.timestamp }}
+            cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-
 
     - name: Build the Docker image
       run: docker build . --file hip-clang.docker --tag migraphx

CMakeLists.txt

@@ -190,6 +190,7 @@ rocm_enable_cppcheck(
         shadowVariable
         unsafeClassDivZero
         definePrefix:*test/include/test.hpp
+        ctuOneDefinitionRuleViolation:*test/*
         useSmartPointer:*src/api/api.cpp
         useSmartPointer:*make_shared_array.hpp
         constParameter:*src/targets/gpu/*.cpp

cppcheck.rules

@@ -154,7 +154,7 @@
 </rule>
 <rule>
     <tokenlist>normal</tokenlist>
-    <pattern><![CDATA[((?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)*) (\w) ; \2 = static_cast < \1 > (\([^()]*(?-1)*[^()]*\)) ;]]></pattern>
+    <pattern><![CDATA[((?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)*) (\w+) ; \2 = static_cast < \1 > (\([^()]*(?-1)*[^()]*\)) ;]]></pattern>
     <message>
         <id>RedundantCast</id>
         <severity>style</severity>
@@ -163,7 +163,7 @@
 </rule>
 <rule>
     <tokenlist>normal</tokenlist>
-    <pattern><![CDATA[auto (\w) ; \1 = static_cast < (?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)* > (\([^()]*(?-1)*[^()]*\)) ;]]></pattern>
+    <pattern><![CDATA[auto (\w+) ; \1 = static_cast < (?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)* > (\([^()]*(?-1)*[^()]*\)) ;]]></pattern>
     <message>
         <id>RedundantCast</id>
         <severity>style</severity>

src/CMakeLists.txt

@@ -26,6 +26,7 @@ add_library(migraphx
     eliminate_pad.cpp
     env.cpp
     file_buffer.cpp
+    fuse_pointwise.cpp
     generate.cpp
     inline_module.cpp
     insert_pad.cpp
@@ -134,6 +135,7 @@ register_migraphx_ops(
     nonzero
     outline
     pad
+    pointwise
     pooling
     pow
     prefix_scan_sum
@@ -199,6 +201,9 @@ target_link_libraries(migraphx PRIVATE -ldl)
 
 target_include_directories(migraphx SYSTEM PUBLIC $<BUILD_INTERFACE:${HALF_INCLUDE_DIR}>)
 
+find_package(Threads)
+target_link_libraries(migraphx PUBLIC Threads::Threads)
+
 find_package(msgpack REQUIRED)
 target_link_libraries(migraphx PRIVATE msgpackc-cxx)
 # Make this available to the tests
@@ -260,6 +265,7 @@ rocm_export_targets(
   TARGETS migraphx::migraphx migraphx_all_targets
   NAMESPACE migraphx::
   DEPENDS
+    Threads
     ${PACKAGE_DEPENDS}
 )
  

src/api/include/migraphx/migraphx.h

@@ -209,7 +209,8 @@ migraphx_status migraphx_operation_destroy(migraphx_operation_t operation);
 
 migraphx_status migraphx_operation_create(migraphx_operation_t* operation,
                                           const char* name,
-                                          const char* attributes);
+                                          const char* attributes,
+                                          ...);
 
 migraphx_status migraphx_operation_name(char* out, size_t out_size, migraphx_operation_t operation);
 

src/api/include/migraphx/migraphx.hpp

@@ -599,9 +599,10 @@ struct operation : MIGRAPHX_HANDLE_BASE(operation)
 
     operation(migraphx_operation* p, borrow) { this->set_handle(p, borrow{}); }
 
-    operation(const char* name, const char* attributes = nullptr)
+    template <class... Ts>
+    operation(const char* name, const char* attributes = nullptr, Ts... xs)
     {
-        this->make_handle(&migraphx_operation_create, name, attributes);
+        this->make_handle(&migraphx_operation_create, name, attributes, xs...);
     }
 
     std::string name()

src/api/migraphx.py

@@ -212,7 +212,9 @@ def program(h):
 @auto_handle()
 def operation(h):
     h.constructor('create',
-                  api.params(name='const char*', attributes='const char*'),
+                  api.params(name='const char*',
+                             attributes='const char*',
+                             vlist='...'),
                   fname='migraphx::create_op')
     h.method('name', returns='std::string')
 

src/argument.cpp

@@ -155,5 +155,13 @@ std::vector<argument> argument::get_sub_objects() const
     return result;
 }
 
+argument argument::element(std::size_t i) const
+{
+    assert(this->get_shape().sub_shapes().empty());
+    auto idx    = this->get_shape().index(i);
+    auto offset = this->get_shape().type_size() * idx;
+    return argument{shape{this->get_shape().type()}, this->data() + offset};
+}
+
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx

src/driver/CMakeLists.txt

@@ -6,6 +6,7 @@ add_executable(driver
     resnet50.cpp
     inceptionv3.cpp
     alexnet.cpp
+    marker_roctx.cpp
 )
 set_target_properties(driver PROPERTIES OUTPUT_NAME migraphx-driver)
 # Copy driver for backwards compatibility

src/driver/main.cpp

@@ -3,6 +3,7 @@
 #include "verify.hpp"
 #include "perf.hpp"
 #include "models.hpp"
+#include "marker_roctx.hpp"
 
 #include <migraphx/tf.hpp>
 #include <migraphx/onnx.hpp>
@@ -483,6 +484,23 @@ struct perf : command<perf>
     }
 };
 
+struct roctx : command<roctx>
+{
+    compiler c;
+    void parse(argument_parser& ap) { c.parse(ap); }
+
+    void run()
+    {
+        std::cout << "Compiling ... " << std::endl;
+        auto p = c.compile();
+        std::cout << "Allocating params ... " << std::endl;
+        auto m = c.params(p);
+        std::cout << "rocTX:\tLoading rocTX library..." << std::endl;
+        auto rtx = create_marker_roctx();
+        p.mark(m, std::move(rtx));
+    }
+};
+
 struct op : command<op>
 {
     bool show_ops = false;

src/driver/marker_roctx.cpp

+#include "marker_roctx.hpp"
+
+#include <migraphx/dynamic_loader.hpp>
+#include <migraphx/instruction.hpp>
+#include <migraphx/instruction_ref.hpp>
+
+namespace migraphx {
+namespace driver {
+inline namespace MIGRAPHX_INLINE_NS {
+
+class marker_roctx
+{
+    std::function<void(const char*)> sym_roctx_mark;
+    std::function<uint64_t(const char*)> sym_roctx_range_start;
+    std::function<void(uint64_t)> sym_roctx_range_stop;
+
+    std::function<int(const char*)> sym_roctx_range_push;
+    std::function<int()> sym_roctx_range_pop;
+
+    uint64_t range_id;
+
+    public:
+    marker_roctx()
+    {
+        dynamic_loader lib    = migraphx::dynamic_loader{"libroctx64.so"};
+        sym_roctx_mark        = lib.get_function<void(const char*)>("roctxMarkA");
+        sym_roctx_range_start = lib.get_function<uint64_t(const char*)>("roctxRangeStartA");
+        sym_roctx_range_stop  = lib.get_function<void(uint64_t)>("roctxRangeStop");
+
+        sym_roctx_range_push = lib.get_function<int(const char*)>("roctxRangePushA");
+        sym_roctx_range_pop  = lib.get_function<int()>("roctxRangePop");
+
+        sym_roctx_mark("rocTX marker created.");
+    }
+
+    void mark_start(instruction_ref ins_ref)
+    {
+        std::string text = "Marker start: " + ins_ref->name();
+        sym_roctx_range_push(text.c_str());
+    }
+    void mark_stop(instruction_ref) { sym_roctx_range_pop(); }
+    void mark_start(const program&) { range_id = sym_roctx_range_start("0"); }
+    void mark_stop(const program&) { sym_roctx_range_stop(range_id); }
+};
+
+marker create_marker_roctx() { return marker_roctx(); }
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace driver
+} // namespace migraphx

src/driver/marker_roctx.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_MARKER_ROCTX_HPP
+#define MIGRAPHX_GUARD_RTGLIB_MARKER_ROCTX_HPP
+
+#include <migraphx/marker.hpp>
+
+namespace migraphx {
+namespace driver {
+inline namespace MIGRAPHX_INLINE_NS {
+
+marker create_marker_roctx();
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace driver
+} // namespace migraphx
+
+#endif

src/dynamic_loader.cpp

@@ -45,6 +45,7 @@ dynamic_loader::dynamic_loader(const std::vector<char>& buffer)
 
 std::shared_ptr<void> dynamic_loader::get_symbol(const std::string& name) const
 {
+    dlerror();
     void* symbol = dlsym(impl->handle.get(), name.c_str());
     if(symbol == nullptr)
         MIGRAPHX_THROW("Symbol not found: " + name);

src/fuse_pointwise.cpp

+#include <migraphx/fuse_pointwise.hpp>
+#include <migraphx/pass_manager.hpp>
+#include <migraphx/dead_code_elimination.hpp>
+#include <migraphx/instruction.hpp>
+#include <migraphx/program.hpp>
+#include <migraphx/make_op.hpp>
+#include <migraphx/iterator_for.hpp>
+#include <migraphx/ranges.hpp>
+#include <iterator>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+static literal get_scalar(instruction_ref ins)
+{
+    const auto& s = ins->get_shape();
+    if(not(s.elements() == 1 or s.scalar()))
+        return {};
+    if(not ins->can_eval())
+        return {};
+    auto e = ins->eval();
+    literal r{};
+    e.visit_at([&](auto x) { r = literal{x}; });
+    return r;
+}
+
+static void create_pointwise_modules(module_pass_manager& mpm)
+{
+    std::size_t n = 0;
+    for(auto ins : iterator_for(mpm.get_module()))
+    {
+        if(not ins->get_operator().attributes().get("pointwise", false))
+            continue;
+        auto* pm = mpm.create_module("pointwise" + std::to_string(n++));
+        pm->set_bypass();
+
+        std::unordered_map<instruction_ref, instruction_ref> param_map;
+        std::vector<instruction_ref> pointwise_inputs;
+        for(auto input : ins->inputs())
+        {
+            if(contains(param_map, input))
+                continue;
+            auto scalar = get_scalar(input);
+            if(scalar.empty())
+            {
+                pointwise_inputs.push_back(input);
+                param_map[input] = pm->add_parameter("x" + std::to_string(param_map.size()),
+                                                     shape{input->get_shape().type()});
+            }
+            else
+            {
+                param_map[input] = pm->add_literal(scalar);
+            }
+        }
+
+        std::vector<instruction_ref> inputs;
+        std::transform(ins->inputs().begin(),
+                       ins->inputs().end(),
+                       std::back_inserter(inputs),
+                       [&](auto input) { return param_map[input]; });
+        auto r = pm->add_instruction(ins->get_operator(), inputs);
+        pm->add_return({r});
+
+        mpm.get_module().replace_instruction(ins, make_op("pointwise"), pointwise_inputs, {pm});
+    }
+}
+
+static std::vector<instruction_ref> append_pointwise_module(instruction_ref ins,
+                                                            instruction_ref output)
+{
+    module_ref pm = ins->module_inputs().at(0);
+    module_ref xm = output->module_inputs().at(0);
+
+    auto last = std::prev(pm->end());
+    assert(last->name() == "@return");
+    assert(last->inputs().size() == 1);
+
+    std::vector<instruction_ref> inputs = ins->inputs();
+    std::unordered_map<instruction_ref, instruction_ref> map_ins;
+    std::unordered_map<instruction_ref, instruction_ref> input_map;
+    // Copy inputs to input_map
+    for(auto i : range(inputs.size()))
+    {
+        auto input       = inputs[i];
+        auto param       = pm->get_parameter("x" + std::to_string(i));
+        input_map[input] = param;
+    }
+    // Add the new parameter and additional inputs
+    for(auto i : range(output->inputs().size()))
+    {
+        auto input = output->inputs()[i];
+        auto param = xm->get_parameter("x" + std::to_string(i));
+        if(input == ins)
+        {
+            map_ins[param]   = last->inputs().front();
+            input_map[input] = map_ins[param];
+        }
+        // Avoid duplicate paramter inputs
+        else if(contains(input_map, input))
+        {
+            map_ins[param] = input_map[input];
+        }
+        else
+        {
+            map_ins[param] =
+                pm->add_parameter("x" + std::to_string(inputs.size()), {input->get_shape().type()});
+            inputs.push_back(input);
+            input_map[input] = map_ins[param];
+        }
+    }
+    pm->replace_return(pm->insert_module_instructions(last, xm, map_ins));
+    return inputs;
+}
+
+static bool find_pointwise_modules(module& m)
+{
+    bool changed = false;
+    for(auto ins : iterator_for(m))
+    {
+        if(ins->name() != "pointwise")
+            continue;
+        if(ins->outputs().empty())
+            continue;
+        auto it = std::find_if(ins->inputs().begin(), ins->inputs().end(), [&](auto i) {
+            return i->name() == "pointwise" and i->outputs().size() == 1;
+        });
+        if(it == ins->inputs().end())
+            continue;
+
+        auto new_inputs = append_pointwise_module(*it, ins);
+        m.replace_instruction(*it, (*it)->get_operator(), new_inputs, (*it)->module_inputs());
+        m.replace_instruction(ins, *it);
+        m.move_instruction(*it, ins);
+        changed = true;
+    }
+    return changed;
+}
+
+void fuse_pointwise::apply(module_pass_manager& mpm) const
+{
+    create_pointwise_modules(mpm);
+    mpm.run_pass(dead_code_elimination{});
+    for(int i = 0; i < 8; i++)
+    {
+        if(not find_pointwise_modules(mpm.get_module()))
+            break;
+        mpm.run_pass(dead_code_elimination{});
+    }
+}
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx

src/include/migraphx/argument.hpp

@@ -67,6 +67,9 @@ struct argument : raw_data<argument>
 
     std::vector<argument> get_sub_objects() const;
 
+    /// Return the ith element
+    argument element(std::size_t i) const;
+
     private:
     void assign_buffer(std::function<char*()> d);
     struct data_t

src/include/migraphx/fuse_pointwise.hpp

+#ifndef MIGRAPHX_GUARD_MIGRAPHX_FUSE_POINTWISE_HPP
+#define MIGRAPHX_GUARD_MIGRAPHX_FUSE_POINTWISE_HPP
+
+#include <migraphx/config.hpp>
+#include <string>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+struct module_pass_manager;
+
+struct fuse_pointwise
+{
+    std::string name() const { return "fuse_pointwise"; }
+    void apply(module_pass_manager& mpm) const;
+};
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+#endif // MIGRAPHX_GUARD_MIGRAPHX_FUSE_POINTWISE_HPP

src/include/migraphx/instruction.hpp

@@ -152,30 +152,4 @@ struct instruction
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx
 
-namespace std {
-template <>
-struct hash<migraphx::instruction_ref>
-{
-    using argument_type = migraphx::instruction_ref;
-    using result_type   = std::size_t;
-    result_type operator()(const argument_type& x) const noexcept
-    {
-        return std::hash<migraphx::instruction*>{}(&*x);
-    }
-};
-
-template <>
-struct equal_to<migraphx::instruction_ref>
-{
-    using argument_type = migraphx::instruction_ref;
-    using result_type   = bool;
-    result_type operator()(const migraphx::instruction_ref& x,
-                           const migraphx::instruction_ref& y) const noexcept
-    {
-        return &*x == &*y;
-    }
-};
-
-} // namespace std
-
 #endif

src/include/migraphx/instruction_ref.hpp

@@ -11,7 +11,35 @@ inline namespace MIGRAPHX_INLINE_NS {
 struct instruction;
 using instruction_ref = std::list<instruction>::iterator;
 
+migraphx::instruction* as_address(const instruction_ref& ins) noexcept;
+
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx
 
+namespace std {
+template <>
+struct hash<migraphx::instruction_ref>
+{
+    using argument_type = migraphx::instruction_ref;
+    using result_type   = std::size_t;
+    result_type operator()(const migraphx::instruction_ref& x) const noexcept
+    {
+        return std::hash<migraphx::instruction*>{}(migraphx::as_address(x));
+    }
+};
+
+template <>
+struct equal_to<migraphx::instruction_ref>
+{
+    using argument_type = migraphx::instruction_ref;
+    using result_type   = bool;
+    result_type operator()(const migraphx::instruction_ref& x,
+                           const migraphx::instruction_ref& y) const noexcept
+    {
+        return migraphx::as_address(x) == migraphx::as_address(y);
+    }
+};
+
+} // namespace std
+
 #endif

src/include/migraphx/marker.hpp

+#ifndef MIGRAPHX_GUARD_MARKER_HPP
+#define MIGRAPHX_GUARD_MARKER_HPP
+
+#include <cassert>
+#include <string>
+#include <functional>
+#include <memory>
+#include <type_traits>
+#include <utility>
+#include <migraphx/config.hpp>
+#include <migraphx/instruction_ref.hpp>
+#include <migraphx/program.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+#ifdef DOXYGEN
+
+/// Marker is an interface to general marking functions, such as rocTX markers.
+
+#else
+
+/*
+ * Type-erased interface for:
+ *
+ * struct marker
+ * {
+ *      void mark_start(instruction_ref ins_ref) ;
+ *      void mark_start(const program& prog) ;
+ *      void mark_stop(instruction_ref ins) ;
+ *      void mark_stop(const program& prog) ;
+ * };
+ *
+ */
+
+struct marker
+{
+    // Constructors
+    marker() = default;
+
+    template <typename PrivateDetailTypeErasedT>
+    marker(PrivateDetailTypeErasedT value)
+        : private_detail_te_handle_mem_var(
+              std::make_shared<private_detail_te_handle_type<
+                  typename std::remove_reference<PrivateDetailTypeErasedT>::type>>(
+                  std::forward<PrivateDetailTypeErasedT>(value)))
+    {
+    }
+
+    // Assignment
+    template <typename PrivateDetailTypeErasedT>
+    marker& operator=(PrivateDetailTypeErasedT value)
+    {
+        using std::swap;
+        auto* derived = this->any_cast<PrivateDetailTypeErasedT>();
+        if(derived and private_detail_te_handle_mem_var.unique())
+        {
+            *derived = std::forward<PrivateDetailTypeErasedT>(value);
+        }
+        else
+        {
+            marker rhs(value);
+            swap(private_detail_te_handle_mem_var, rhs.private_detail_te_handle_mem_var);
+        }
+        return *this;
+    }
+
+    // Cast
+    template <typename PrivateDetailTypeErasedT>
+    PrivateDetailTypeErasedT* any_cast()
+    {
+        return this->type_id() == typeid(PrivateDetailTypeErasedT)
+                   ? std::addressof(static_cast<private_detail_te_handle_type<
+                                        typename std::remove_cv<PrivateDetailTypeErasedT>::type>&>(
+                                        private_detail_te_get_handle())
+                                        .private_detail_te_value)
+                   : nullptr;
+    }
+
+    template <typename PrivateDetailTypeErasedT>
+    const typename std::remove_cv<PrivateDetailTypeErasedT>::type* any_cast() const
+    {
+        return this->type_id() == typeid(PrivateDetailTypeErasedT)
+                   ? std::addressof(static_cast<const private_detail_te_handle_type<
+                                        typename std::remove_cv<PrivateDetailTypeErasedT>::type>&>(
+                                        private_detail_te_get_handle())
+                                        .private_detail_te_value)
+                   : nullptr;
+    }
+
+    const std::type_info& type_id() const
+    {
+        if(private_detail_te_handle_empty())
+            return typeid(std::nullptr_t);
+        else
+            return private_detail_te_get_handle().type();
+    }
+
+    void mark_start(instruction_ref ins_ref)
+    {
+        assert((*this).private_detail_te_handle_mem_var);
+        (*this).private_detail_te_get_handle().mark_start(ins_ref);
+    }
+
+    void mark_start(const program& prog)
+    {
+        assert((*this).private_detail_te_handle_mem_var);
+        (*this).private_detail_te_get_handle().mark_start(prog);
+    }
+
+    void mark_stop(instruction_ref ins)
+    {
+        assert((*this).private_detail_te_handle_mem_var);
+        (*this).private_detail_te_get_handle().mark_stop(ins);
+    }
+
+    void mark_stop(const program& prog)
+    {
+        assert((*this).private_detail_te_handle_mem_var);
+        (*this).private_detail_te_get_handle().mark_stop(prog);
+    }
+
+    friend bool is_shared(const marker& private_detail_x, const marker& private_detail_y)
+    {
+        return private_detail_x.private_detail_te_handle_mem_var ==
+               private_detail_y.private_detail_te_handle_mem_var;
+    }
+
+    private:
+    struct private_detail_te_handle_base_type
+    {
+        virtual ~private_detail_te_handle_base_type() {}
+        virtual std::shared_ptr<private_detail_te_handle_base_type> clone() const = 0;
+        virtual const std::type_info& type() const                                = 0;
+
+        virtual void mark_start(instruction_ref ins_ref) = 0;
+        virtual void mark_start(const program& prog)     = 0;
+        virtual void mark_stop(instruction_ref ins)      = 0;
+        virtual void mark_stop(const program& prog)      = 0;
+    };
+
+    template <typename PrivateDetailTypeErasedT>
+    struct private_detail_te_handle_type : private_detail_te_handle_base_type
+    {
+        template <typename PrivateDetailTypeErasedU = PrivateDetailTypeErasedT>
+        private_detail_te_handle_type(
+            PrivateDetailTypeErasedT value,
+            typename std::enable_if<std::is_reference<PrivateDetailTypeErasedU>::value>::type* =
+                nullptr)
+            : private_detail_te_value(value)
+        {
+        }
+
+        template <typename PrivateDetailTypeErasedU = PrivateDetailTypeErasedT>
+        private_detail_te_handle_type(
+            PrivateDetailTypeErasedT value,
+            typename std::enable_if<!std::is_reference<PrivateDetailTypeErasedU>::value,
+                                    int>::type* = nullptr) noexcept
+            : private_detail_te_value(std::move(value))
+        {
+        }
+
+        std::shared_ptr<private_detail_te_handle_base_type> clone() const override
+        {
+            return std::make_shared<private_detail_te_handle_type>(private_detail_te_value);
+        }
+
+        const std::type_info& type() const override { return typeid(private_detail_te_value); }
+
+        void mark_start(instruction_ref ins_ref) override
+        {
+
+            private_detail_te_value.mark_start(ins_ref);
+        }
+
+        void mark_start(const program& prog) override { private_detail_te_value.mark_start(prog); }
+
+        void mark_stop(instruction_ref ins) override { private_detail_te_value.mark_stop(ins); }
+
+        void mark_stop(const program& prog) override { private_detail_te_value.mark_stop(prog); }
+
+        PrivateDetailTypeErasedT private_detail_te_value;
+    };
+
+    template <typename PrivateDetailTypeErasedT>
+    struct private_detail_te_handle_type<std::reference_wrapper<PrivateDetailTypeErasedT>>
+        : private_detail_te_handle_type<PrivateDetailTypeErasedT&>
+    {
+        private_detail_te_handle_type(std::reference_wrapper<PrivateDetailTypeErasedT> ref)
+            : private_detail_te_handle_type<PrivateDetailTypeErasedT&>(ref.get())
+        {
+        }
+    };
+
+    bool private_detail_te_handle_empty() const
+    {
+        return private_detail_te_handle_mem_var == nullptr;
+    }
+
+    const private_detail_te_handle_base_type& private_detail_te_get_handle() const
+    {
+        assert(private_detail_te_handle_mem_var != nullptr);
+        return *private_detail_te_handle_mem_var;
+    }
+
+    private_detail_te_handle_base_type& private_detail_te_get_handle()
+    {
+        assert(private_detail_te_handle_mem_var != nullptr);
+        if(!private_detail_te_handle_mem_var.unique())
+            private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone();
+        return *private_detail_te_handle_mem_var;
+    }
+
+    std::shared_ptr<private_detail_te_handle_base_type> private_detail_te_handle_mem_var;
+};
+
+template <typename ValueType>
+inline const ValueType* any_cast(const marker* x)
+{
+    return x->any_cast<ValueType>();
+}
+
+template <typename ValueType>
+inline ValueType* any_cast(marker* x)
+{
+    return x->any_cast<ValueType>();
+}
+
+template <typename ValueType>
+inline ValueType& any_cast(marker& x)
+{
+    auto* y = x.any_cast<typename std::remove_reference<ValueType>::type>();
+    if(y == nullptr)
+        throw std::bad_cast();
+    return *y;
+}
+
+template <typename ValueType>
+inline const ValueType& any_cast(const marker& x)
+{
+    const auto* y = x.any_cast<typename std::remove_reference<ValueType>::type>();
+    if(y == nullptr)
+        throw std::bad_cast();
+    return *y;
+}
+
+#endif
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif