Merge branch 'develop' into blas_tuning

src/include/migraphx/raw_data.hpp

@@ -187,6 +187,7 @@ struct raw_data : raw_data_base
     std::string to_string() const
     {
         std::stringstream ss;
+        ss.precision(std::numeric_limits<double>::max_digits10);
         ss << static_cast<const Derived&>(*this);
         return ss.str();
     }

src/include/migraphx/reduce_dims.hpp

@@ -31,7 +31,7 @@
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 
-std::vector<shape> reduce_dims(const std::vector<shape>& shapes);
+MIGRAPHX_EXPORT std::vector<shape> reduce_dims(const std::vector<shape>& shapes);
 
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx

src/include/migraphx/reflect.hpp

@@ -78,7 +78,7 @@ template <class T>
 struct wrapper
 {
     using type = typename remove_rvalue_reference<T>::type;
-    type data;
+    type data; // NOLINT
     type get() const { return data; }
 };
 

src/include/migraphx/register_op.hpp

@@ -34,7 +34,7 @@ namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 
 // unregister all ops for specified target, useful when unloading dynamically plugged-in target lib
-void unregister_op(const std::string& op_name);
+MIGRAPHX_EXPORT void unregister_op(const std::string& op_name);
 
 namespace detail {
 struct op_handler
@@ -47,15 +47,15 @@ struct op_handler
 
 } // namespace detail
 
-void register_op_init();
+MIGRAPHX_EXPORT void register_op_init();
 
-void register_op(const operation& op);
+MIGRAPHX_EXPORT void register_op(const operation& op);
 
-operation load_op(const std::string& name);
+MIGRAPHX_EXPORT operation load_op(const std::string& name);
 
-bool has_op(const std::string& name);
+MIGRAPHX_EXPORT bool has_op(const std::string& name);
 
-std::vector<std::string> get_operators();
+MIGRAPHX_EXPORT std::vector<std::string> get_operators();
 
 template <class T>
 void register_op()

src/include/migraphx/register_target.hpp

@@ -33,11 +33,11 @@
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 
-void register_target_init();
-void register_target(const target& t);
-void unregister_target(const std::string& name);
-target make_target(const std::string& name);
-std::vector<std::string> get_targets();
+MIGRAPHX_EXPORT void register_target_init();
+MIGRAPHX_EXPORT void register_target(const target& t);
+MIGRAPHX_EXPORT void unregister_target(const std::string& name);
+MIGRAPHX_EXPORT target make_target(const std::string& name);
+MIGRAPHX_EXPORT std::vector<std::string> get_targets();
 
 namespace detail {
 struct target_handler

src/include/migraphx/replace_allocate.hpp

@@ -30,17 +30,17 @@
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 
-struct module;
+struct module_pass_manager;
 
 /**
  *  Replace `allocate` instructions with target allocations or output parameters.
  */
-struct replace_allocate
+struct MIGRAPHX_EXPORT replace_allocate
 {
     allocation_model model;
     bool offload_copy = false;
     std::string name() const { return "replace_allocate"; }
-    void apply(module& m) const;
+    void apply(module_pass_manager& mpm) const;
 };
 
 } // namespace MIGRAPHX_INLINE_NS

src/include/migraphx/rewrite_gelu.hpp

@@ -36,7 +36,7 @@ struct module;
 /**
  * Rewrite gelu standard formula as the sigmoid approximation formula
  */
-struct rewrite_gelu
+struct MIGRAPHX_EXPORT rewrite_gelu
 {
     std::string name() const { return "rewrite_gelu"; }
     void apply(module& m) const;

src/include/migraphx/rewrite_pooling.hpp

@@ -35,7 +35,7 @@ struct module;
 /**
  * Rewrite pooling to reduce_mean
  */
-struct rewrite_pooling
+struct MIGRAPHX_EXPORT rewrite_pooling
 {
     std::string name() const { return "rewrite_pooling"; }
     void apply(module& m) const;

src/include/migraphx/rewrite_quantization.hpp

@@ -35,7 +35,7 @@ struct module;
 /**
  * Rewrite quantization ops to equivalent operators
  */
-struct rewrite_quantization
+struct MIGRAPHX_EXPORT rewrite_quantization
 {
     std::string name() const { return "rewrite_quantization"; }
     void apply(module& m) const;

src/include/migraphx/rewrite_rnn.hpp

@@ -39,7 +39,7 @@ struct module;
 /**
  * Rewrite rnn to gemm and add.
  */
-struct rewrite_rnn
+struct MIGRAPHX_EXPORT rewrite_rnn
 {
     std::string name() const { return "rewrite_rnn"; }
     void apply(module& m) const;

src/include/migraphx/schedule.hpp

@@ -37,7 +37,7 @@ struct module;
 /**
  * Schedule instructions for concurrent execution
  */
-struct schedule
+struct MIGRAPHX_EXPORT schedule
 {
     schedule_model model{};
     bool enable = true;

src/include/migraphx/schedule_model.hpp

@@ -63,7 +63,7 @@ struct schedule_model
 #ifdef TYPE_ERASED_DECLARATION
 
 // Type-erased interface for:
-struct schedule_model
+struct MIGRAPHX_EXPORT schedule_model
 {
     //
     std::size_t concurrency() const;
@@ -99,7 +99,7 @@ struct schedule_model
     {
         using std::swap;
         auto* derived = this->any_cast<PrivateDetailTypeErasedT>();
-        if(derived and private_detail_te_handle_mem_var.unique())
+        if(derived and private_detail_te_handle_mem_var.use_count() == 1)
         {
             *derived = std::forward<PrivateDetailTypeErasedT>(value);
         }
@@ -274,7 +274,7 @@ struct schedule_model
     private_detail_te_handle_base_type& private_detail_te_get_handle()
     {
         assert(private_detail_te_handle_mem_var != nullptr);
-        if(not private_detail_te_handle_mem_var.unique())
+        if(private_detail_te_handle_mem_var.use_count() > 1)
             private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone();
         return *private_detail_te_handle_mem_var;
     }

src/include/migraphx/shape.hpp

 /*
  * The MIT License (MIT)
  *
- * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -43,7 +43,7 @@ inline namespace MIGRAPHX_INLINE_NS {
 struct value;
 struct shape_impl;
 
-struct shape
+struct MIGRAPHX_EXPORT shape
 {
 
 // Add new types here
@@ -85,7 +85,7 @@ struct shape
     {
     };
 
-    struct dynamic_dimension
+    struct MIGRAPHX_EXPORT dynamic_dimension
     {
         std::size_t min = 0;
         std::size_t max = 0;
@@ -100,22 +100,28 @@ struct shape
         bool is_fixed() const;
         bool has_optimal() const;
 
-        friend bool operator==(const dynamic_dimension& x, const dynamic_dimension& y);
-        friend bool operator!=(const dynamic_dimension& x, const dynamic_dimension& y);
-        friend std::ostream& operator<<(std::ostream& os, const dynamic_dimension& x);
+        MIGRAPHX_EXPORT friend bool operator==(const dynamic_dimension& x,
+                                               const dynamic_dimension& y);
+        MIGRAPHX_EXPORT friend bool operator!=(const dynamic_dimension& x,
+                                               const dynamic_dimension& y);
+        MIGRAPHX_EXPORT friend std::ostream& operator<<(std::ostream& os,
+                                                        const dynamic_dimension& x);
 
         // compare to fixed std::size_t dimension
-        friend bool operator==(const dynamic_dimension& x, const std::size_t& y);
-        friend bool operator==(const std::size_t& x, const dynamic_dimension& y);
-        friend bool operator!=(const dynamic_dimension& x, const std::size_t& y);
-        friend bool operator!=(const std::size_t& x, const dynamic_dimension& y);
+        MIGRAPHX_EXPORT friend bool operator==(const dynamic_dimension& x, const std::size_t& y);
+        MIGRAPHX_EXPORT friend bool operator==(const std::size_t& x, const dynamic_dimension& y);
+        MIGRAPHX_EXPORT friend bool operator!=(const dynamic_dimension& x, const std::size_t& y);
+        MIGRAPHX_EXPORT friend bool operator!=(const std::size_t& x, const dynamic_dimension& y);
 
         // add and subtract fixed std::size_t dimension
         dynamic_dimension& operator+=(const std::size_t& x);
         dynamic_dimension& operator-=(const std::size_t& x);
-        friend dynamic_dimension operator+(const dynamic_dimension& x, const std::size_t& y);
-        friend dynamic_dimension operator+(const std::size_t& x, const dynamic_dimension& y);
-        friend dynamic_dimension operator-(const dynamic_dimension& x, const std::size_t& y);
+        MIGRAPHX_EXPORT friend dynamic_dimension operator+(const dynamic_dimension& x,
+                                                           const std::size_t& y);
+        MIGRAPHX_EXPORT friend dynamic_dimension operator+(const std::size_t& x,
+                                                           const dynamic_dimension& y);
+        MIGRAPHX_EXPORT friend dynamic_dimension operator-(const dynamic_dimension& x,
+                                                           const std::size_t& y);
     };
 
     static const std::vector<type_t>& types();
@@ -156,14 +162,34 @@ struct shape
 
     shape(const std::vector<shape>& subs);
 
+    /**
+     * Creates an output shape with dimensions equal to the input lengths and strides determined
+     * by the permutation argument such that find_permutation() of the output shape returns the
+     * inputted permuation.
+     *
+     * 2D example:
+     *   parameters:
+     *     l = [2, 3], perm = [1, 0]
+     *   therefore:
+     *     "original" shape = {lens = [3, 2], strides = [2, 1]}
+     *     output_shape = {lens = [2, 3], strides = [1, 2]
+     *
+     * 3D example:
+     *   parameters:
+     *     l = [2, 3, 4], perm = [1, 2, 0]
+     *   therefore:
+     *     "original" shape = {lens = [3, 4, 2], strides = [8, 2, 1]}
+     *     output_shape = {lens = [2, 3, 4], strides = [1, 8, 2]}
+     */
     static shape
     from_permutation(type_t t, const std::vector<std::size_t>& l, const std::vector<int64_t>& perm);
+
     type_t type() const;
     const std::vector<std::size_t>& lens() const;
     const std::vector<std::size_t>& strides() const;
 
     /*!
-     * The number of dimensions in the shape.
+     * The number of dimensions in the shape, either static or dynamic.
      * Same as the number of indices required to get a data value.
      */
     std::size_t ndim() const;
@@ -214,6 +240,10 @@ struct shape
     template <class Iterator>
     std::size_t index(Iterator start, Iterator last) const
     {
+        if(this->dynamic())
+        {
+            MIGRAPHX_THROW("SHAPE: index() called on dynamic shape");
+        }
         assert(std::distance(start, last) <= this->lens().size());
         assert(this->lens().size() == this->strides().size());
         return std::inner_product(start, last, this->strides().begin(), std::size_t{0}); // NOLINT
@@ -266,9 +296,9 @@ struct shape
     // convert the shape to a static one setting any non-fixed dynamic_dimensions to x
     shape to_static(std::size_t x) const;
 
-    friend bool operator==(const shape& x, const shape& y);
-    friend bool operator!=(const shape& x, const shape& y);
-    friend std::ostream& operator<<(std::ostream& os, const shape& x);
+    MIGRAPHX_EXPORT friend bool operator==(const shape& x, const shape& y);
+    MIGRAPHX_EXPORT friend bool operator!=(const shape& x, const shape& y);
+    MIGRAPHX_EXPORT friend std::ostream& operator<<(std::ostream& os, const shape& x);
 
     template <class T>
     struct as
@@ -279,6 +309,8 @@ struct shape
 
         type min() const { return std::numeric_limits<type>::lowest(); }
 
+        type nan() const { return std::numeric_limits<type>::quiet_NaN(); }
+
         template <class U>
         type operator()(U u) const
         {
@@ -374,8 +406,8 @@ struct shape
     std::shared_ptr<const shape_impl> impl;
 };
 
-void migraphx_to_value(value& v, const shape& s);
-void migraphx_from_value(const value& v, shape& s);
+MIGRAPHX_EXPORT void migraphx_to_value(value& v, const shape& s);
+MIGRAPHX_EXPORT void migraphx_from_value(const value& v, shape& s);
 
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx

src/include/migraphx/simplify_algebra.hpp

@@ -35,7 +35,7 @@ struct module;
 /**
  * Simplify many algebraic instructions to more efficient versions.
  */
-struct simplify_algebra
+struct MIGRAPHX_EXPORT simplify_algebra
 {
     std::string name() const { return "simplify_algebra"; }
     void apply(module& m) const;

src/include/migraphx/simplify_qdq.hpp

@@ -36,7 +36,7 @@ struct module;
  * Inserts quantized operators in place of dq->quantizable_op->q
  * then removes remaining fake quantization (q->dq pairs)
  */
-struct simplify_qdq
+struct MIGRAPHX_EXPORT simplify_qdq
 {
     std::string name() const { return "simplify_qdq"; }
     void apply(module& m) const;

src/include/migraphx/simplify_reshapes.hpp

@@ -36,7 +36,7 @@ struct module;
 /**
  * Eliminate redundant reshapes.
  */
-struct simplify_reshapes
+struct MIGRAPHX_EXPORT simplify_reshapes
 {
     std::string name() const { return "simplify_reshapes"; }
     void apply(module& m) const;

src/include/migraphx/source_location.hpp

+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef MIGRAPHX_GUARD_MIGRAPHX_SOURCE_LOCATION_HPP
+#define MIGRAPHX_GUARD_MIGRAPHX_SOURCE_LOCATION_HPP
+
+#include <migraphx/config.hpp>
+
+#if defined(CPPCHECK)
+#define MIGRAPHX_HAS_SOURCE_LOCATION 1
+#define MIGRAPHX_HAS_SOURCE_LOCATION_TS 1
+#elif defined(__has_include)
+#if __has_include(<source_location>) && __cplusplus >= 202003L
+#define MIGRAPHX_HAS_SOURCE_LOCATION 1
+#else
+#define MIGRAPHX_HAS_SOURCE_LOCATION 0
+#endif
+#if __has_include(<experimental/source_location>) && __cplusplus >= 201103L
+#define MIGRAPHX_HAS_SOURCE_LOCATION_TS 1
+#else
+#define MIGRAPHX_HAS_SOURCE_LOCATION_TS 0
+#endif
+#else
+#define MIGRAPHX_HAS_SOURCE_LOCATION 0
+#define MIGRAPHX_HAS_SOURCE_LOCATION_TS 0
+#endif
+
+#if MIGRAPHX_HAS_SOURCE_LOCATION
+#include <source_location>
+#elif MIGRAPHX_HAS_SOURCE_LOCATION_TS
+#include <experimental/source_location>
+#endif
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+#if MIGRAPHX_HAS_SOURCE_LOCATION
+using source_location = std::source_location;
+#elif MIGRAPHX_HAS_SOURCE_LOCATION_TS
+using source_location = std::experimental::source_location;
+#else
+struct source_location
+{
+    static constexpr source_location current() noexcept { return source_location{}; }
+    constexpr std::uint_least32_t line() const noexcept { return 0; }
+    constexpr std::uint_least32_t column() const noexcept { return 0; }
+    constexpr const char* file_name() const noexcept { return ""; }
+    constexpr const char* function_name() const noexcept { return ""; }
+};
+#endif
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+#endif // MIGRAPHX_GUARD_MIGRAPHX_SOURCE_LOCATION_HPP

src/include/migraphx/split_single_dyn_dim.hpp

@@ -36,7 +36,7 @@ inline namespace MIGRAPHX_INLINE_NS {
  * Split dynamic dimension over submodules if exactly one dimension in the parameter list is
  * dynamic.
  */
-struct split_single_dyn_dim
+struct MIGRAPHX_EXPORT split_single_dyn_dim
 {
     std::string name() const { return "split_single_dyn_dim"; }
     void apply(module_pass_manager&) const;

src/include/migraphx/sqlite.hpp

@@ -35,7 +35,7 @@ inline namespace MIGRAPHX_INLINE_NS {
 
 struct sqlite_impl;
 
-struct sqlite
+struct MIGRAPHX_EXPORT sqlite
 {
     sqlite() = default;
     static sqlite read(const fs::path& p);

src/include/migraphx/stream_model.hpp

@@ -62,7 +62,7 @@ struct stream_model
 #ifdef TYPE_ERASED_DECLARATION
 
 // Type-erased interface for:
-struct stream_model
+struct MIGRAPHX_EXPORT stream_model
 {
     //
     std::size_t get_nstream() const;
@@ -100,7 +100,7 @@ struct stream_model
     {
         using std::swap;
         auto* derived = this->any_cast<PrivateDetailTypeErasedT>();
-        if(derived and private_detail_te_handle_mem_var.unique())
+        if(derived and private_detail_te_handle_mem_var.use_count() == 1)
         {
             *derived = std::forward<PrivateDetailTypeErasedT>(value);
         }
@@ -288,7 +288,7 @@ struct stream_model
     private_detail_te_handle_base_type& private_detail_te_get_handle()
     {
         assert(private_detail_te_handle_mem_var != nullptr);
-        if(not private_detail_te_handle_mem_var.unique())
+        if(private_detail_te_handle_mem_var.use_count() > 1)
             private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone();
         return *private_detail_te_handle_mem_var;
     }