manual merge

src/CMakeLists.txt

@@ -58,14 +58,14 @@ add_library(migraphx
     layout_nhwc.cpp
     load_save.cpp
     make_op.cpp
+    memory_coloring.cpp
     module.cpp
     msgpack.cpp
     normalize_attributes.cpp
     normalize_ops.cpp
     op_enums.cpp
     operation.cpp
-    opt/memory_coloring.cpp
-    opt/memory_coloring_impl.cpp
+    optimize_module.cpp
     pad_calc.cpp
     pass_manager.cpp
     permutation.cpp
@@ -199,6 +199,7 @@ register_migraphx_ops(
     scatternd_add
     scatternd_mul
     scatternd_none
+    select_module
     sigmoid
     sign
     sinh
@@ -250,7 +251,10 @@ find_package(PkgConfig)
 pkg_check_modules(SQLITE3 REQUIRED IMPORTED_TARGET sqlite3)
 target_link_libraries(migraphx PRIVATE PkgConfig::SQLITE3)
 
-find_package(msgpack REQUIRED)
+find_package(msgpackc-cxx QUIET)
+if(NOT msgpackc-cxx_FOUND)
+    find_package(msgpack REQUIRED)
+endif()
 target_link_libraries(migraphx PRIVATE msgpackc-cxx)
 # Make this available to the tests
 target_link_libraries(migraphx INTERFACE $<BUILD_INTERFACE:msgpackc-cxx>)
@@ -288,6 +292,10 @@ if(HAVE_HALF_EXPR)
     target_compile_definitions(migraphx PUBLIC -DHAS_HALF_V1)
 endif()
 
+if(BUILD_DEV)
+    target_compile_definitions(migraphx PUBLIC -DBUILD_DEV)
+endif()
+
 rocm_export_targets(
   TARGETS migraphx::migraphx_c
   NAMESPACE migraphx::

src/api/CMakeLists.txt

@@ -26,10 +26,13 @@ add_library(migraphx_c
     api.cpp
 )
 set_target_properties(migraphx_c PROPERTIES EXPORT_NAME c)
+
+# migraphx_c is stable API interface library. SO version of this should be 
+# bumped when binary compatibility is broken. 
 rocm_set_soversion(migraphx_c 3.0)   
 
 rocm_clang_tidy_check(migraphx_c)
-target_link_libraries(migraphx_c PRIVATE migraphx migraphx_tf migraphx_onnx migraphx_all_targets)
+target_link_libraries(migraphx_c PRIVATE migraphx migraphx_tf migraphx_onnx)
 
 rocm_install_targets(
   TARGETS migraphx_c

src/api/api.cpp

@@ -32,7 +32,6 @@
 #include <migraphx/register_target.hpp>
 #include <migraphx/generate.hpp>
 #include <migraphx/quantization.hpp>
-#include <migraphx/ref/target.hpp>
 #include <migraphx/load_save.hpp>
 #include <migraphx/make_op.hpp>
 #include <migraphx/register_op.hpp>
@@ -134,6 +133,11 @@ void set_offload_copy(compile_options& options, bool value) { options.offload_co
 
 void set_fast_math(compile_options& options, bool value) { options.fast_math = value; }
 
+void set_exhaustive_tune_flag(compile_options& options, bool value)
+{
+    options.exhaustive_tune = value;
+}
+
 void set_file_format(file_options& options, const char* format) { options.format = format; }
 
 void set_default_dim_value(onnx_options& options, size_t value)
@@ -1690,6 +1694,19 @@ migraphx_compile_options_set_fast_math(migraphx_compile_options_t compile_option
     return api_error_result;
 }
 
+extern "C" migraphx_status
+migraphx_compile_options_set_exhaustive_tune_flag(migraphx_compile_options_t compile_options,
+                                                  bool value)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(compile_options == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param,
+                           "Bad parameter compile_options: Null pointer");
+        migraphx::set_exhaustive_tune_flag((compile_options->object), (value));
+    });
+    return api_error_result;
+}
+
 extern "C" migraphx_status
 migraphx_parse_onnx(migraphx_program_t* out, const char* name, migraphx_onnx_options_t options)
 {

src/api/include/migraphx/migraphx.h

@@ -427,6 +427,10 @@ migraphx_compile_options_set_offload_copy(migraphx_compile_options_t compile_opt
 migraphx_status migraphx_compile_options_set_fast_math(migraphx_compile_options_t compile_options,
                                                        bool value);
 
+migraphx_status
+migraphx_compile_options_set_exhaustive_tune_flag(migraphx_compile_options_t compile_options,
+                                                  bool value);
+
 migraphx_status
 migraphx_parse_onnx(migraphx_program_t* out, const char* name, migraphx_onnx_options_t options);
 

src/api/include/migraphx/migraphx.hpp

@@ -1015,6 +1015,12 @@ struct compile_options : MIGRAPHX_HANDLE_BASE(compile_options)
     {
         call(&migraphx_compile_options_set_fast_math, this->get_handle_ptr(), value);
     }
+
+    /// Set or un-set exhaustive search to find fastest kernel
+    void set_exhaustive_tune_flag(bool value = true)
+    {
+        call(&migraphx_compile_options_set_exhaustive_tune_flag, this->get_handle_ptr(), value);
+    }
 };
 
 /// A program represents the all computation graphs to be compiled and executed

src/api/migraphx.py

@@ -354,6 +354,9 @@ def compile_options(h):
     h.method('set_fast_math',
              api.params(value='bool'),
              invoke='migraphx::set_fast_math($@)')
+    h.method('set_exhaustive_tune_flag',
+             api.params(value='bool'),
+             invoke='migraphx::set_exhaustive_tune_flag($@)')
 
 
 api.add_function('migraphx_parse_onnx',

src/compile_src.cpp

@@ -70,9 +70,6 @@ std::vector<char> src_compiler::compile(const std::vector<src_file>& srcs) const
     if(not fs::exists(out_path))
         MIGRAPHX_THROW("Output file missing: " + out);
 
-    if(process)
-        out_path = process(out_path);
-
     return read_buffer(out_path.string());
 }
 

src/driver/CMakeLists.txt

@@ -42,6 +42,7 @@ add_custom_command(
 )
 set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/driver)
 rocm_clang_tidy_check(driver)
+
 target_link_libraries(driver migraphx_all_targets migraphx_onnx migraphx_tf)
 
 rocm_install_targets(

src/driver/main.cpp

@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
 #include "verify.hpp"
 #include "argument_parser.hpp"
 #include "command.hpp"
@@ -304,8 +305,12 @@ struct compiler_target
 {
 #ifdef HAVE_GPU
     std::string target_name = "gpu";
-#else
+#elif HAVE_CPU
     std::string target_name = "cpu";
+#elif HAVE_FPGA
+    std::string target_name = "fpga"
+#else
+    std::string target_name = "ref"
 #endif
 
     void parse(argument_parser& ap)
@@ -326,8 +331,7 @@ struct compiler
     loader l;
     program_params parameters;
     compiler_target ct;
-    bool offload_copy  = false;
-    bool fast_math     = true;
+    compile_options co;
     precision quantize = precision::fp32;
 
     std::vector<std::string> fill0;
@@ -337,19 +341,26 @@ struct compiler
         l.parse(ap);
         parameters.parse(ap);
         ct.parse(ap);
-        ap(offload_copy,
+        ap(co.offload_copy,
            {"--enable-offload-copy"},
            ap.help("Enable implicit offload copying"),
            ap.set_value(true));
-        ap(fast_math,
+        ap(co.fast_math,
            {"--disable-fast-math"},
            ap.help("Disable fast math optimization"),
            ap.set_value(false));
+        ap(co.exhaustive_tune,
+           {"--exhaustive-tune"},
+           ap.help("Exhastively search for best tuning parameters for kernels"),
+           ap.set_value(true));
         ap(quantize, {"--fp16"}, ap.help("Quantize for fp16"), ap.set_value(precision::fp16));
         ap(quantize, {"--int8"}, ap.help("Quantize for int8"), ap.set_value(precision::int8));
     }
 
-    auto params(const program& p) { return parameters.generate(p, ct.get_target(), offload_copy); }
+    auto params(const program& p)
+    {
+        return parameters.generate(p, ct.get_target(), co.offload_copy);
+    }
 
     program compile()
     {
@@ -366,10 +377,7 @@ struct compiler
         {
             quantize_int8(p, t, {params(p)});
         }
-        compile_options options;
-        options.offload_copy = offload_copy;
-        options.fast_math    = fast_math;
-        p.compile(t, options);
+        p.compile(t, co);
         l.save(p);
         return p;
     }
@@ -402,60 +410,41 @@ struct params : command<params>
 
 struct verify : command<verify>
 {
-    loader l;
-    program_params parameters;
-    compiler_target ct;
+    compiler c;
     double tolerance     = 80;
     bool per_instruction = false;
     bool reduce          = false;
-    bool offload_copy    = false;
-    bool fast_math       = true;
-    precision quantize   = precision::fp32;
     void parse(argument_parser& ap)
     {
-        l.parse(ap);
-        parameters.parse(ap);
-        ct.parse(ap);
-        ap(offload_copy,
-           {"--enable-offload-copy"},
-           ap.help("Enable implicit offload copying"),
-           ap.set_value(true));
-        ap(fast_math,
-           {"--disable-fast-math"},
-           ap.help("Disable fast math optimization"),
-           ap.set_value(false));
+        c.parse(ap);
         ap(tolerance, {"--tolerance"}, ap.help("Tolerance for errors"));
         ap(per_instruction,
            {"-i", "--per-instruction"},
            ap.help("Verify each instruction"),
            ap.set_value(true));
         ap(reduce, {"-r", "--reduce"}, ap.help("Reduce program and verify"), ap.set_value(true));
-        ap(quantize, {"--fp16"}, ap.help("Quantize for fp16"), ap.set_value(precision::fp16));
     }
 
     void run()
     {
-        auto p = l.load();
-        l.save(p);
+        auto p = c.l.load();
+        c.l.save(p);
         std::cout << p << std::endl;
 
-        compile_options options;
-        options.offload_copy = offload_copy;
-        options.fast_math    = fast_math;
-        auto t               = ct.get_target();
-        auto m               = parameters.generate(p, t, true);
+        auto t = c.ct.get_target();
+        auto m = c.parameters.generate(p, t, true);
 
         if(per_instruction)
         {
-            verify_instructions(p, t, options, quantize, tolerance);
+            verify_instructions(p, t, c.co, c.quantize, tolerance);
         }
         else if(reduce)
         {
-            verify_reduced_program(p, t, options, quantize, m, tolerance);
+            verify_reduced_program(p, t, c.co, c.quantize, m, tolerance);
         }
         else
         {
-            verify_program(l.file, p, t, options, quantize, m, tolerance);
+            verify_program(c.l.file, p, t, c.co, c.quantize, m, tolerance);
         }
     }
 };
@@ -466,7 +455,8 @@ struct version : command<version>
     void run() const
     {
         std::cout << "MIGraphX Version: " << MIGRAPHX_VERSION_MAJOR << "." << MIGRAPHX_VERSION_MINOR
-                  << std::endl;
+                  << "." << MIGRAPHX_VERSION_PATCH << "."
+                  << MIGRAPHX_STRINGIZE(MIGRAPHX_VERSION_TWEAK) << std::endl;
     }
 };
 
@@ -603,7 +593,9 @@ struct main_command
     void parse(argument_parser& ap)
     {
         std::string version_str = "MIGraphX Version: " + std::to_string(MIGRAPHX_VERSION_MAJOR) +
-                                  "." + std::to_string(MIGRAPHX_VERSION_MINOR);
+                                  "." + std::to_string(MIGRAPHX_VERSION_MINOR) + "." +
+                                  std::to_string(MIGRAPHX_VERSION_PATCH) + "." +
+                                  MIGRAPHX_STRINGIZE(MIGRAPHX_VERSION_TWEAK);
         ap(wrong_commands, {}, ap.metavar("<command>"), ap.append());
         ap(nullptr, {"-h", "--help"}, ap.help("Show help"), ap.show_help(get_command_help()));
         ap(nullptr,

src/driver/perf.cpp

@@ -108,8 +108,6 @@ target get_target(bool gpu)
         return make_target("cpu");
 }
 
-void compile_program(program& p, bool gpu) { p.compile(get_target(gpu)); }
-
 } // namespace  MIGRAPHX_INLINE_NS
 } // namespace driver
 } // namespace migraphx

src/driver/perf.hpp

@@ -37,7 +37,6 @@ parameter_map create_param_map(const program& p, const target& t, bool offload =
 parameter_map fill_param_map(parameter_map& m, const program& p, bool gpu);
 parameter_map create_param_map(const program& p, bool gpu = true);
 target get_target(bool gpu);
-void compile_program(program& p, bool gpu = true);
 
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace driver

src/driver/verify.cpp

@@ -24,7 +24,7 @@
 #include "verify.hpp"
 #include "perf.hpp"
 
-#include <migraphx/ref/target.hpp>
+#include <migraphx/register_target.hpp>
 #include <migraphx/generate.hpp>
 #include <migraphx/verify_args.hpp>
 #include <migraphx/instruction.hpp>
@@ -37,7 +37,7 @@ inline namespace MIGRAPHX_INLINE_NS {
 
 std::vector<argument> run_ref(program p, const parameter_map& inputs)
 {
-    p.compile(ref::target{});
+    p.compile(migraphx::make_target("ref"));
     auto out = p.eval(inputs);
     std::cout << p << std::endl;
     return out;

src/dynamic_loader.cpp

@@ -21,25 +21,44 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include <migraphx/manage_ptr.hpp>
 #include <migraphx/dynamic_loader.hpp>
 #include <migraphx/errors.hpp>
 #include <migraphx/file_buffer.hpp>
 #include <migraphx/tmp_dir.hpp>
 #include <utility>
-
 #include <dlfcn.h>
 
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 
+void check_load_error(bool flush = false)
+{
+    char* error_msg = dlerror();
+    if(not flush and error_msg != nullptr)
+        MIGRAPHX_THROW("Dynamic loading or symbol lookup failed with " + std::string(error_msg));
+}
+
 struct dynamic_loader_impl
 {
     dynamic_loader_impl() = default;
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wignored-attributes"
+#endif
     dynamic_loader_impl(const fs::path& p, std::shared_ptr<tmp_dir> t = nullptr)
-        : handle(dlopen(p.string().c_str(), RTLD_LAZY), &dlclose), temp(std::move(t))
+        : handle(dlopen(p.string().c_str(), RTLD_LAZY),
+                 manage_deleter<decltype(&dlclose), &dlclose>{}),
+          temp(std::move(t))
     {
+        check_load_error();
     }
 
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
     static std::shared_ptr<dynamic_loader_impl> from_buffer(const char* image, std::size_t size)
     {
         auto t = std::make_shared<tmp_dir>("dloader");
@@ -68,10 +87,11 @@ dynamic_loader::dynamic_loader(const std::vector<char>& buffer)
 
 std::shared_ptr<void> dynamic_loader::get_symbol(const std::string& name) const
 {
-    dlerror();
+    // flush any previous error messages
+    check_load_error(true);
     void* symbol = dlsym(impl->handle.get(), name.c_str());
     if(symbol == nullptr)
-        MIGRAPHX_THROW("Symbol not found: " + name);
+        check_load_error();
     return {impl, symbol};
 }
 

src/eliminate_allocation.cpp

@@ -28,11 +28,8 @@
 #include <migraphx/ranges.hpp>
 #include <migraphx/stringutils.hpp>
 #include <migraphx/serialize.hpp>
-
 #include <migraphx/make_op.hpp>
 
-#include <migraphx/pass_config.hpp>
-
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 

src/eliminate_data_type.cpp

@@ -38,6 +38,7 @@ void eliminate_data_type::apply(module& m) const
                                                            "if",
                                                            "loop",
                                                            "roialign",
+                                                           "nonmaxsuppression",
                                                            "scatternd_add",
                                                            "scatternd_mul",
                                                            "scatternd_none"};

src/include/migraphx/check_shapes.hpp

@@ -87,7 +87,7 @@ struct check_shapes
     }
 
     /*!
-     * Check if the number of shape objects is equal to atleast one of the
+     * Require the number of shape objects to equal to one of the
      * given sizes.
      * \param ns template parameter pack of sizes to check against
      */
@@ -100,6 +100,23 @@ struct check_shapes
         return *this;
     }
 
+    /*!
+     * Require the number of shape objects to equal at least a given amount.  Use this
+     * method for ops that can take any number (variadic) of inputs.
+     * \param n min. number of shapes
+     */
+    const check_shapes& has_at_least(std::size_t n) const
+    {
+        if(this->size() < n)
+            MIGRAPHX_THROW(prefix() + "Wrong number of arguments: expected at least " +
+                           to_string(n) + " but given " + std::to_string(size()));
+        return *this;
+    }
+
+    /*!
+     * Require all shapes to have the same number of elements.
+     * \param n  number of
+     */
     const check_shapes& nelements(std::size_t n) const
     {
         if(not this->all_of([&](const shape& s) { return s.elements() == n; }))

src/include/migraphx/compile_options.hpp

@@ -34,6 +34,7 @@ struct compile_options
 {
     bool offload_copy    = false;
     bool fast_math       = true;
+    bool exhaustive_tune = false;
     tracer trace{};
 };
 

src/include/migraphx/config.hpp

@@ -24,22 +24,32 @@
 #ifndef MIGRAPHX_GUARD_CONFIG_HPP
 #define MIGRAPHX_GUARD_CONFIG_HPP
 
-namespace migraphx {
-
 #if !defined(MIGRAPHX_USE_CLANG_TIDY) && !defined(DOXYGEN)
+
+#ifdef BUILD_DEV
 #define MIGRAPHX_INLINE_NS version_1
-#endif
+#else
+#include <migraphx/version.h>
+
+#define MIGRAPHX_VERSION_PRIMITIVE_CONCAT(x, y) x##_##y
+#define MIGRAPHX_VERSION_CONCAT(x, y) MIGRAPHX_VERSION_PRIMITIVE_CONCAT(x, y)
+
+#define MIGRAPHX_VERSION                                                         \
+    MIGRAPHX_VERSION_CONCAT(                                                     \
+        MIGRAPHX_VERSION_CONCAT(MIGRAPHX_VERSION_MAJOR, MIGRAPHX_VERSION_MINOR), \
+        MIGRAPHX_VERSION_PATCH)
+
+#define MIGRAPHX_INLINE_NS MIGRAPHX_VERSION_CONCAT(version, MIGRAPHX_VERSION)
+#endif // build_dev
+#endif // clang_tidy
 
 #ifdef DOXYGEN
 #define MIGRAPHX_INLINE_NS internal
-#endif
+#endif // doxygen
 
 #ifdef MIGRAPHX_USE_CLANG_TIDY
 #define MIGRAPHX_TIDY_CONST const
 #else
 #define MIGRAPHX_TIDY_CONST
-#endif
-
-} // namespace migraphx
-
-#endif
+#endif // tidy_const
+#endif // clang_tidy

src/include/migraphx/context.hpp

@@ -66,6 +66,7 @@ any_ptr get_queue_context(T&)
 {
     return {};
 }
+
 template <class T>
 void wait_for_context(T&, any_ptr)
 {
@@ -302,7 +303,7 @@ struct context
             PrivateDetailTypeErasedT value,
             typename std::enable_if<not std::is_reference<PrivateDetailTypeErasedU>::value,
                                     int>::type* = nullptr) noexcept
-            : private_detail_te_value(value)
+            : private_detail_te_value(std::move(value))
         {
         }
 
@@ -412,6 +413,7 @@ inline const ValueType& any_cast(const context& x)
 #endif
 
 inline void migraphx_to_value(value& v, const context& ctx) { v = ctx.to_value(); }
+
 inline void migraphx_from_value(const value& v, context& ctx) { ctx.from_value(v); }
 
 #endif

src/include/migraphx/half.hpp

@@ -25,7 +25,7 @@
 #ifndef MIGRAPHX_GUARD_RTGLIB_HALF_HPP
 #define MIGRAPHX_GUARD_RTGLIB_HALF_HPP
 
-#include <half.hpp>
+#include <half/half.hpp>
 #include <migraphx/config.hpp>
 
 namespace migraphx {
@@ -58,12 +58,12 @@ using deduce = typename detail::deduce<T>::type;
 namespace std {
 
 template <class T>
-struct common_type<migraphx::half, T> : std::common_type<float, T>
+struct common_type<migraphx::half, T> : std::common_type<float, T> // NOLINT
 {
 };
 
 template <class T>
-struct common_type<T, migraphx::half> : std::common_type<float, T>
+struct common_type<T, migraphx::half> : std::common_type<float, T> // NOLINT
 {
 };