merge develop branch changes

src/CMakeLists.txt

@@ -40,6 +40,7 @@ target_include_directories(migraphx SYSTEM PUBLIC $<BUILD_INTERFACE:${HALF_INCLU
 
 set(PACKAGE_DEPENDS)
 
+add_subdirectory(driver)
 add_subdirectory(onnx)
 add_subdirectory(tf)
 

src/driver/CMakeLists.txt

+
+add_executable(driver main.cpp verify.cpp perf.cpp)
+rocm_clang_tidy_check(driver)
+target_link_libraries(driver migraphx_cpu migraphx_onnx migraphx_tf)
+if(MIGRAPHX_ENABLE_GPU)
+target_link_libraries(driver migraphx_gpu)
+target_compile_definitions(driver PRIVATE -DHAVE_GPU)
+endif()

src/driver/argument_parser.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_ARGUMENT_PARSER_HPP
+#define MIGRAPHX_GUARD_RTGLIB_ARGUMENT_PARSER_HPP
+
+#include <algorithm>
+#include <functional>
+#include <iostream>
+#include <set>
+#include <string>
+#include <sstream>
+#include <type_traits>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <migraphx/config.hpp>
+#include <migraphx/requires.hpp>
+#include <migraphx/type_name.hpp>
+#include <migraphx/functional.hpp>
+#include <migraphx/stringutils.hpp>
+
+namespace migraphx {
+namespace driver {
+inline namespace MIGRAPHX_INLINE_NS {
+
+#ifdef MIGRAPHX_USE_CLANG_TIDY
+#define MIGRAPHX_DRIVER_STATIC
+#else
+#define MIGRAPHX_DRIVER_STATIC static
+#endif
+
+template <class T>
+struct value_parser
+{
+    template <MIGRAPHX_REQUIRES(not std::is_enum<T>{})>
+    static T apply(const std::string& x)
+    {
+        T result;
+        std::stringstream ss;
+        ss.str(x);
+        ss >> result;
+        if(ss.fail())
+            throw std::runtime_error("Failed to parse: " + x);
+        return result;
+    }
+
+    template <MIGRAPHX_REQUIRES(std::is_enum<T>{})>
+    static T apply(const std::string& x)
+    {
+        std::ptrdiff_t i;
+        std::stringstream ss;
+        ss.str(x);
+        ss >> i;
+        if(ss.fail())
+            throw std::runtime_error("Failed to parse: " + x);
+        return static_cast<T>(i);
+    }
+};
+
+struct argument_parser
+{
+    struct argument
+    {
+        std::vector<std::string> flags;
+        std::function<bool(argument_parser&, const std::vector<std::string>&)> action{};
+        std::string type          = "";
+        std::string help          = "";
+        std::string metavar       = "";
+        std::string default_value = "";
+        unsigned nargs            = 1;
+    };
+
+    template <class T, class... Fs>
+    void operator()(T& x, const std::vector<std::string>& flags, Fs... fs)
+    {
+        arguments.push_back({flags, [&](auto&&, const std::vector<std::string>& params) {
+                                 if(params.empty())
+                                     throw std::runtime_error("Flag with no value.");
+                                 x = value_parser<T>::apply(params.back());
+                                 return false;
+                             }});
+
+        argument& arg     = arguments.back();
+        arg.type          = migraphx::get_type_name<T>();
+        arg.default_value = to_string(x);
+        migraphx::each_args([&](auto f) { f(x, arg); }, fs...);
+    }
+
+    template <class... Fs>
+    void operator()(std::nullptr_t x, std::vector<std::string> flags, Fs... fs)
+    {
+        arguments.push_back({std::move(flags)});
+
+        argument& arg = arguments.back();
+        arg.type      = "";
+        arg.nargs     = 0;
+        migraphx::each_args([&](auto f) { f(x, arg); }, fs...);
+    }
+
+    MIGRAPHX_DRIVER_STATIC auto nargs(unsigned n = 1)
+    {
+        return [=](auto&&, auto& arg) { arg.nargs = n; };
+    }
+
+    template <class F>
+    MIGRAPHX_DRIVER_STATIC auto write_action(F f)
+    {
+        return [=](auto& x, auto& arg) {
+            arg.action = [&, f](auto& self, const std::vector<std::string>& params) {
+                f(self, x, params);
+                return false;
+            };
+        };
+    }
+
+    template <class F>
+    MIGRAPHX_DRIVER_STATIC auto do_action(F f)
+    {
+        return [=](auto&, auto& arg) {
+            arg.nargs  = 0;
+            arg.action = [&, f](auto& self, const std::vector<std::string>&) {
+                f(self);
+                return true;
+            };
+        };
+    }
+
+    MIGRAPHX_DRIVER_STATIC auto append()
+    {
+        return write_action([](auto&, auto& x, auto& params) {
+            using type = typename decltype(params)::value_type;
+            std::transform(params.begin(),
+                           params.end(),
+                           std::inserter(x, x.end()),
+                           [](std::string y) { return value_parser<type>::apply(y); });
+        });
+    }
+
+    MIGRAPHX_DRIVER_STATIC auto show_help(const std::string& msg = "")
+    {
+        return do_action([=](auto& self) {
+            for(auto&& arg : self.arguments)
+            {
+                std::cout << std::endl;
+                std::string prefix = "    ";
+                if(arg.flags.empty())
+                {
+                    std::cout << prefix;
+                    std::cout << arg.metavar;
+                }
+                for(const std::string& a : arg.flags)
+                {
+                    std::cout << prefix;
+                    std::cout << a;
+                    prefix = ", ";
+                }
+                if(not arg.type.empty())
+                {
+                    std::cout << " [" << arg.type << "]";
+                    if(not arg.default_value.empty())
+                        std::cout << " (Default: " << arg.default_value << ")";
+                }
+                std::cout << std::endl;
+                std::cout << "        " << arg.help << std::endl;
+            }
+            std::cout << std::endl;
+            if(not msg.empty())
+                std::cout << msg << std::endl;
+        });
+    }
+
+    MIGRAPHX_DRIVER_STATIC auto help(const std::string& help)
+    {
+        return [=](auto&, auto& arg) { arg.help = help; };
+    }
+
+    MIGRAPHX_DRIVER_STATIC auto metavar(const std::string& metavar)
+    {
+        return [=](auto&, auto& arg) { arg.metavar = metavar; };
+    }
+
+    template <class T>
+    MIGRAPHX_DRIVER_STATIC auto set_value(T value)
+    {
+        return [=](auto& x, auto& arg) {
+            arg.nargs  = 0;
+            arg.type   = "";
+            arg.action = [&, value](auto&, const std::vector<std::string>&) {
+                x = value;
+                return false;
+            };
+        };
+    }
+
+    bool parse(std::vector<std::string> args)
+    {
+        std::unordered_map<std::string, unsigned> keywords;
+        for(auto&& arg : arguments)
+        {
+            for(auto&& flag : arg.flags)
+                keywords[flag] = arg.nargs + 1;
+        }
+        auto arg_map =
+            generic_parse(std::move(args), [&](const std::string& x) { return keywords[x]; });
+        for(auto&& arg : arguments)
+        {
+            auto flags = arg.flags;
+            if(flags.empty())
+                flags = {""};
+            for(auto&& flag : flags)
+            {
+                if(arg_map.count(flag) > 0)
+                {
+                    if(arg.action(*this, arg_map[flag]))
+                        return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    using string_map = std::unordered_map<std::string, std::vector<std::string>>;
+    template <class IsKeyword>
+    static string_map generic_parse(std::vector<std::string> as, IsKeyword is_keyword)
+    {
+        string_map result;
+
+        std::string flag;
+        bool clear = false;
+        for(auto&& x : as)
+        {
+            auto k = is_keyword(x);
+            if(k > 0)
+            {
+                flag = x;
+                result[flag]; // Ensure the flag exists
+                if(k == 1)
+                    flag = "";
+                else if(k == 2)
+                    clear = true;
+                else
+                    clear = false;
+            }
+            else
+            {
+                result[flag].push_back(x);
+                if(clear)
+                    flag = "";
+                clear = false;
+            }
+        }
+        return result;
+    }
+
+    private:
+    std::vector<argument> arguments;
+};
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace driver
+} // namespace migraphx
+
+#endif

src/driver/command.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_COMMAND_HPP
+#define MIGRAPHX_GUARD_RTGLIB_COMMAND_HPP
+
+#include "argument_parser.hpp"
+
+#include <migraphx/config.hpp>
+#include <migraphx/type_name.hpp>
+#include <migraphx/stringutils.hpp>
+
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace migraphx {
+namespace driver {
+inline namespace MIGRAPHX_INLINE_NS {
+
+inline auto& get_commands()
+{
+    static std::unordered_map<std::string, std::function<void(std::vector<std::string> args)>> m;
+    return m;
+}
+
+template <class T>
+std::string compute_command_name()
+{
+    static const std::string& tname = get_type_name<T>();
+    auto name                       = tname.substr(tname.rfind("::") + 2);
+    if(ends_with(name, "_command"))
+        name = name.substr(0, name.size() - 8);
+    if(ends_with(name, "_cmd"))
+        name = name.substr(0, name.size() - 4);
+    return name;
+}
+
+template <class T>
+const std::string& command_name()
+{
+    static const std::string& name = compute_command_name<T>();
+    return name;
+}
+
+template <class T>
+void run_command(std::vector<std::string> args, bool add_help = false)
+{
+    T x;
+    argument_parser ap;
+    if(add_help)
+        ap(nullptr, {"-h", "--help"}, ap.help("Show help"), ap.show_help());
+    x.parse(ap);
+    if(ap.parse(std::move(args)))
+        return;
+    x.run();
+}
+
+template <class T>
+int auto_register_command()
+{
+    auto& m              = get_commands();
+    m[command_name<T>()] = [](std::vector<std::string> args) { run_command<T>(args, true); };
+    return 0;
+}
+
+template <class T>
+struct command
+{
+    static int static_register;
+    // This typedef ensures that the static member will be instantiated if
+    // the class itself is instantiated
+    using static_register_type =
+        std::integral_constant<decltype(&static_register), &static_register>;
+};
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wglobal-constructors"
+#endif
+
+template <class T>
+int command<T>::static_register = auto_register_command<T>(); // NOLINT
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace driver
+} // namespace migraphx
+
+#endif

src/driver/main.cpp

+#include "argument_parser.hpp"
+#include "command.hpp"
+#include "verify.hpp"
+#include "perf.hpp"
+
+#include <migraphx/tf.hpp>
+#include <migraphx/onnx.hpp>
+#include <migraphx/stringutils.hpp>
+
+#include <migraphx/pass_manager.hpp>
+#include <migraphx/dead_code_elimination.hpp>
+#include <migraphx/eliminate_identity.hpp>
+#include <migraphx/eliminate_pad.hpp>
+#include <migraphx/propagate_constant.hpp>
+#include <migraphx/simplify_algebra.hpp>
+#include <migraphx/simplify_reshapes.hpp>
+
+namespace migraphx {
+namespace driver {
+inline namespace MIGRAPHX_INLINE_NS {
+
+struct loader
+{
+    std::string file;
+    std::string file_type;
+    bool is_nhwc  = true;
+    unsigned trim = 0;
+    bool optimize = false;
+
+    void parse(argument_parser& ap)
+    {
+        ap(file, {}, ap.metavar("<input file>"));
+        ap(file_type, {"--onnx"}, ap.help("Load as onnx"), ap.set_value("onnx"));
+        ap(file_type, {"--tf"}, ap.help("Load as tensorflow"), ap.set_value("tf"));
+        ap(is_nhwc, {"--nhwc"}, ap.help("Treat tensorflow format as nhwc"), ap.set_value(true));
+        ap(is_nhwc, {"--nchw"}, ap.help("Treat tensorflow format as nchw"), ap.set_value(false));
+        ap(trim, {"--trim", "-t"}, ap.help("Trim instructions from the end"));
+        ap(optimize, {"--optimize"}, ap.help("Optimize when reading"), ap.set_value(true));
+    }
+
+    program load()
+    {
+        program p;
+        if(file_type.empty())
+        {
+            if(ends_with(file, ".onnx"))
+                file_type = "onnx";
+            else if(ends_with(file, ".pb"))
+                file_type = "tf";
+        }
+        std::cout << "Reading: " << file << std::endl;
+        if(file_type == "onnx")
+            p = parse_onnx(file);
+        else if(file_type == "tf")
+            p = parse_tf(file, is_nhwc);
+        if(trim > 0)
+        {
+            auto last = std::prev(p.end(), trim);
+            p.remove_instructions(last, p.end());
+        }
+        if(optimize)
+            migraphx::run_passes(p,
+                                 {
+                                     migraphx::eliminate_identity{},
+                                     migraphx::dead_code_elimination{},
+                                     migraphx::simplify_algebra{},
+                                     migraphx::dead_code_elimination{},
+                                     migraphx::simplify_reshapes{},
+                                     migraphx::dead_code_elimination{},
+                                     migraphx::propagate_constant{},
+                                     migraphx::dead_code_elimination{},
+                                     migraphx::eliminate_pad{},
+                                     migraphx::dead_code_elimination{},
+                                 });
+        return p;
+    }
+};
+
+struct compiler
+{
+    loader l;
+    bool gpu = true;
+    void parse(argument_parser& ap)
+    {
+        l.parse(ap);
+        ap(gpu, {"--gpu"}, ap.help("Compile on the gpu"), ap.set_value(true));
+        ap(gpu, {"--cpu"}, ap.help("Compile on the cpu"), ap.set_value(false));
+    }
+
+    program compile()
+    {
+        auto p = l.load();
+        compile_program(p, gpu);
+        return p;
+    }
+
+    auto params(const program& p) { return create_param_map(p, gpu); }
+};
+
+struct read : command<read>
+{
+    loader l;
+    void parse(argument_parser& ap) { l.parse(ap); }
+
+    void run()
+    {
+        auto p = l.load();
+        std::cout << p << std::endl;
+    }
+};
+
+struct verify : command<verify>
+{
+    loader l;
+    double tolerance     = 80;
+    bool per_instruction = false;
+    bool reduce          = false;
+    void parse(argument_parser& ap)
+    {
+        l.parse(ap);
+        ap(tolerance, {"--tolerance"}, ap.help("Tolerance for errors"));
+        ap(per_instruction,
+           {"-i", "--per-instruction"},
+           ap.help("Verify each instruction"),
+           ap.set_value(true));
+        ap(reduce, {"-r", "--reduce"}, ap.help("Reduce program and verify"), ap.set_value(true));
+    }
+
+    void run()
+    {
+        auto p = l.load();
+        std::cout << p << std::endl;
+
+        if(per_instruction)
+        {
+            verify_instructions(p, tolerance);
+        }
+        else if(reduce)
+        {
+            verify_reduced_program(p, tolerance);
+        }
+        else
+        {
+            verify_program(l.file, p, tolerance);
+        }
+    }
+};
+
+struct compile : command<compile>
+{
+    compiler c;
+    void parse(argument_parser& ap) { c.parse(ap); }
+
+    void run()
+    {
+        std::cout << "Compiling ... " << std::endl;
+        auto p = c.compile();
+        std::cout << p << std::endl;
+    }
+};
+
+struct run_cmd : command<run_cmd>
+{
+    compiler c;
+    void parse(argument_parser& ap) { c.parse(ap); }
+
+    void run()
+    {
+        std::cout << "Compiling ... " << std::endl;
+        auto p = c.compile();
+        std::cout << "Allocating params ... " << std::endl;
+        auto m = c.params(p);
+        p.eval(m);
+        std::cout << p << std::endl;
+    }
+};
+
+struct perf : command<perf>
+{
+    compiler c;
+    unsigned n = 100;
+    void parse(argument_parser& ap)
+    {
+        c.parse(ap);
+        ap(n, {"--iterations", "-n"}, ap.help("Number of iterations to run for perf report"));
+    }
+
+    void run()
+    {
+        std::cout << "Compiling ... " << std::endl;
+        auto p = c.compile();
+        std::cout << "Allocating params ... " << std::endl;
+        auto m = c.params(p);
+        std::cout << "Running performance report ... " << std::endl;
+        p.perf_report(std::cout, n, m);
+    }
+};
+
+struct main_command
+{
+    static std::string get_command_help()
+    {
+        std::string result = "Commands:\n";
+        return std::accumulate(get_commands().begin(),
+                               get_commands().end(),
+                               result,
+                               [](auto r, auto&& p) { return r + "    " + p.first + "\n"; });
+    }
+    void parse(argument_parser& ap)
+    {
+        ap(nullptr, {"-h", "--help"}, ap.help("Show help"), ap.show_help(get_command_help()));
+    }
+
+    void run() {}
+};
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace driver
+} // namespace migraphx
+
+using namespace migraphx::driver; // NOLINT
+int main(int argc, const char* argv[])
+{
+    std::vector<std::string> args(argv + 1, argv + argc);
+    if(args.empty())
+        return 0;
+    auto&& m = get_commands();
+    auto cmd = args.front();
+    if(m.count(cmd) > 0)
+    {
+        m.at(cmd)({args.begin() + 1, args.end()});
+    }
+    else
+    {
+        run_command<main_command>(args);
+    }
+    return 0;
+}

src/driver/perf.cpp

+#include "perf.hpp"
+
+#include <migraphx/cpu/target.hpp>
+#include <migraphx/generate.hpp>
+#ifdef HAVE_GPU
+#include <migraphx/gpu/target.hpp>
+#include <migraphx/gpu/hip.hpp>
+#endif
+
+namespace migraphx {
+namespace driver {
+inline namespace MIGRAPHX_INLINE_NS {
+
+program::parameter_map create_param_map(const program& p, bool gpu)
+{
+    program::parameter_map m;
+    for(auto&& x : p.get_parameter_shapes())
+    {
+#ifdef HAVE_GPU
+        if(gpu)
+            m[x.first] = gpu::to_gpu(generate_argument(x.second));
+        else
+#else
+        (void)gpu;
+#endif
+            m[x.first] = generate_argument(x.second);
+    }
+    return m;
+}
+
+void compile_program(program& p, bool gpu)
+{
+    if(gpu)
+    {
+#ifdef HAVE_GPU
+        p.compile(gpu::target{});
+#else
+        MIGRAPHX_THROW("Gpu not supported.");
+#endif
+    }
+    else
+    {
+        p.compile(cpu::target{});
+    }
+}
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace driver
+} // namespace migraphx

src/driver/perf.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_PERF_HPP
+#define MIGRAPHX_GUARD_RTGLIB_PERF_HPP
+
+#include <migraphx/program.hpp>
+
+namespace migraphx {
+namespace driver {
+inline namespace MIGRAPHX_INLINE_NS {
+
+program::parameter_map create_param_map(const program& p, bool gpu = true);
+void compile_program(program& p, bool gpu = true);
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace driver
+} // namespace migraphx
+
+#endif

src/driver/verify.cpp

+#include "verify.hpp"
+
+#include <migraphx/cpu/target.hpp>
+#include <migraphx/generate.hpp>
+#include <migraphx/verify_args.hpp>
+#include <migraphx/instruction.hpp>
+
+#ifdef HAVE_GPU
+#include <migraphx/gpu/target.hpp>
+#include <migraphx/gpu/hip.hpp>
+#endif
+
+namespace migraphx {
+namespace driver {
+inline namespace MIGRAPHX_INLINE_NS {
+
+template <class T>
+auto get_hash(const T& x)
+{
+    return std::hash<T>{}(x);
+}
+
+argument run_cpu(program p)
+{
+    p.compile(cpu::target{});
+    program::parameter_map m;
+    for(auto&& x : p.get_parameter_shapes())
+    {
+        m[x.first] = generate_argument(x.second, get_hash(x.first));
+    }
+    auto out = p.eval(m);
+    std::cout << p << std::endl;
+    return out;
+}
+
+argument run_gpu(program p)
+{
+#ifdef HAVE_GPU
+    p.compile(gpu::target{});
+
+    program::parameter_map m;
+    for(auto&& x : p.get_parameter_shapes())
+    {
+        m[x.first] = gpu::to_gpu(generate_argument(x.second, get_hash(x.first)));
+    }
+    auto out = gpu::from_gpu(p.eval(m));
+    std::cout << p << std::endl;
+    return gpu::from_gpu(out);
+#else
+    (void)p;
+    MIGRAPHX_THROW("Gpu unsupported!");
+#endif
+}
+
+void verify_program(const std::string& name, const program& p, double tolerance)
+{
+    auto x = run_cpu(p);
+    auto y = run_gpu(p);
+    verify_args(name, x, y, tolerance);
+    // std::cout << "cpu: " << x << std::endl;
+    // std::cout << "gpu: " << y << std::endl;
+}
+
+void verify_instructions(const program& prog, double tolerance)
+{
+    for(auto&& ins : prog)
+    {
+        if(ins.name().front() == '@')
+            continue;
+        if(ins.name() == "broadcast")
+            continue;
+        if(ins.name() == "transpose")
+            continue;
+        if(ins.name() == "reshape")
+            continue;
+        program p;
+        std::vector<instruction_ref> inputs;
+        for(auto&& arg : ins.inputs())
+        {
+            if(arg->name() == "@literal")
+                inputs.push_back(p.add_literal(arg->get_literal()));
+            else
+                inputs.push_back(p.add_parameter(std::to_string(inputs.size()), arg->get_shape()));
+        }
+        p.add_instruction(ins.get_operator(), inputs);
+        try
+        {
+            std::cout << "Verify: " << ins.name() << std::endl;
+            std::cout << p << std::endl;
+            verify_program(ins.name(), p, tolerance);
+        }
+        catch(...)
+        {
+            std::cout << "Instruction " << ins.name() << " threw an exception." << std::endl;
+            throw;
+        }
+    }
+}
+
+void verify_reduced(program p, int n, double tolerance)
+{
+    auto last = std::prev(p.end(), n + 1);
+    p.remove_instructions(last, p.end());
+    std::cout << "Verify: " << std::endl;
+    std::cout << p << std::endl;
+    verify_program(std::to_string(n), p, tolerance);
+}
+
+void verify_reduced_program(const program& p, double tolerance)
+{
+    auto n = std::distance(p.begin(), p.end());
+    for(std::size_t i = 0; i < n; i++)
+    {
+        verify_reduced(p, i, tolerance);
+    }
+}
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace driver
+} // namespace migraphx

src/driver/verify.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_DRIVER_VERIFY_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DRIVER_VERIFY_HPP
+
+#include <migraphx/program.hpp>
+
+namespace migraphx {
+namespace driver {
+inline namespace MIGRAPHX_INLINE_NS {
+
+argument run_cpu(program p);
+argument run_gpu(program p);
+void verify_program(const std::string& name, const program& p, double tolerance = 100);
+void verify_instructions(const program& prog, double tolerance = 80);
+void verify_reduced_program(const program& p, double tolerance = 80);
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace driver
+} // namespace migraphx
+
+#endif

src/eliminate_pad.cpp

@@ -43,9 +43,7 @@ void eliminate_pad::update_op(T,
     std::vector<int64_t> pads = pad_op.pads;
     std::array<size_t, 2> new_pads{static_cast<size_t>(pads[2]), static_cast<size_t>(pads[3])};
 
-    T op = any_cast<T>(ins->get_operator());
-    if(op.padding_mode != op::padding_mode_t::default_)
-        return;
+    T op       = any_cast<T>(ins->get_operator());
     op.padding = new_pads;
 
     std::vector<instruction_ref> new_inputs{ins->inputs()};

src/include/migraphx/array.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_ARRAY_HPP
+#define MIGRAPHX_GUARD_RTGLIB_ARRAY_HPP
+
+#include <migraphx/config.hpp>
+#include <migraphx/functional.hpp>
+#include <migraphx/requires.hpp>
+#include <type_traits>
+#include <array>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+namespace detail {
+
+template <class R, class...>
+struct array_type
+{
+    using type = R;
+};
+template <class... Ts>
+struct array_type<void, Ts...> : std::common_type<Ts...>
+{
+};
+
+template <class R, class... Ts>
+using array_type_t = typename array_type<R, Ts...>::type;
+
+template <class T, std::size_t N, std::size_t... I>
+constexpr std::array<std::remove_cv_t<T>, N> to_array_impl(T (&a)[N], seq<I...>)
+{
+    return {{a[I]...}};
+}
+
+} // namespace detail
+
+template <class Result = void, class... Ts, MIGRAPHX_REQUIRES((sizeof...(Ts) > 0))>
+constexpr std::array<detail::array_type_t<Result, Ts...>, sizeof...(Ts)> make_array(Ts&&... xs)
+{
+    return {static_cast<detail::array_type_t<Result, Ts...>>(std::forward<Ts>(xs))...};
+}
+
+constexpr std::array<int, 0> make_array() { return {}; }
+
+template <class T, std::size_t N>
+constexpr auto to_array(T (&a)[N])
+{
+    return detail::to_array_impl(a, detail::gens<N>{});
+}
+
+namespace detail {
+
+template <std::size_t Offset = 0, class Array, std::size_t... I>
+constexpr auto rearray_impl(Array a, seq<I...>)
+{
+    return make_array(a[I + Offset]...);
+}
+
+} // namespace detail
+
+template <class T, std::size_t N>
+constexpr auto pop_front(std::array<T, N> a)
+{
+    return detail::rearray_impl(a, detail::gens<N - 1>{});
+}
+
+template <class T, std::size_t N>
+constexpr auto pop_back(std::array<T, N> a)
+{
+    return detail::rearray_impl<1>(a, detail::gens<N - 1>{});
+}
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/functional.hpp

@@ -15,6 +15,12 @@ struct swallow
     }
 };
 
+template <class T>
+auto tuple_size(const T&)
+{
+    return typename std::tuple_size<T>::type{};
+}
+
 namespace detail {
 
 template <class R, class F>
@@ -83,6 +89,12 @@ constexpr auto sequence_c(F&& f)
     return detail::sequence_c_impl(f, detail::gens<N>{});
 }
 
+template <class IntegerConstant, class F>
+constexpr auto sequence(IntegerConstant ic, F&& f)
+{
+    return sequence_c<ic>(f);
+}
+
 template <class F, class... Ts>
 constexpr void each_args(F f, Ts&&... xs)
 {
@@ -95,9 +107,9 @@ constexpr void each_args(F)
 }
 
 template <class F, class T>
-auto unpack(F f, T& x)
+auto unpack(F f, T&& x)
 {
-    return sequence_c<std::tuple_size<T>{}>([&](auto... is) { f(std::get<is>(x)...); });
+    return sequence(tuple_size(x), [&](auto... is) { f(std::get<is>(static_cast<T&&>(x))...); });
 }
 
 /// Implements a fix-point combinator
@@ -149,6 +161,52 @@ auto index_of(T& x)
     return [&](auto&& y) { return x[y]; };
 }
 
+template <class T, class... Ts>
+decltype(auto) front_args(T&& x, Ts&&...)
+{
+    return static_cast<T&&>(x);
+}
+
+template <class... Ts>
+decltype(auto) back_args(Ts&&... xs)
+{
+    return std::get<sizeof...(Ts) - 1>(std::tuple<Ts&&...>(static_cast<Ts&&>(xs)...));
+}
+
+template <class T, class... Ts>
+auto pop_front_args(T&&, Ts&&... xs)
+{
+    return [&](auto f) { f(static_cast<Ts&&>(xs)...); };
+}
+
+template <class... Ts>
+auto pop_back_args(Ts&&... xs)
+{
+    return [&](auto f) {
+        using tuple_type = std::tuple<Ts&&...>;
+        auto t           = tuple_type(static_cast<Ts&&>(xs)...);
+        return sequence_c<sizeof...(Ts) - 1>(
+            [&](auto... is) { return f(std::get<is>(static_cast<tuple_type&&>(t))...); });
+    };
+}
+
+template <class T>
+struct always_f
+{
+    T x;
+    template <class... Ts>
+    constexpr T operator()(Ts&&...) const
+    {
+        return x;
+    }
+};
+
+template <class T>
+auto always(T x)
+{
+    return always_f<T>{x};
+}
+
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx
 

src/include/migraphx/generate.hpp

@@ -25,7 +25,7 @@ constexpr T normalize(unsigned long z)
 template <class T, MIGRAPHX_REQUIRES(is_signed<T>{} and not is_floating_point<T>{})>
 constexpr T normalize(unsigned long z)
 {
-    const auto max      = std::numeric_limits<T>::max();
+    const auto max      = std::numeric_limits<T>::max() / 64;
     const auto half_max = max / 2;
     return half_max - (z % max);
 }
@@ -33,7 +33,7 @@ constexpr T normalize(unsigned long z)
 template <class T, MIGRAPHX_REQUIRES(not is_signed<T>{} and std::is_integral<T>{})>
 constexpr T normalize(unsigned long z)
 {
-    const auto max = std::numeric_limits<T>::max();
+    const auto max = std::numeric_limits<T>::max() / 64;
     return z % max;
 }
 

src/include/migraphx/literal.hpp

@@ -79,6 +79,7 @@ struct literal : raw_data<literal>
     template <class Iterator>
     void fill(Iterator start, Iterator end)
     {
+        assert(std::distance(start, end) == m_shape.elements());
         if(m_shape.standard())
         {
             m_shape.visit_type([&](auto as) { std::copy(start, end, as.from(buffer.get())); });

src/include/migraphx/matcher.hpp

@@ -8,6 +8,7 @@
 #include <migraphx/iterator_for.hpp>
 #include <migraphx/config.hpp>
 #include <unordered_map>
+#include <unordered_set>
 
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
@@ -20,6 +21,12 @@ struct matcher_context
     std::unordered_map<std::string, instruction_ref> instructions;
     instruction_ref not_found() const { return last; }
 
+    template <class M>
+    bool matched(M m, instruction_ref ins)
+    {
+        return m.match(*this, ins) != this->not_found();
+    }
+
     private:
     instruction_ref last;
 };
@@ -205,74 +212,147 @@ matcher_result match_instruction(program& p, instruction_ref ins, M&& m)
     return result;
 }
 
+/// Find matches for an instruction in the program
+template <class... Ms>
+void find_matches(program& p, instruction_ref ins, Ms&&... ms)
+{
+    bool match = false;
+    each_args(
+        [&](auto&& m) {
+            if(match)
+                return;
+            auto r = match_instruction(p, ins, m.matcher());
+            if(r.result == p.end())
+                return;
+            m.apply(p, r);
+            match = true;
+        },
+        ms...);
+}
+
 /// Find matches in a program
 template <class... Ms>
 void find_matches(program& p, Ms&&... ms)
 {
     for(auto ins : iterator_for(p))
     {
-        bool match = false;
-        each_args(
-            [&](auto&& m) {
-                if(match)
-                    return;
-                auto r = match_instruction(p, ins, m.matcher());
-                if(r.result == p.end())
-                    return;
-                m.apply(p, r);
-                match = true;
-            },
-            ms...);
+        find_matches(p, ins, ms...);
     }
 }
 
-template <class... Ts>
-auto all_of(Ts... ms)
+struct lazy_and
 {
-    return make_bf_matcher([=](matcher_context& ctx, instruction_ref ins) {
-        bool matches = fold([&](auto x, auto y) {
-            return x and y.match(ctx, ins) != ctx.not_found();
-        })(true, ms...);
-        if(matches)
-            return ins;
-        return ctx.not_found();
-    });
-}
+    template <class F, class G>
+    bool operator()(F f, G g) const
+    {
+        return f() and g();
+    }
+};
 
-template <class... Ts>
-auto none_of(Ts... ms)
+struct lazy_or
 {
-    return make_bf_matcher([=](matcher_context& ctx, instruction_ref ins) {
-        bool matches = fold([&](auto x, auto y) {
-            return x and y.match(ctx, ins) == ctx.not_found();
-        })(true, ms...);
-        if(matches)
-            return ins;
-        return ctx.not_found();
-    });
+    template <class F, class G>
+    bool operator()(F f, G g) const
+    {
+        return f() or g();
+    }
+};
+
+template <class Op, bool Start, bool Matches>
+struct match_fold_f
+{
+    template <class... Ms>
+    static bool fold_matchers(matcher_context& ctx, instruction_ref ins, Ms... ms)
+    {
+        Op op;
+        auto matched = [&](auto m) { return [=, &ctx] { return ctx.matched(m, ins); }; };
+        return fold([&](auto x, auto y) { return op(always(x), matched(y)); })(Start, ms...);
+    }
+
+    template <class Pack>
+    static bool fold_matchers_pack(matcher_context& ctx, instruction_ref ins, Pack p)
+    {
+        return p([&](auto... ms) { return match_fold_f::fold_matchers(ctx, ins, ms...); });
+    }
+
+    template <class... Ts>
+    auto operator()(Ts... ms) const
+    {
+        return make_bf_matcher([=](matcher_context& ctx, instruction_ref ins) {
+            bool matches = match_fold_f::fold_matchers(ctx, ins, ms...);
+            if(matches == Matches)
+                return ins;
+            return ctx.not_found();
+        });
+    }
+
+    template <class Selector>
+    auto operator[](Selector select) const
+    {
+        return [=](auto... ms) {
+            // Workaround ICE on gcc by packing matchers into an object
+            auto mpack = pack(ms...);
+            return make_bf_matcher([=](matcher_context& ctx, instruction_ref start) {
+                Op op;
+                bool matches = Start;
+                select(start, [&](auto ins) {
+                    auto fm = [&] { return match_fold_f::fold_matchers_pack(ctx, ins, mpack); };
+                    matches = op(always(matches), fm);
+                });
+                if(matches == Matches)
+                    return start;
+                return ctx.not_found();
+            });
+        };
+    }
+};
+
+const constexpr auto all_of  = match_fold_f<lazy_and, true, true>{};
+const constexpr auto any_of  = match_fold_f<lazy_or, false, true>{};
+const constexpr auto none_of = match_fold_f<lazy_or, false, false>{};
+
+inline auto inputs()
+{
+    return [](auto ins, auto f) {
+        for(auto&& x : ins->inputs())
+            f(x);
+    };
 }
 
-template <class... Ts>
-auto any_of(Ts... ms)
+inline auto outputs()
 {
-    return make_bf_matcher([=](matcher_context& ctx, instruction_ref ins) {
-        bool matches = fold([&](auto x, auto y) {
-            return x or y.match(ctx, ins) != ctx.not_found();
-        })(false, ms...);
-        if(matches)
-            return ins;
-        return ctx.not_found();
-    });
+    return [](auto ins, auto f) {
+        for(auto&& x : ins->outputs())
+            f(x);
+    };
 }
 
 MIGRAPHX_PRED_MATCHER(any, instruction_ref) { return true; }
 MIGRAPHX_PRED_MATCHER(none, instruction_ref) { return false; }
 MIGRAPHX_PRED_MATCHER(standard_shape, instruction_ref ins) { return ins->get_shape().standard(); }
+MIGRAPHX_PRED_MATCHER(not_standard_shape, instruction_ref ins)
+{
+    return not ins->get_shape().standard();
+}
 MIGRAPHX_PRED_MATCHER(broadcast_shape, instruction_ref ins)
 {
     return ins->get_shape().broadcasted();
 }
 
+MIGRAPHX_PRED_MATCHER(transpose_shape, instruction_ref ins)
+{
+    return ins->get_shape().transposed();
+}
+
+MIGRAPHX_PRED_MATCHER(same_input_shapes, instruction_ref ins)
+{
+    if(ins->inputs().empty())
+        return false;
+    auto s = ins->inputs().front()->get_shape();
+    return std::all_of(
+        ins->inputs().begin(), ins->inputs().end(), [&](auto x) { return x->get_shape() == s; });
+}
+
 MIGRAPHX_BASIC_MATCHER(output, matcher_context& ctx, instruction_ref ins)
 {
     if(ins->outputs().size() == 1)
@@ -289,10 +369,39 @@ MIGRAPHX_BASIC_MATCHER(used_once, matcher_context& ctx, instruction_ref ins)
     return ctx.not_found();
 }
 
-inline auto name(std::string name)
+template <class... Ms>
+auto skip_output(Ms... ms)
+{
+    auto m = any_of(ms...);
+    return make_basic_fun_matcher([=](matcher_context& ctx, instruction_ref start) {
+        return fix<instruction_ref>([&](auto self, auto ins) {
+            if(ins->outputs().size() == 1)
+            {
+                auto next = ins->outputs().front();
+                if(ctx.matched(m, next))
+                {
+                    auto skipped_next = self(next);
+                    if(skipped_next != ctx.not_found())
+                        return skipped_next;
+                }
+                return next;
+            }
+            return ctx.not_found();
+        })(start);
+    });
+}
+
+inline auto name(std::string s)
 {
     return make_basic_pred_matcher(
-        [ =, name = std::move(name) ](instruction_ref ins) { return ins->name() == name; });
+        [ =, s = std::move(s) ](instruction_ref ins) { return ins->name() == s; });
+}
+
+inline auto name(std::unordered_set<std::string> names)
+{
+    return make_basic_pred_matcher([ =, names = std::move(names) ](instruction_ref ins) {
+        return names.count(ins->name()) > 0;
+    });
 }
 
 inline auto nargs(std::size_t n)
@@ -338,6 +447,23 @@ inline auto either_arg(std::size_t i, std::size_t j)
     };
 }
 
+template <class M>
+auto same_shape(M m)
+{
+    return make_basic_fun_matcher([=](matcher_context& ctx, instruction_ref ins) {
+        auto i = m.match(ctx, ins);
+        if(i != ctx.not_found() and i->get_shape() == ins->get_shape())
+            return ins;
+        return ctx.not_found();
+    });
+}
+
+template <class... Ms>
+auto same_shape(Ms... ms)
+{
+    return all_of(same_shape(ms)...);
+}
+
 } // namespace match
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx

src/include/migraphx/op/argmax.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_ARGMAX_HPP
+#define MIGRAPHX_GUARD_OPERATORS_ARGMAX_HPP
+
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/par_dfor.hpp>
+#include <migraphx/config.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+struct argmax
+{
+    int64_t axis = 0;
+
+    template <class Self, class F>
+    static auto reflect(Self& self, F f)
+    {
+        return pack(f(self.axis, "axis"));
+    }
+
+    std::string name() const { return "argmax"; }
+
+    shape compute_shape(std::vector<shape> inputs) const
+    {
+        check_shapes{inputs, *this}.has(1).standard();
+        auto lens     = inputs[0].lens();
+        int64_t n_dim = static_cast<int64_t>(lens.size());
+        if(axis >= n_dim || axis < 0)
+        {
+            MIGRAPHX_THROW("ARGMAX: axis is out of range.");
+        }
+
+        lens[axis] = 1;
+
+        return {shape::int64_type, lens};
+    }
+
+    template <class T>
+    int64_t calc_argmax(T& input, std::vector<std::size_t>& indices, size_t item_num) const
+    {
+        auto max_val      = input(indices.begin(), indices.end());
+        int64_t max_index = 0;
+        for(std::size_t i = 1; i < item_num; ++i)
+        {
+            indices[axis] = i;
+            auto cur_val  = input(indices.begin(), indices.end());
+            if(max_val < cur_val)
+            {
+                max_val   = cur_val;
+                max_index = i;
+            }
+        }
+
+        return max_index;
+    }
+
+    argument compute(const shape& output_shape, std::vector<argument> args) const
+    {
+        argument result{output_shape};
+        auto batch_item_num = args.front().get_shape().lens()[axis];
+
+        result.visit([&](auto output) {
+            args[0].visit([&](auto input) {
+                par_for(output_shape.elements(), [&](auto i) {
+                    auto data_idx = output_shape.multi(i);
+                    output[i]     = this->calc_argmax(input, data_idx, batch_item_num);
+                });
+            });
+        });
+
+        return result;
+    }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/argmin.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_ARGMIN_HPP
+#define MIGRAPHX_GUARD_OPERATORS_ARGMIN_HPP
+
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/par_dfor.hpp>
+#include <migraphx/config.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+struct argmin
+{
+    int64_t axis = 0;
+
+    template <class Self, class F>
+    static auto reflect(Self& self, F f)
+    {
+        return pack(f(self.axis, "axis"));
+    }
+
+    std::string name() const { return "argmin"; }
+
+    shape compute_shape(std::vector<shape> inputs) const
+    {
+        check_shapes{inputs, *this}.has(1).standard();
+        auto lens     = inputs[0].lens();
+        int64_t n_dim = static_cast<int64_t>(lens.size());
+        if(axis >= n_dim || axis < 0)
+        {
+            MIGRAPHX_THROW("ARGMIN: axis is out of range.");
+        }
+
+        lens[axis] = 1;
+
+        return {shape::int64_type, lens};
+    }
+
+    template <class T>
+    int64_t calc_argmin(T& input, std::vector<std::size_t>& indices, size_t item_num) const
+    {
+        auto min_val      = input(indices.begin(), indices.end());
+        int64_t min_index = 0;
+        for(std::size_t i = 1; i < item_num; ++i)
+        {
+            indices[axis] = i;
+            auto cur_val  = input(indices.begin(), indices.end());
+            if(min_val > cur_val)
+            {
+                min_val   = cur_val;
+                min_index = i;
+            }
+        }
+
+        return min_index;
+    }
+
+    argument compute(const shape& output_shape, std::vector<argument> args) const
+    {
+        argument result{output_shape};
+        std::size_t batch_item_num = args.front().get_shape().lens()[axis];
+
+        result.visit([&](auto output) {
+            args[0].visit([&](auto input) {
+                par_for(output_shape.elements(), [&](auto i) {
+                    auto data_idx = output_shape.multi(i);
+                    output[i]     = this->calc_argmin(input, data_idx, batch_item_num);
+                });
+            });
+        });
+
+        return result;
+    }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/convolution.hpp

@@ -44,51 +44,24 @@ struct convolution
         const shape& input   = inputs.at(0);
         const shape& weights = inputs.at(1);
         auto t               = input.type();
-        if(padding_mode == default_)
-        {
-            return {t,
-                    {
-                        input.lens()[0],
-                        weights.lens()[0],
-                        std::size_t(std::max<std::ptrdiff_t>(
-                            1,
-                            (input.lens()[2] - (1 + dilation[0] * (weights.lens()[2] - 1)) +
-                             2 * padding[0]) /
-                                    stride[0] +
-                                1)),
-                        std::size_t(std::max<std::ptrdiff_t>(
-                            1,
-                            (input.lens()[3] - (1 + dilation[1] * (weights.lens()[3] - 1)) +
-                             2 * padding[1]) /
-                                    stride[1] +
-                                1)),
-                    }};
-        }
-        else if(padding_mode == same)
-        {
-            return {t,
-                    {input.lens()[0],
-                     weights.lens()[0],
-                     static_cast<std::size_t>(
-                         std::ceil(static_cast<double>(input.lens()[2]) / stride[0])),
-                     static_cast<std::size_t>(
-                         std::ceil(static_cast<double>(input.lens()[3]) / stride[1]))}};
-        }
-        else if(padding_mode == valid)
-        {
-            return {
-                t,
-                {input.lens()[0],
-                 weights.lens()[0],
-                 static_cast<std::size_t>(std::ceil(
-                     static_cast<double>(input.lens()[2] - weights.lens()[2] + 1) / stride[0])),
-                 static_cast<std::size_t>(std::ceil(
-                     static_cast<double>(input.lens()[3] - weights.lens()[3] + 1) / stride[1]))}};
-        }
-        else
-        {
-            MIGRAPHX_THROW("Invalid padding mode");
-        }
+
+        return {t,
+                {
+                    input.lens()[0],
+                    weights.lens()[0],
+                    std::size_t(std::max<std::ptrdiff_t>(
+                        1,
+                        (input.lens()[2] - (1 + dilation[0] * (weights.lens()[2] - 1)) +
+                         2 * padding[0]) /
+                                stride[0] +
+                            1)),
+                    std::size_t(std::max<std::ptrdiff_t>(
+                        1,
+                        (input.lens()[3] - (1 + dilation[1] * (weights.lens()[3] - 1)) +
+                         2 * padding[1]) /
+                                stride[1] +
+                            1)),
+                }};
     }
 };
 

src/include/migraphx/op/erf.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_ERF_HPP
+#define MIGRAPHX_GUARD_OPERATORS_ERF_HPP
+
+#include <migraphx/op/unary.hpp>
+#include <migraphx/config.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+struct erf : unary<erf>
+{
+    auto apply() const
+    {
+        return [](auto x) { return std::erf(x); };
+    }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/logsoftmax.hpp

 #ifndef MIGRAPHX_GUARD_OPERATORS_LOGSOFTMAX_HPP
 #define MIGRAPHX_GUARD_OPERATORS_LOGSOFTMAX_HPP
 
-#include <array>
 #include <migraphx/operation.hpp>
 #include <migraphx/check_shapes.hpp>
-#include <migraphx/stringutils.hpp>
-#include <migraphx/streamutils.hpp>
-#include <migraphx/literal.hpp>
-#include <migraphx/shape_for_each.hpp>
 #include <migraphx/config.hpp>
-#include <cmath>
-#include <utility>
 
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {