merge changes from develop branch and resolve merge conflicts

test/stringutils.cpp

+#include <migraphx/stringutils.hpp>
+#include <test.hpp>
+
+TEST_CASE(interpolate_string_simple1)
+{
+    std::string input = "Hello ${w}!";
+    auto s            = migraphx::interpolate_string(input, {{"w", "world"}});
+    EXPECT(s == "Hello world!");
+}
+
+TEST_CASE(interpolate_string_simple2)
+{
+    std::string input = "${hello}";
+    auto s            = migraphx::interpolate_string(input, {{"hello", "bye"}});
+    EXPECT(s == "bye");
+}
+
+TEST_CASE(interpolate_string_unbalanced)
+{
+    std::string input = "${hello";
+    EXPECT(test::throws([&] { migraphx::interpolate_string(input, {{"hello", "bye"}}); }));
+}
+
+TEST_CASE(interpolate_string_extra_space)
+{
+    std::string input = "${  hello  }";
+    auto s            = migraphx::interpolate_string(input, {{"hello", "bye"}});
+    EXPECT(s == "bye");
+}
+
+TEST_CASE(interpolate_string_multiple)
+{
+    std::string input = "${h} ${w}!";
+    auto s            = migraphx::interpolate_string(input, {{"w", "world"}, {"h", "Hello"}});
+    EXPECT(s == "Hello world!");
+}
+
+TEST_CASE(interpolate_string_next)
+{
+    std::string input = "${hh}${ww}!";
+    auto s            = migraphx::interpolate_string(input, {{"ww", "world"}, {"hh", "Hello"}});
+    EXPECT(s == "Helloworld!");
+}
+
+TEST_CASE(interpolate_string_dollar_sign)
+{
+    std::string input = "$hello";
+    auto s            = migraphx::interpolate_string(input, {{"hello", "bye"}});
+    EXPECT(s == "$hello");
+}
+
+TEST_CASE(interpolate_string_missing)
+{
+    std::string input = "${hello}";
+    EXPECT(test::throws([&] { migraphx::interpolate_string(input, {{"h", "bye"}}); }));
+}
+
+TEST_CASE(interpolate_string_custom1)
+{
+    std::string input = "****{{a}}****";
+    auto s            = migraphx::interpolate_string(input, {{"a", "b"}}, "{{", "}}");
+    EXPECT(s == "****b****");
+}
+
+TEST_CASE(interpolate_string_custom2)
+{
+    std::string input = "****{{{a}}}****";
+    auto s            = migraphx::interpolate_string(input, {{"a", "b"}}, "{{{", "}}}");
+    EXPECT(s == "****b****");
+}
+
+TEST_CASE(interpolate_string_custom3)
+{
+    std::string input = "****{{{{a}}}}****";
+    auto s            = migraphx::interpolate_string(input, {{"a", "b"}}, "{{{{", "}}}}");
+    EXPECT(s == "****b****");
+}
+
+int main(int argc, const char* argv[]) { test::run(argc, argv); }

test/verify/main.cpp

@@ -45,14 +45,6 @@ int main(int argc, const char* argv[])
     run_verify rv;
     rv.add_validation_for("gpu", &validate_gpu);
     rv.disable_test_for("cpu", {"test_if_lp", "test_if_param", "test_if_literal"});
-    rv.disable_test_for("gpu",
-                        {"batch_quant_dot_2",
-                         "batch_quant_dot_3",
-                         "batch_quant_dot_5",
-                         "quant_dot_3args_1",
-                         "quant_dot_3args_2",
-                         "quant_dot_3args_3",
-                         "quant_dot_3args_4",
-                         "quant_dot_3args_5"});
+    rv.disable_test_for("gpu", {"test_conv_bn_add"});
     rv.run(argc, argv);
 }

test/verify/run_verify.cpp

@@ -6,6 +6,7 @@
 #include <migraphx/ref/target.hpp>
 #include <migraphx/ranges.hpp>
 #include <migraphx/generate.hpp>
+#include <migraphx/load_save.hpp>
 #include <migraphx/verify_args.hpp>
 #include <set>
 
@@ -15,6 +16,7 @@
 
 MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_TEST_COMPILE)
 MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_TEST)
+MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DUMP_TEST)
 
 // An improved async, that doesn't block
 template <class Function>
@@ -125,6 +127,8 @@ void run_verify::verify(const std::string& name, const migraphx::program& p) con
     using result_future =
         std::future<std::pair<migraphx::program, std::vector<migraphx::argument>>>;
     auto_print::set_terminate_handler(name);
+    if(migraphx::enabled(MIGRAPHX_DUMP_TEST{}))
+        migraphx::save(p, name + ".mx");
     std::vector<std::pair<std::string, result_future>> results;
     std::vector<std::string> target_names;
     for(const auto& tname : migraphx::get_targets())

test/verify/test_arg_ops.cpp

@@ -2,34 +2,92 @@
 #include "verify_program.hpp"
 #include <migraphx/program.hpp>
 #include <migraphx/generate.hpp>
+#include <migraphx/make_op.hpp>
 #include <migraphx/op/argmax.hpp>
 #include <migraphx/op/argmin.hpp>
 
-template <class T, int Axis>
-struct test_arg_ops : verify_program<test_arg_ops<T, Axis>>
+template <class T, int Axis, int NonStdShape>
+struct test_arg_ops : verify_program<test_arg_ops<T, Axis, NonStdShape>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {2, 3, 4, 1025}};
+        migraphx::shape s{migraphx::shape::float_type, {2, 1, 4, 1025}};
         auto param = mm->add_parameter("data", s);
+        switch(NonStdShape)
+        {
+        case 0:
+            param = mm->add_instruction(
+                migraphx::make_op("transpose", {{"permutation", {0, 2, 3, 1}}}), param);
+            break;
+        case 1:
+            param = mm->add_instruction(
+                migraphx::make_op("multibroadcast", {{"out_lens", {2, 3, 4, 1025}}}), param);
+            break;
+        case 2:
+            param = mm->add_instruction(
+                migraphx::make_op("slice", {{"axes", {2}}, {"starts", {1}}, {"ends", {3}}}), param);
+            break;
+        default: break;
+        }
         mm->add_instruction(T{Axis}, param);
-
         return p;
     }
 };
-
-template struct test_arg_ops<migraphx::op::argmax, 0>;
-template struct test_arg_ops<migraphx::op::argmax, 1>;
-template struct test_arg_ops<migraphx::op::argmax, 2>;
-template struct test_arg_ops<migraphx::op::argmax, 3>;
-template struct test_arg_ops<migraphx::op::argmax, -1>;
-template struct test_arg_ops<migraphx::op::argmax, -2>;
-
-template struct test_arg_ops<migraphx::op::argmin, 0>;
-template struct test_arg_ops<migraphx::op::argmin, 1>;
-template struct test_arg_ops<migraphx::op::argmin, 2>;
-template struct test_arg_ops<migraphx::op::argmin, 3>;
-template struct test_arg_ops<migraphx::op::argmin, -3>;
-template struct test_arg_ops<migraphx::op::argmin, -4>;
+// transpose argmax tests
+template struct test_arg_ops<migraphx::op::argmax, 0, 0>;
+template struct test_arg_ops<migraphx::op::argmax, 1, 0>;
+template struct test_arg_ops<migraphx::op::argmax, 2, 0>;
+template struct test_arg_ops<migraphx::op::argmax, 3, 0>;
+template struct test_arg_ops<migraphx::op::argmax, -1, 0>;
+template struct test_arg_ops<migraphx::op::argmax, -2, 0>;
+// transpose argmin tests
+template struct test_arg_ops<migraphx::op::argmin, 0, 0>;
+template struct test_arg_ops<migraphx::op::argmin, 1, 0>;
+template struct test_arg_ops<migraphx::op::argmin, 2, 0>;
+template struct test_arg_ops<migraphx::op::argmin, 3, 0>;
+template struct test_arg_ops<migraphx::op::argmin, -3, 0>;
+template struct test_arg_ops<migraphx::op::argmin, -4, 0>;
+// broadcast argmax tests
+template struct test_arg_ops<migraphx::op::argmax, 0, 1>;
+template struct test_arg_ops<migraphx::op::argmax, 1, 1>;
+template struct test_arg_ops<migraphx::op::argmax, 2, 1>;
+template struct test_arg_ops<migraphx::op::argmax, 3, 1>;
+template struct test_arg_ops<migraphx::op::argmax, -1, 1>;
+template struct test_arg_ops<migraphx::op::argmax, -2, 1>;
+// broadcast argmin tests
+template struct test_arg_ops<migraphx::op::argmin, 0, 1>;
+template struct test_arg_ops<migraphx::op::argmin, 1, 1>;
+template struct test_arg_ops<migraphx::op::argmin, 2, 1>;
+template struct test_arg_ops<migraphx::op::argmin, 3, 1>;
+template struct test_arg_ops<migraphx::op::argmin, -3, 1>;
+template struct test_arg_ops<migraphx::op::argmin, -4, 1>;
+// slice argmax tests
+template struct test_arg_ops<migraphx::op::argmax, 0, 2>;
+template struct test_arg_ops<migraphx::op::argmax, 1, 2>;
+template struct test_arg_ops<migraphx::op::argmax, 2, 2>;
+template struct test_arg_ops<migraphx::op::argmax, 3, 2>;
+template struct test_arg_ops<migraphx::op::argmax, -1, 2>;
+template struct test_arg_ops<migraphx::op::argmax, -2, 2>;
+// slice argmin tests
+template struct test_arg_ops<migraphx::op::argmin, 0, 2>;
+template struct test_arg_ops<migraphx::op::argmin, 1, 2>;
+template struct test_arg_ops<migraphx::op::argmin, 2, 2>;
+template struct test_arg_ops<migraphx::op::argmin, 3, 2>;
+template struct test_arg_ops<migraphx::op::argmin, -3, 2>;
+template struct test_arg_ops<migraphx::op::argmin, -4, 2>;
+// default case, standard shape argmax tests
+template struct test_arg_ops<migraphx::op::argmax, 0, 3>;
+template struct test_arg_ops<migraphx::op::argmax, 1, 3>;
+template struct test_arg_ops<migraphx::op::argmax, 2, 3>;
+template struct test_arg_ops<migraphx::op::argmax, 3, 3>;
+template struct test_arg_ops<migraphx::op::argmax, -1, 3>;
+template struct test_arg_ops<migraphx::op::argmax, -2, 3>;
+// default case, standard shape argmin tests
+template struct test_arg_ops<migraphx::op::argmin, 0, 3>;
+template struct test_arg_ops<migraphx::op::argmin, 1, 3>;
+template struct test_arg_ops<migraphx::op::argmin, 2, 3>;
+template struct test_arg_ops<migraphx::op::argmin, 3, 3>;
+template struct test_arg_ops<migraphx::op::argmin, -3, 3>;
+template struct test_arg_ops<migraphx::op::argmin, -4, 3>;

test/verify/test_conv_bias_clipped_relu.cpp

@@ -28,9 +28,9 @@ struct test_conv_bias_clipped_relu : verify_program<test_conv_bias_clipped_relu>
         auto min_val  = mm->add_literal(0.0f);
         auto max_val  = mm->add_literal(6.0f);
         min_val       = mm->add_instruction(
-            migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), min_val);
+            migraphx::make_op("multibroadcast", {{"out_lens", conv->get_shape().lens()}}), min_val);
         max_val = mm->add_instruction(
-            migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), max_val);
+            migraphx::make_op("multibroadcast", {{"out_lens", conv->get_shape().lens()}}), max_val);
         mm->add_instruction(migraphx::make_op("clip"), bias_add, min_val, max_val);
         return p;
     }

test/verify/test_conv_bn_add.cpp

@@ -6,26 +6,26 @@
 
 struct test_conv_bn_add : verify_program<test_conv_bn_add>
 {
-    static migraphx::instruction_ref add_bn(migraphx::module_ref mm,
+    static migraphx::instruction_ref add_bn(migraphx::module& m,
                                             migraphx::instruction_ref x,
                                             std::size_t channels,
                                             std::size_t seed = 1)
     {
         migraphx::shape vars{migraphx::shape::float_type, {channels}};
-        auto scale    = mm->add_literal(migraphx::abs(migraphx::generate_literal(vars, 1 + seed)));
-        auto bias     = mm->add_literal(migraphx::abs(migraphx::generate_literal(vars, 2 + seed)));
-        auto mean     = mm->add_literal(migraphx::abs(migraphx::generate_literal(vars, 3 + seed)));
-        auto variance = mm->add_literal(migraphx::abs(migraphx::generate_literal(vars, 4 + seed)));
-        return mm->add_instruction(
+        auto scale    = m.add_literal(migraphx::abs(migraphx::generate_literal(vars, 1 + seed)));
+        auto bias     = m.add_literal(migraphx::abs(migraphx::generate_literal(vars, 2 + seed)));
+        auto mean     = m.add_literal(migraphx::abs(migraphx::generate_literal(vars, 3 + seed)));
+        auto variance = m.add_literal(migraphx::abs(migraphx::generate_literal(vars, 4 + seed)));
+        return m.add_instruction(
             migraphx::make_op("batch_norm_inference"), x, scale, bias, mean, variance);
     }
 
     migraphx::program create_program() const
     {
         migraphx::program p;
+        auto* mm              = p.get_main_module();
         std::size_t ichannels = 64;
         std::size_t ochannels = 256;
-        auto* mm              = p.get_main_module();
         auto x     = mm->add_parameter("x", {migraphx::shape::float_type, {1, ichannels, 56, 56}});
         auto w     = mm->add_literal(migraphx::generate_literal(
             {migraphx::shape::float_type, {ochannels, ichannels, 1, 1}}, 1));
@@ -34,14 +34,12 @@ struct test_conv_bn_add : verify_program<test_conv_bn_add>
             {migraphx::shape::float_type, {ochannels, ichannels, 1, 1}}, 2));
         auto relu1 = mm->add_instruction(migraphx::make_op("relu"), x);
         auto conv1 = mm->add_instruction(migraphx::make_op("convolution"), relu1, w);
-        auto bn1   = add_bn(mm, conv1, ochannels, 1);
+        auto bn1   = add_bn(*mm, conv1, ochannels, 1);
         auto relu2 = mm->add_instruction(migraphx::make_op("relu"), y);
         auto conv2 = mm->add_instruction(migraphx::make_op("convolution"), relu2, v);
-        auto bn2   = add_bn(mm, conv2, ochannels, 1);
+        auto bn2   = add_bn(*mm, conv2, ochannels, 1);
         auto sum   = mm->add_instruction(migraphx::make_op("add"), bn1, bn2);
-        auto r     = mm->add_instruction(migraphx::make_op("relu"), sum);
-        mm->add_return({r});
-
+        mm->add_instruction(migraphx::make_op("relu"), sum);
         return p;
     }
 };

test/verify/test_hsqrt.cpp

+
+#include "verify_program.hpp"
+#include <migraphx/program.hpp>
+#include <migraphx/generate.hpp>
+#include <migraphx/make_op.hpp>
+
+struct test_hsqrt : verify_program<test_hsqrt>
+{
+    migraphx::program create_program() const
+    {
+        migraphx::program p;
+        auto* mm = p.get_main_module();
+        migraphx::shape s{migraphx::shape::half_type, {2, 3, 4, 6}};
+        auto param     = mm->add_parameter("x", s);
+        auto param_abs = mm->add_instruction(migraphx::make_op("abs"), param);
+        mm->add_instruction(migraphx::make_op("sqrt"), param_abs);
+        return p;
+    }
+};

test/verify/test_nms.cpp

+
+#include "verify_program.hpp"
+#include <migraphx/program.hpp>
+#include <migraphx/generate.hpp>
+#include <migraphx/make_op.hpp>
+
+struct test_nms : verify_program<test_nms>
+{
+    migraphx::program create_program() const
+    {
+        migraphx::program p;
+        auto* mm = p.get_main_module();
+
+        migraphx::shape boxes_s{migraphx::shape::float_type, {1, 6, 4}};
+
+        migraphx::shape scores_s{migraphx::shape::float_type, {1, 1, 6}};
+        std::vector<float> scores_vec = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3};
+
+        auto boxes_l         = mm->add_parameter("boxes", boxes_s);
+        auto scores_l        = mm->add_literal(migraphx::literal(scores_s, scores_vec));
+        auto max_out_l       = mm->add_literal(int64_t{4});
+        auto iou_threshold   = mm->add_literal(0.5f);
+        auto score_threshold = mm->add_literal(0.0f);
+
+        auto r =
+            mm->add_instruction(migraphx::make_op("nonmaxsuppression", {{"center_point_box", 1}}),
+                                boxes_l,
+                                scores_l,
+                                max_out_l,
+                                iou_threshold,
+                                score_threshold);
+        mm->add_return({r});
+
+        return p;
+    }
+};

test/verify/test_nonstd_gather.cpp

+
+#include "verify_program.hpp"
+#include <migraphx/program.hpp>
+#include <migraphx/generate.hpp>
+#include <migraphx/make_op.hpp>
+
+struct test_nonstd_gather : verify_program<test_nonstd_gather>
+{
+    migraphx::program create_program() const
+    {
+        migraphx::program p;
+        auto* mm = p.get_main_module();
+        migraphx::shape s{migraphx::shape::float_type, {3, 3}};
+        migraphx::shape s_indices{migraphx::shape::int32_type, {2, 2}};
+        std::vector<int> indices{1, 1, 0, 2};
+        auto d  = mm->add_parameter("data", s);
+        auto td = mm->add_instruction(migraphx::make_op("transpose", {{"permutation", {1, 0}}}), d);
+        auto ind = mm->add_literal(migraphx::literal{s_indices, indices});
+        auto tind =
+            mm->add_instruction(migraphx::make_op("transpose", {{"permutation", {1, 0}}}), ind);
+        auto r = mm->add_instruction(migraphx::make_op("gather", {{"axis", 1}}), td, tind);
+        mm->add_return({r});
+
+        return p;
+    }
+};

test/verify/test_roialign.cpp

+
+#include "verify_program.hpp"
+#include <migraphx/program.hpp>
+#include <migraphx/generate.hpp>
+#include <migraphx/make_op.hpp>
+
+struct test_roialign : verify_program<test_roialign>
+{
+    migraphx::program create_program() const
+    {
+        migraphx::program p;
+        auto* mm = p.get_main_module();
+        migraphx::shape x_s{migraphx::shape::float_type, {5, 4, 10, 10}};
+
+        migraphx::shape roi_s{migraphx::shape::float_type, {5, 4}};
+
+        migraphx::shape ind_s{migraphx::shape::int64_type, {5}};
+        std::vector<int64_t> ind_vec = {0, 2, 3, 4, 1};
+
+        auto x   = mm->add_parameter("x", x_s);
+        auto roi = mm->add_parameter("roi", roi_s);
+        auto ind = mm->add_literal(migraphx::literal(ind_s, ind_vec));
+        auto r   = mm->add_instruction(migraphx::make_op("roialign",
+                                                       {{"spatial_scale", 1.0},
+                                                        {"output_height", 5},
+                                                        {"output_width", 5},
+                                                        {"sampling_ratio", 2}}),
+                                     x,
+                                     roi,
+                                     ind);
+        mm->add_return({r});
+
+        return p;
+    }
+};

test/verify/test_roialign_nondefault.cpp

+
+#include "verify_program.hpp"
+#include <migraphx/program.hpp>
+#include <migraphx/generate.hpp>
+#include <migraphx/make_op.hpp>
+
+struct test_roialign_nondefault : verify_program<test_roialign_nondefault>
+{
+    migraphx::program create_program() const
+    {
+        migraphx::program p;
+        auto* mm = p.get_main_module();
+        migraphx::shape x_s{migraphx::shape::float_type, {5, 4, 10, 10}};
+
+        migraphx::shape roi_s{migraphx::shape::float_type, {5, 4}};
+
+        migraphx::shape ind_s{migraphx::shape::int64_type, {5}};
+        std::vector<int64_t> ind_vec = {0, 2, 3, 4, 1};
+
+        auto x   = mm->add_parameter("x", x_s);
+        auto roi = mm->add_parameter("roi", roi_s);
+        auto ind = mm->add_literal(migraphx::literal(ind_s, ind_vec));
+        auto r   = mm->add_instruction(
+            migraphx::make_op("roialign",
+                              {{"coordinate_transformation_mode", "output_half_pixel"},
+                               {"mode", "max"},
+                               {"spatial_scale", 1.0},
+                               {"output_height", 5},
+                               {"output_width", 5},
+                               {"sampling_ratio", 2}}),
+            x,
+            roi,
+            ind);
+        mm->add_return({r});
+
+        return p;
+    }
+};

tools/api.py

 import string, sys, re, runpy
 from functools import wraps
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
-type_map = {}
-cpp_type_map = {}
-functions = []
-cpp_classes = []
+type_map: Dict[str, Callable[['Parameter'], None]] = {}
+cpp_type_map: Dict[str, str] = {}
+functions: List['Function'] = []
+cpp_classes: List['CPPClass'] = []
 error_type = ''
 success_type = ''
 try_wrap = ''
 
-c_header_preamble = []
-c_api_body_preamble = []
-cpp_header_preamble = []
+c_header_preamble: List[str] = []
+c_api_body_preamble: List[str] = []
+cpp_header_preamble: List[str] = []
 
 
 def bad_param_error(msg):
@@ -23,28 +24,31 @@ class Template(string.Template):
 
 
 class Type:
-    def __init__(self, name):
+    def __init__(self, name: str) -> None:
         self.name = name.strip()
 
-    def is_pointer(self):
+    def is_pointer(self) -> bool:
         return self.name.endswith('*')
 
-    def is_reference(self):
+    def is_reference(self) -> bool:
         return self.name.endswith('&')
 
-    def is_const(self):
+    def is_const(self) -> bool:
         return self.name.startswith('const ')
 
-    def add_pointer(self):
+    def is_variadic(self):
+        return self.name.startswith('...')
+
+    def add_pointer(self) -> 'Type':
         return Type(self.name + '*')
 
     def add_reference(self):
         return Type(self.name + '&')
 
-    def add_const(self):
+    def add_const(self) -> 'Type':
         return Type('const ' + self.name)
 
-    def inner_type(self):
+    def inner_type(self) -> Optional['Type']:
         i = self.name.find('<')
         j = self.name.rfind('>')
         if i > 0 and j > 0:
@@ -52,7 +56,7 @@ class Type:
         else:
             return None
 
-    def remove_generic(self):
+    def remove_generic(self) -> 'Type':
         i = self.name.find('<')
         j = self.name.rfind('>')
         if i > 0 and j > 0:
@@ -60,25 +64,25 @@ class Type:
         else:
             return self
 
-    def remove_pointer(self):
+    def remove_pointer(self) -> 'Type':
         if self.is_pointer():
             return Type(self.name[0:-1])
         return self
 
-    def remove_reference(self):
+    def remove_reference(self) -> 'Type':
         if self.is_reference():
             return Type(self.name[0:-1])
         return self
 
-    def remove_const(self):
+    def remove_const(self) -> 'Type':
         if self.is_const():
             return Type(self.name[6:])
         return self
 
-    def basic(self):
+    def basic(self) -> 'Type':
         return self.remove_pointer().remove_const().remove_reference()
 
-    def decay(self):
+    def decay(self) -> 'Type':
         t = self.remove_reference()
         if t.is_pointer():
             return t
@@ -90,7 +94,7 @@ class Type:
             return self.add_const()
         return self
 
-    def str(self):
+    def str(self) -> str:
         return self.name
 
 
@@ -101,51 +105,69 @@ ${error_type} ${name}(${params});
 c_api_impl = Template('''
 extern "C" ${error_type} ${name}(${params})
 {
-    return ${try_wrap}([&] {
+    ${va_start}auto api_error_result = ${try_wrap}([&] {
         ${body};
     });
+    ${va_end}return api_error_result;
 }
 ''')
 
 
 class CFunction:
-    def __init__(self, name):
+    def __init__(self, name: str) -> None:
         self.name = name
-        self.params = []
-        self.body = []
+        self.params: List[str] = []
+        self.body: List[str] = []
+        self.va_start: List[str] = []
+        self.va_end: List[str] = []
 
-    def add_param(self, type, pname):
+    def add_param(self, type: str, pname: str) -> None:
         self.params.append('{} {}'.format(type, pname))
 
-    def add_statement(self, stmt):
+    def add_statement(self, stmt: str) -> None:
         self.body.append(stmt)
 
-    def substitute(self, form):
+    def add_vlist(self, name: str) -> None:
+        last_param = self.params[-1].split()[-1]
+        self.va_start = [
+            'va_list {};'.format(name),
+            'va_start({}, {});'.format(name, last_param)
+        ]
+        self.va_end = ['va_end({});'.format(name)]
+        self.add_param('...', '')
+
+    def substitute(self, form: Template) -> str:
         return form.substitute(error_type=error_type,
                                try_wrap=try_wrap,
                                name=self.name,
                                params=', '.join(self.params),
-                               body=";\n        ".join(self.body))
+                               body=";\n        ".join(self.body),
+                               va_start="\n    ".join(self.va_start),
+                               va_end="\n    ".join(self.va_end))
 
-    def generate_header(self):
+    def generate_header(self) -> str:
         return self.substitute(header_function)
 
-    def generate_body(self):
+    def generate_body(self) -> str:
         return self.substitute(c_api_impl)
 
 
 class BadParam:
-    def __init__(self, cond, msg):
+    def __init__(self, cond: str, msg: str) -> None:
         self.cond = cond
         self.msg = msg
 
 
 class Parameter:
-    def __init__(self, name, type, optional=False, returns=False):
+    def __init__(self,
+                 name: str,
+                 type: str,
+                 optional: bool = False,
+                 returns: bool = False) -> None:
         self.name = name
         self.type = Type(type)
         self.optional = optional
-        self.cparams = []
+        self.cparams: List[Tuple[str, str]] = []
         self.size_cparam = -1
         self.size_name = ''
         self.read = '${name}'
@@ -153,15 +175,15 @@ class Parameter:
         self.cpp_read = '${name}'
         self.cpp_write = '${name}'
         self.returns = returns
-        self.bad_param_check = None
+        self.bad_param_check: Optional[BadParam] = None
 
-    def get_name(self, prefix=None):
+    def get_name(self, prefix: Optional[str] = None) -> str:
         if prefix:
             return prefix + self.name
         else:
             return self.name
 
-    def get_cpp_type(self):
+    def get_cpp_type(self) -> str:
         if self.type.str() in cpp_type_map:
             return cpp_type_map[self.type.basic().str()]
         elif self.type.basic().str() in cpp_type_map:
@@ -171,7 +193,10 @@ class Parameter:
         else:
             return self.type.str()
 
-    def substitute(self, s, prefix=None, result=None):
+    def substitute(self,
+                   s: str,
+                   prefix: Optional[str] = None,
+                   result: Optional[str] = None) -> str:
         ctype = None
         if len(self.cparams) > 0:
             ctype = Type(self.cparams[0][0]).basic().str()
@@ -182,12 +207,13 @@ class Parameter:
                                            size=self.size_name,
                                            result=result or '')
 
-    def add_param(self, t, name=None):
+    def add_param(self, t: Union[str, Type],
+                  name: Optional[str] = None) -> None:
         if not isinstance(t, str):
             t = t.str()
         self.cparams.append((t, name or self.name))
 
-    def add_size_param(self, name=None):
+    def add_size_param(self, name: Optional[str] = None) -> None:
         self.size_cparam = len(self.cparams)
         self.size_name = name or self.name + '_size'
         if self.returns:
@@ -195,7 +221,7 @@ class Parameter:
         else:
             self.add_param('size_t', self.size_name)
 
-    def bad_param(self, cond, msg):
+    def bad_param(self, cond: str, msg: str) -> None:
         self.bad_param_check = BadParam(cond, msg)
 
     def remove_size_param(self, name):
@@ -206,7 +232,7 @@ class Parameter:
             self.size_name = name
         return p
 
-    def update(self):
+    def update(self) -> None:
         t = self.type.basic().str()
         g = self.type.remove_generic().basic().str()
         if t in type_map:
@@ -222,18 +248,18 @@ class Parameter:
             raise ValueError("Error for {}: write cannot be a string".format(
                 self.type.str()))
 
-    def cpp_param(self, prefix=None):
+    def cpp_param(self, prefix: Optional[str] = None) -> str:
         return self.substitute('${cpptype} ${name}', prefix=prefix)
 
-    def cpp_arg(self, prefix=None):
+    def cpp_arg(self, prefix: Optional[str] = None) -> str:
         return self.substitute(self.cpp_read, prefix=prefix)
 
-    def cpp_output_args(self, prefix=None):
+    def cpp_output_args(self, prefix: Optional[str] = None) -> List[str]:
         return [
             '&{prefix}{n}'.format(prefix=prefix, n=n) for t, n in self.cparams
         ]
 
-    def output_declarations(self, prefix=None):
+    def output_declarations(self, prefix: Optional[str] = None) -> List[str]:
         return [
             '{type} {prefix}{n};'.format(type=Type(t).remove_pointer().str(),
                                          prefix=prefix,
@@ -245,18 +271,21 @@ class Parameter:
             '&{prefix}{n};'.format(prefix=prefix, n=n) for t, n in self.cparams
         ]
 
-    def cpp_output(self, prefix=None):
+    def cpp_output(self, prefix: Optional[str] = None) -> str:
         return self.substitute(self.cpp_write, prefix=prefix)
 
-    def input(self, prefix=None):
+    def input(self, prefix: Optional[str] = None) -> str:
         return '(' + self.substitute(self.read, prefix=prefix) + ')'
 
-    def outputs(self, result=None):
+    def outputs(self, result: Optional[str] = None) -> List[str]:
         return [self.substitute(w, result=result) for w in self.write]
 
-    def add_to_cfunction(self, cfunction):
+    def add_to_cfunction(self, cfunction: CFunction) -> None:
         for t, name in self.cparams:
-            cfunction.add_param(self.substitute(t), self.substitute(name))
+            if t.startswith('...'):
+                cfunction.add_vlist(name)
+            else:
+                cfunction.add_param(self.substitute(t), self.substitute(name))
         if self.bad_param_check:
             msg = 'Bad parameter {name}: {msg}'.format(
                 name=self.name, msg=self.bad_param_check.msg)
@@ -265,35 +294,35 @@ class Parameter:
                 body=bad_param_error(msg)))
 
 
-def template_var(s):
+def template_var(s: str) -> str:
     return '${' + s + '}'
 
 
-def to_template_vars(params):
+def to_template_vars(params: List[Union[Any, Parameter]]) -> str:
     return ', '.join([template_var(p.name) for p in params])
 
 
 class Function:
     def __init__(self,
-                 name,
-                 params=None,
-                 shared_size=False,
-                 returns=None,
-                 invoke=None,
-                 fname=None,
-                 return_name=None,
-                 **kwargs):
+                 name: str,
+                 params: Optional[List[Parameter]] = None,
+                 shared_size: bool = False,
+                 returns: Optional[str] = None,
+                 invoke: Optional[str] = None,
+                 fname: Optional[str] = None,
+                 return_name: Optional[str] = None,
+                 **kwargs) -> None:
         self.name = name
         self.params = params or []
         self.shared_size = False
-        self.cfunction = None
+        self.cfunction: Optional[CFunction] = None
         self.fname = fname
         self.invoke = invoke or '${__fname__}($@)'
         self.return_name = return_name or 'out'
         self.returns = Parameter(self.return_name, returns,
                                  returns=True) if returns else None
 
-    def share_params(self):
+    def share_params(self) -> None:
         if self.shared_size == True:
             size_param_name = 'size'
             size_type = Type('size_t')
@@ -303,7 +332,7 @@ class Function:
                     size_type = Type(p[0])
             self.params.append(Parameter(size_param_name, size_type.str()))
 
-    def update(self):
+    def update(self) -> None:
         self.share_params()
         for param in self.params:
             param.update()
@@ -311,11 +340,12 @@ class Function:
             self.returns.update()
         self.create_cfunction()
 
-    def inputs(self):
+    def inputs(self) -> str:
         return ', '.join([p.input() for p in self.params])
 
-    def input_map(self):
-        m = {}
+    # TODO: Shoule we remove Optional?
+    def input_map(self) -> Dict[str, Optional[str]]:
+        m: Dict[str, Optional[str]] = {}
         for p in self.params:
             m[p.name] = p.input()
         m['return'] = self.return_name
@@ -323,14 +353,22 @@ class Function:
         m['__fname__'] = self.fname
         return m
 
-    def get_invoke(self):
+    def get_invoke(self) -> str:
         return Template(self.invoke).safe_substitute(self.input_map())
 
-    def write_to_tmp_var(self):
+    def write_to_tmp_var(self) -> bool:
+        if not self.returns:
+            return False
         return len(self.returns.write) > 1 or self.returns.write[0].count(
             '${result}') > 1
 
-    def create_cfunction(self):
+    def get_cfunction(self) -> CFunction:
+        if self.cfunction:
+            return self.cfunction
+        raise Exception(
+            "self.cfunction is None: self.update() needs to be called.")
+
+    def create_cfunction(self) -> None:
         self.cfunction = CFunction(self.name)
         # Add the return as a parameter
         if self.returns:
@@ -338,12 +376,12 @@ class Function:
         # Add the input parameters
         for param in self.params:
             param.add_to_cfunction(self.cfunction)
-        f = self.get_invoke()
+        f: Optional[str] = self.get_invoke()
         # Write the assignments
         assigns = []
         if self.returns:
             result = f
-            if self.write_to_tmp_var():
+            if self.write_to_tmp_var() and f:
                 f = 'auto&& api_result = ' + f
                 result = 'api_result'
             else:
@@ -396,31 +434,37 @@ cpp_class_constructor_template = Template('''
 
 
 class CPPMember:
-    def __init__(self, name, function, prefix, method=True):
+    def __init__(self,
+                 name: str,
+                 function: Function,
+                 prefix: str,
+                 method: bool = True) -> None:
         self.name = name
         self.function = function
         self.prefix = prefix
         self.method = method
 
-    def get_function_params(self):
+    def get_function_params(self) -> List[Union[Any, Parameter]]:
         if self.method:
             return self.function.params[1:]
         else:
             return self.function.params
 
-    def get_args(self):
+    def get_args(self) -> str:
         output_args = []
         if self.function.returns:
             output_args = self.function.returns.cpp_output_args(self.prefix)
+        if not self.function.cfunction:
+            raise Exception('self.function.update() must be called')
         return ', '.join(
             ['&{}'.format(self.function.cfunction.name)] + output_args +
             [p.cpp_arg(self.prefix) for p in self.get_function_params()])
 
-    def get_params(self):
+    def get_params(self) -> str:
         return ', '.join(
             [p.cpp_param(self.prefix) for p in self.get_function_params()])
 
-    def get_return_declarations(self):
+    def get_return_declarations(self) -> str:
         if self.function.returns:
             return '\n        '.join([
                 d
@@ -432,7 +476,9 @@ class CPPMember:
     def get_result(self):
         return self.function.returns.input(self.prefix)
 
-    def generate_method(self):
+    def generate_method(self) -> str:
+        if not self.function.cfunction:
+            raise Exception('self.function.update() must be called')
         if self.function.returns:
             return_type = self.function.returns.get_cpp_type()
             return cpp_class_method_template.safe_substitute(
@@ -452,7 +498,9 @@ class CPPMember:
                 args=self.get_args(),
                 success=success_type)
 
-    def generate_constructor(self, name):
+    def generate_constructor(self, name: str) -> str:
+        if not self.function.cfunction:
+            raise Exception('self.function.update() must be called')
         return cpp_class_constructor_template.safe_substitute(
             name=name,
             cfunction=self.function.cfunction.name,
@@ -462,98 +510,101 @@ class CPPMember:
 
 
 class CPPClass:
-    def __init__(self, name, ctype):
+    def __init__(self, name: str, ctype: str) -> None:
         self.name = name
         self.ctype = ctype
-        self.constructors = []
-        self.methods = []
+        self.constructors: List[CPPMember] = []
+        self.methods: List[CPPMember] = []
         self.prefix = 'p'
 
-    def add_method(self, name, f):
+    def add_method(self, name: str, f: Function) -> None:
         self.methods.append(CPPMember(name, f, self.prefix, method=True))
 
-    def add_constructor(self, name, f):
+    def add_constructor(self, name: str, f: Function) -> None:
         self.constructors.append(CPPMember(name, f, self.prefix, method=True))
 
-    def generate_methods(self):
+    def generate_methods(self) -> str:
         return '\n    '.join([m.generate_method() for m in self.methods])
 
-    def generate_constructors(self):
+    def generate_constructors(self) -> str:
         return '\n    '.join(
             [m.generate_constructor(self.name) for m in self.constructors])
 
-    def substitute(self, s, **kwargs):
-        t = s
-        if isinstance(s, str):
-            t = string.Template(s)
+    def substitute(self, s: Union[string.Template, str], **kwargs) -> str:
+        t = string.Template(s) if isinstance(s, str) else s
         destroy = self.ctype + '_destroy'
         return t.safe_substitute(name=self.name,
                                  ctype=self.ctype,
                                  destroy=destroy,
                                  **kwargs)
 
-    def generate(self):
+    def generate(self) -> str:
         return self.substitute(
             cpp_class_template,
             constructors=self.substitute(self.generate_constructors()),
             methods=self.substitute(self.generate_methods()))
 
 
-def params(virtual=None, **kwargs):
+def params(virtual: Optional[Dict[str, str]] = None,
+           **kwargs) -> List[Parameter]:
     result = []
-    for name in virtual or {}:
-        result.append(Parameter(name, virtual[name]))
+    v: Dict[str, str] = virtual or {}
+    for name in v:
+        result.append(Parameter(name, v[name]))
     for name in kwargs:
         result.append(Parameter(name, kwargs[name]))
     return result
 
 
-def add_function(name, *args, **kwargs):
+def add_function(name: str, *args, **kwargs) -> Function:
     f = Function(name, *args, **kwargs)
     functions.append(f)
     return f
 
 
-def once(f):
+def once(f: Callable) -> Any:
     @wraps(f)
     def decorated(*args, **kwargs):
         if not decorated.has_run:
             decorated.has_run = True
             return f(*args, **kwargs)
 
-    decorated.has_run = False
-    return decorated
+    d: Any = decorated
+    d.has_run = False
+    return d
 
 
 @once
-def process_functions():
+def process_functions() -> None:
     for f in functions:
         f.update()
 
 
-def generate_lines(p):
+def generate_lines(p: List[str]) -> str:
     return '\n'.join(p)
 
 
-def generate_c_header():
+def generate_c_header() -> str:
     process_functions()
-    return generate_lines(c_header_preamble +
-                          [f.cfunction.generate_header() for f in functions])
+    return generate_lines(
+        c_header_preamble +
+        [f.get_cfunction().generate_header() for f in functions])
 
 
-def generate_c_api_body():
+def generate_c_api_body() -> str:
     process_functions()
-    return generate_lines(c_api_body_preamble +
-                          [f.cfunction.generate_body() for f in functions])
+    return generate_lines(
+        c_api_body_preamble +
+        [f.get_cfunction().generate_body() for f in functions])
 
 
-def generate_cpp_header():
+def generate_cpp_header() -> str:
     process_functions()
     return generate_lines(cpp_header_preamble +
                           [c.generate() for c in cpp_classes])
 
 
-def cwrap(name):
+def cwrap(name: str) -> Callable:
     def with_cwrap(f):
         type_map[name] = f
 
@@ -657,13 +708,17 @@ protected:
 
 
 @once
-def add_handle_preamble():
+def add_handle_preamble() -> None:
     c_api_body_preamble.append(handle_preamble)
     cpp_header_preamble.append(
         string.Template(cpp_handle_preamble).substitute(success=success_type))
 
 
-def add_handle(name, ctype, cpptype, destroy=None, ref=None):
+def add_handle(name: str,
+               ctype: str,
+               cpptype: str,
+               destroy: Optional[str] = None,
+               ref: Optional[bool] = None) -> None:
     opaque_type = ctype + '_t'
 
     def handle_wrap(p):
@@ -698,8 +753,12 @@ def add_handle(name, ctype, cpptype, destroy=None, ref=None):
 
 
 @cwrap('std::vector')
-def vector_c_wrap(p):
-    t = p.type.inner_type().add_pointer()
+def vector_c_wrap(p: Parameter) -> None:
+    inner = p.type.inner_type()
+    # Not a generic type
+    if not inner:
+        return
+    t = inner.add_pointer()
     if p.returns:
         if p.type.is_reference():
             if p.type.is_const():
@@ -727,7 +786,7 @@ def vector_c_wrap(p):
 
 
 @cwrap('std::string')
-def string_c_wrap(p):
+def string_c_wrap(p: Parameter) -> None:
     t = Type('char*')
     if p.returns:
         if p.type.is_reference():
@@ -751,7 +810,11 @@ def string_c_wrap(p):
 
 
 class Handle:
-    def __init__(self, name, ctype, cpptype, ref=None):
+    def __init__(self,
+                 name: str,
+                 ctype: str,
+                 cpptype: str,
+                 ref: Optional[bool] = None) -> None:
         self.name = name
         self.ctype = ctype
         self.cpptype = cpptype
@@ -759,17 +822,21 @@ class Handle:
         add_handle(name, ctype, cpptype, ref=ref)
         cpp_type_map[cpptype] = name
 
-    def cname(self, name):
+    def cname(self, name: str) -> str:
         return self.ctype + '_' + name
 
-    def substitute(self, s, **kwargs):
+    def substitute(self, s: str, **kwargs) -> str:
         return Template(s).safe_substitute(name=self.name,
                                            ctype=self.ctype,
                                            cpptype=self.cpptype,
                                            **kwargs)
 
-    def constructor(self, name, params=None, fname=None, invoke=None,
-                    **kwargs):
+    def constructor(self,
+                    name: str,
+                    params: Optional[List[Parameter]] = None,
+                    fname: Optional[str] = None,
+                    invoke: Optional[str] = None,
+                    **kwargs) -> 'Handle':
         create = self.substitute('allocate<${cpptype}>($@)')
         if fname:
             create = self.substitute('allocate<${cpptype}>(${fname}($@))',
@@ -785,13 +852,13 @@ class Handle:
         return self
 
     def method(self,
-               name,
-               params=None,
-               fname=None,
-               invoke=None,
-               cpp_name=None,
-               const=None,
-               **kwargs):
+               name: str,
+               params: Optional[List[Parameter]] = None,
+               fname: Optional[str] = None,
+               invoke: Optional[str] = None,
+               cpp_name: Optional[str] = None,
+               const: Optional[bool] = None,
+               **kwargs) -> 'Handle':
         cpptype = self.cpptype
         if const:
             cpptype = Type(cpptype).add_const().str()
@@ -812,11 +879,14 @@ class Handle:
         add_function(self.cname(name), params=params, **kwargs)
         return self
 
-    def add_cpp_class(self):
+    def add_cpp_class(self) -> None:
         cpp_classes.append(self.cpp_class)
 
 
-def handle(ctype, cpptype, name=None, ref=None):
+def handle(ctype: str,
+           cpptype: str,
+           name: Optional[str] = None,
+           ref: Optional[bool] = None) -> Callable:
     def with_handle(f):
         n = name or f.__name__
         h = Handle(n, ctype, cpptype, ref=ref)
@@ -845,10 +915,10 @@ def template_eval(template, **kwargs):
     return template
 
 
-def run():
-    runpy.run_path(sys.argv[1])
-    if len(sys.argv) > 2:
-        f = open(sys.argv[2]).read()
+def run(args: List[str]) -> None:
+    runpy.run_path(args[0])
+    if len(args) > 1:
+        f = open(args[1]).read()
         r = template_eval(f)
         sys.stdout.write(r)
     else:
@@ -859,4 +929,4 @@ def run():
 
 if __name__ == "__main__":
     sys.modules['api'] = sys.modules['__main__']
-    run()
+    run(sys.argv[1:])

tools/api/api.cpp

@@ -13,6 +13,7 @@
 #include <migraphx/json.hpp>
 #include <migraphx/convert_to_json.hpp>
 #include <algorithm>
+#include <cstdarg>
 
 namespace migraphx {
 
@@ -155,18 +156,30 @@ void quantize_int8_wrap(program& prog, const target& t, quantize_int8_options& o
     migraphx::quantize_int8(prog, t, options.calibration, options.op_names);
 }
 
-operation create_op(const char* name, const char* attributes)
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wformat-nonliteral"
+#endif
+
+operation create_op(const char* name, const char* attributes, va_list vlist)
 {
+    std::string sattributes = attributes == nullptr ? "" : attributes;
+    std::vector<char> buffer(sattributes.size() * 2);
+    std::vsnprintf(buffer.data(), buffer.size(), sattributes.c_str(), vlist);
     value v = value::object{};
     if(attributes != nullptr)
     {
-        v = from_json_string(convert_to_json(std::string(attributes)));
+        v = from_json_string(convert_to_json(std::string(buffer.data())));
     }
     auto op = make_op(name, v);
 
     return op;
 }
 
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
 template <class T>
 bool equal(const T& x, const T& y)
 {

tools/generate.sh

 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 SRC_DIR=$DIR/../src
-ls -1 $DIR/include/ | xargs -n 1 -P $(nproc) -I{} -t bash -c "python3.6 $DIR/te.py $DIR/include/{} | clang-format-5.0 -style=file > $SRC_DIR/include/migraphx/{}"
+PYTHON=python3
+if type -p python3.6 > /dev/null ; then
+    PYTHON=python3.6
+fi
+if type -p python3.8 > /dev/null ; then
+    PYTHON=python3.8
+fi
+ls -1 $DIR/include/ | xargs -n 1 -P $(nproc) -I{} -t bash -c "$PYTHON $DIR/te.py $DIR/include/{} | clang-format-5.0 -style=file > $SRC_DIR/include/migraphx/{}"
 
 function api {
-    python3.6 $DIR/api.py $SRC_DIR/api/migraphx.py $1 | clang-format-5.0 -style=file > $2
+    $PYTHON $DIR/api.py $SRC_DIR/api/migraphx.py $1 | clang-format-5.0 -style=file > $2
 }
 
 api $DIR/api/migraphx.h $SRC_DIR/api/include/migraphx/migraphx.h

tools/include/marker.hpp

+#ifndef MIGRAPHX_GUARD_MARKER_HPP
+#define MIGRAPHX_GUARD_MARKER_HPP
+
+#include <cassert>
+#include <string>
+#include <functional>
+#include <memory>
+#include <type_traits>
+#include <utility>
+#include <migraphx/config.hpp>
+#include <migraphx/instruction_ref.hpp>
+#include <migraphx/program.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+#ifdef DOXYGEN
+
+/// Marker is an interface to general marking functions, such as rocTX markers.
+
+#else
+
+<%
+interface('marker',
+           virtual('mark_start', ins_ref = 'instruction_ref', returns = 'void'),
+           virtual('mark_start', prog = 'const program&', returns = 'void'),
+           virtual('mark_stop', ins = 'instruction_ref', returns = 'void'),
+           virtual('mark_stop', prog = 'const program&', returns = 'void')
+        ) %>
+#endif
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

tools/include/operation.hpp

@@ -103,79 +103,69 @@ auto operator==(const T& x, const U& y) -> decltype(x.name() == y.name())
 } // namespace operation_operators
 
 template <class T>
-auto normalize_compute_shape_op(rank<1>, const T& x, const std::vector<shape>& inputs)
-    -> decltype(x.normalize_compute_shape(inputs))
-{
-    dependent_type<operation, T> y = x;
-    normalize_attributes(y, inputs[0].lens());
-    return any_cast<T>(y).normalize_compute_shape(inputs);
-}
-
-template <class T>
-shape normalize_compute_shape_op(rank<0>, const T& x, const std::vector<shape>&)
+auto compute_shape_op(rank<3>, const T& x, const std::vector<shape>& inputs)
+    -> decltype(x.compute_shape(inputs))
 {
-    std::string name = x.name();
-    MIGRAPHX_THROW("Shape not computable: " + name);
+    return x.compute_shape(inputs);
 }
 
 template <class T>
-shape normalize_compute_shape_op(const T& x, const std::vector<shape>& inputs)
+auto compute_shape_op(rank<2>, const T& x, const std::vector<shape>& inputs)
+    -> decltype(x.normalize_compute_shape(inputs))
 {
-    return normalize_compute_shape_op(rank<1>{}, x, inputs);
+    dependent_type<operation, T> y = x;
+    normalize_attributes(y, inputs[0].lens());
+    return any_cast<T>(y).normalize_compute_shape(inputs);
 }
 
 template <class T>
-auto compute_shape_op(rank<1>,
-                      const T& x,
-                      const std::vector<shape>& inputs,
-                      const std::vector<module_ref>& mod_args)
-    -> decltype(x.compute_shape(inputs, mod_args))
+auto compute_shape_op(rank<1>, const T& x, const std::vector<shape>& inputs)
+    -> decltype(x.compute_shape(inputs, {}))
 {
-    return x.compute_shape(inputs, mod_args);
+    return x.compute_shape(inputs, {});
 }
 
 template <class T>
-shape
-    compute_shape_op(rank<0>, const T& x, const std::vector<shape>&, const std::vector<module_ref>&)
+shape compute_shape_op(rank<0>, const T& x, const std::vector<shape>&)
 {
     std::string name = x.name();
     MIGRAPHX_THROW("Shape not computable: " + name);
 }
 
 template <class T>
-shape compute_shape_op(const T& x,
-                       const std::vector<shape>& inputs,
-                       const std::vector<module_ref>& mod_args)
+shape compute_shape_op(const T& x, const std::vector<shape>& inputs)
 {
-    return compute_shape_op(rank<1>{}, x, inputs, mod_args);
+    return compute_shape_op(rank<3>{}, x, inputs);
 }
 
 template <class T>
-auto normalize_compute_shape_op(rank<1>,
-                                const T& x,
-                                const std::vector<shape>& inputs,
-                                std::vector<module_ref>& mod_args)
-    -> decltype(x.normalize_compute_shape(inputs, mod_args))
+auto mod_compute_shape_op(rank<1>,
+                          const T& x,
+                          const std::vector<shape>& inputs,
+                          const std::vector<module_ref>& mod_args)
+    -> decltype(x.compute_shape(inputs, mod_args))
 {
-    return x.normalize_compute_shape(inputs, mod_args);
+    return x.compute_shape(inputs, mod_args);
 }
 
 template <class T>
-shape normalize_compute_shape_op(rank<0>,
-                                 const T& x,
-                                 const std::vector<shape>&,
-                                 const std::vector<module_ref>&)
+shape mod_compute_shape_op(rank<0>,
+                           const T& x,
+                           const std::vector<shape>& inputs,
+                           const std::vector<module_ref>& mod_args)
 {
+    if(mod_args.empty())
+        return compute_shape_op(x, inputs);
     std::string name = x.name();
     MIGRAPHX_THROW("Shape not computable: " + name);
 }
 
 template <class T>
-shape normalize_compute_shape_op(const T& x,
-                                 const std::vector<shape>& inputs,
-                                 std::vector<module_ref>& mod_args)
+shape mod_compute_shape_op(const T& x,
+                           const std::vector<shape>& inputs,
+                           const std::vector<module_ref>& mod_args)
 {
-    return normalize_compute_shape_op(rank<1>{}, x, inputs, mod_args);
+    return mod_compute_shape_op(rank<1>{}, x, inputs, mod_args);
 }
 
 template <class T>
@@ -488,13 +478,13 @@ lifetime get_lifetime_op(const T&)
              returns = 'shape',
              input   = 'const std::vector<shape>&',
              const   = True,
-             default = 'detail::normalize_compute_shape_op'),
+             default = 'detail::compute_shape_op'),
      virtual('compute_shape',
              returns  = 'shape',
              inputs   = 'const std::vector<shape>&',
              mod_args = 'const std::vector<module_ref>&',
              const    = True,
-             default  = 'detail::compute_shape_op'),
+             default  = 'detail::mod_compute_shape_op'),
      virtual('compute',
              returns = 'argument',
              ctx     = 'context&',
@@ -582,7 +572,7 @@ template <class T>
 inline auto compute_shape(const T& op, const std::vector<shape>& inputs)
     -> decltype(op.normalize_compute_shape(inputs))
 {
-    return detail::normalize_compute_shape_op(op, inputs);
+    return detail::compute_shape_op(op, inputs);
 }
 
 inline shape compute_shape(const operation& op,
@@ -607,7 +597,7 @@ inline auto compute_shape(const T& op,
                           const std::vector<module_ref>& mod_args)
     -> decltype(op.normalize_compute_shape(inputs, mod_args))
 {
-    return detail::normalize_compute_shape_op(op, inputs, mod_args);
+    return detail::compute_shape_op(op, inputs, mod_args);
 }
 
 inline bool is_context_free(const operation& op) { return op.is_context_free(); }

tools/roctx.py

+#!/usr/bin/env python3
+
+import json
+import argparse
+import os
+from sys import argv as sysargs
+from sys import version_info as python_version
+from sys import exit as sys_exit
+import pandas as pd
+from datetime import datetime
+import venv
+import shutil
+
+if (python_version[0] < 3) or (python_version[0] < 3
+                               and python_version[1] < 6):
+    raise Exception("Please utilize Python version 3.6 and above. Exiting...")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Parser for MIGraphX ROCTX Markers")
+    parser.add_argument('--json-path',
+                        type=str,
+                        metavar='json-path',
+                        help='Path to json file')
+    parser.add_argument('--out',
+                        type=str,
+                        metavar='out',
+                        help='Output directory for run.')
+    parser.add_argument(
+        '--study-name',
+        type=str,
+        metavar='study-name',
+        help='Study-name is used for naming the output CSV file.')
+    parser.add_argument('--repeat',
+                        type=int,
+                        metavar='repeat',
+                        help='Defines number of runs.',
+                        default=2)
+    parser.add_argument('--parse',
+                        default=False,
+                        action='store_true',
+                        help='Parses given JSON file.')
+    parser.add_argument('--clean',
+                        default=False,
+                        action='store_true',
+                        help='Removes temporary paths')
+    parser.add_argument('--run',
+                        type=str,
+                        metavar='run',
+                        help='Enables run and fetches run configs.')
+    parser.add_argument('--debug', default=False, action='store_true')
+
+    args = parser.parse_args()
+    return args
+
+
+args = parse_args()
+if not len(sysargs) > 1:
+    raise Exception("No arg is passed. Exiting...")
+
+
+def parse(file):
+    with open(file, "r") as read_file:
+        data = json.load(read_file)
+
+    #Get marker names and first marker's time
+    list_names = []
+    first_marker = True
+    first_marker_time = 0
+    for i in data:
+        if (i):
+            if ("Marker start:" in i['name']) and (
+                    i['name'] not in list_names):
+                list_names.append(i['name'])
+                if first_marker:
+                    first_marker_time = i['ts']
+                    first_marker = False
+
+    if (args.debug):
+        print(f"FIRST MARKER TIME DETERMINED: {first_marker_time}")
+
+    if (first_marker_time == 0):
+        raise ("FIRST MARKER TIME IS ZERO. EXITING...")
+
+    kernel_launch_info = []  #kernel description
+    kernel_launch_list = []  #kernel launch details
+    kernel_launch_time = []  #kernel execution time
+    for i in data:
+        if (i and i.get('args')):
+            try:
+                if (("KernelExecution" in i['args']['desc'])
+                        and (i['ts'] >= first_marker_time)):
+                    kernel_launch_info.append(i['args']['desc'])
+                    kernel_launch_list.append(i)
+                    kernel_launch_time.append(int(i['dur']))
+            except:
+                continue
+
+    max_index = kernel_launch_time.index(max(kernel_launch_time))
+    max_kernel_info = kernel_launch_list[max_index]
+
+    if (args.debug):
+        with open('rocTX_kernel_launch_list.txt', 'w') as f:
+            for i in kernel_launch_list:
+                f.write(f'{i}')
+
+    # Get timing information for each marker name
+    list_times_per_names = []
+    for name in list_names:
+        temp_list = []
+        for entry in data:
+            if (entry) and (
+                    name == entry['name']
+            ):  # name can match on gpu or cpu side, for gpu, we need data from gpu markers.
+                if (("gpu::" in name)
+                        and ("UserMarker frame:" in entry['args']['desc'])
+                    ):  #gpu side information
+                    temp_list.append(int(entry.get('dur')))
+                elif (("gpu::" not in name)
+                      and ("Marker start:" in entry['args']['desc'])
+                      ):  #cpu side information
+                    temp_list.append(int(entry.get('dur')))
+        list_times_per_names.append(temp_list)
+
+    if (args.debug):
+        print(list_times_per_names)
+
+    sum_per_name = []  #TODO: refactor stat collection
+    for list in list_times_per_names:
+        sum_per_name.append(sum(list))
+
+    count_per_name = []
+    for list in list_times_per_names:
+        try:
+            count_per_name.append(len(list))
+        except:
+            count_per_name.append(0)
+
+    max_per_name = []
+    for list in list_times_per_names:
+        try:
+            max_per_name.append(max(list))
+        except:
+            max_per_name.append(0)
+
+    min_per_name = []
+    for list in list_times_per_names:
+        try:
+            min_per_name.append(min(list))
+        except:
+            min_per_name.append(0)
+
+    max_index_per_name = []
+    for list in list_times_per_names:
+        try:
+            max_index_per_name.append(list.index(max(list)))
+        except:
+            max_index_per_name.append(0)
+
+    max_occur_per_name = []
+    for list in list_times_per_names:
+        try:
+            max_occur_per_name.append(list.count(max(list)))
+        except:
+            max_occur_per_name.append(0)
+
+    total_time = sum(sum_per_name)
+
+    d = {
+        'SUM': sum_per_name,
+        'MIN': min_per_name,
+        'MAX': max_per_name,
+        'COUNT': count_per_name,
+        'MAX_INDEX': max_index_per_name,
+        'MAX_OCCUR': max_occur_per_name
+    }
+    df2 = pd.DataFrame(d)
+    df2.index = list_names
+    df2.sort_values(by=['SUM'], inplace=True, ascending=False)
+
+    if (args.debug):
+        print(df2)
+        print(f"\nTOTAL TIME: {total_time} us")
+    return df2, total_time, max_kernel_info
+
+
+def run():
+    repeat_count = args.repeat
+    if (repeat_count == 0 or repeat_count == float('inf') or not repeat_count):
+        raise Exception("REPEAT COUNT CANNOT BE ZERO/INFINITY/NULL")
+    run_args = args.run
+    #configurations
+    configs = '--hip-trace --roctx-trace --flush-rate 10ms --timestamp on'
+    output_dir = f"-d {args.out}"
+    executable = f"/opt/rocm/bin/migraphx-driver roctx {run_args}"
+    process_args = configs + ' ' + output_dir + ' ' + executable
+    for i in range(repeat_count):
+        os.system('rocprof ' + process_args)
+    print("RUN COMPLETE.")
+
+
+def clean():
+    shutil.rmtree('/tmp/rocm-profile-data/', ignore_errors=False)
+
+
+def main():
+
+    if (args.clean):
+        clean()
+        sys_exit()
+
+    print("Initiating virtual environment...")
+    builder = venv.EnvBuilder(clear=True, with_pip=True)
+    builder.create('/tmp/rocm-profile-data/py/')
+    python_bin = '/tmp/rocm-profile-data/py' + '/bin/python'
+    file = args.json_path
+
+    if (args.study_name):
+        filename = args.study_name + ".csv"
+    else:
+        filename = "output" + datetime.now().strftime(
+            "%Y_%m_%d-%I:%M:%S_%p") + ".csv"
+
+    with open(filename, 'a') as f:
+        f.write(f"{args.run}\n")
+
+    if (args.run):
+        curr = os.path.abspath(os.getcwd())
+        rpd_path = '/tmp/rocm-profile-data/rocmProfileData/'
+        if not os.path.exists(rpd_path):
+            print("rocmProfileData DOES NOT EXIST. CLONING...")
+            os.system(
+                f"git clone https://github.com/ROCmSoftwarePlatform/rocmProfileData.git {rpd_path}"
+            )
+        os.chdir(rpd_path + "rocpd_python/")
+        os.system(python_bin + ' -m pip install --upgrade pip')
+        os.system(python_bin + ' setup.py install')
+        os.chdir(curr)
+        run()
+        os.chdir(curr + f"/{args.out}/")
+        out_path = os.popen(f"ls -td $PWD/*/*/ | head -{args.repeat}").read()
+        print(f"\nFOLLOWING PATHS WILL BE PARSED:\n{out_path}")
+        out_path = out_path.splitlines()
+        df_tot = pd.DataFrame()
+        tot_time = []
+        max_kernel_info_list = []
+        for path in out_path:
+            path = path.strip('\n')
+            print("\nPARSING OUTPUT PATH: " + path)
+            os.chdir(path)
+            os.system(
+                f"{python_bin} -m rocpd.rocprofiler_import --ops_input_file hcc_ops_trace.txt --api_input_file hip_api_trace.txt --roctx_input_file roctx_trace.txt trace.rpd"
+            )
+            os.system(
+                f"{python_bin} {rpd_path}/rpd2tracing.py trace.rpd trace.json")
+            os.chdir(curr)
+            df, total_time, path_max_kernel_info = parse(path + "trace.json")
+            max_kernel_info_list.append(path_max_kernel_info)
+            tot_time.append(total_time)
+            df_tot = pd.merge(df_tot,
+                              df,
+                              how='outer',
+                              left_index=True,
+                              right_index=True)
+            if (args.debug):
+                print("JSON FILE PATH: " + path + "trace.json")
+
+        df_tot.to_csv("rocTX_runs_dataframe.csv")
+        if (args.debug):
+            print(df_tot)
+
+        tmp_sum = df_tot.loc[:, df_tot.columns.str.contains('SUM')].astype(int)
+        tmp_min = df_tot.loc[:, df_tot.columns.str.contains('MIN')].astype(int)
+        tmp_max = df_tot.loc[:, df_tot.columns.str.match("^MAX_.$")].astype(
+            int)
+        tmp_count = df_tot.loc[:, df_tot.columns.str.match("COUNT")].astype(
+            int)
+
+        tmp_sum['SUM_avg'] = tmp_sum.mean(axis=1).astype(int)
+        tmp_min['MIN_avg'] = tmp_min.mean(axis=1).astype(int)
+        tmp_max['MAX_avg'] = tmp_max.mean(axis=1).astype(int)
+
+        df2 = tmp_sum['SUM_avg'].copy()
+        df2 = pd.merge(df2,
+                       tmp_min['MIN_avg'],
+                       how='outer',
+                       left_index=True,
+                       right_index=True)
+        df2 = pd.merge(df2,
+                       tmp_max['MAX_avg'],
+                       how='outer',
+                       left_index=True,
+                       right_index=True)
+        df2 = pd.merge(df2,
+                       tmp_count['COUNT_x'],
+                       how='outer',
+                       left_index=True,
+                       right_index=True)
+        df2.rename(columns={'COUNT_x': 'COUNT'}, inplace=True)
+        df2 = df2.loc[:, ~df2.columns.duplicated(
+        )]  #there will be many COUNT_x in df2
+        df2.sort_values(by=['SUM_avg'], inplace=True, ascending=False)
+
+        if (args.debug):
+            pd.set_option('display.max_columns', None)
+            print(df_tot)  #all data from all runs
+
+        print("\n*** RESULTS ***")
+        print(df2)
+        out_time = sum(tot_time) / len(tot_time)
+        print(f"\nAVG TOTAL TIME: {out_time} us\n")
+
+        df2.to_csv(filename, mode='a')
+        with open(filename, 'a') as f:
+            f.write(f"AVG TOTAL TIME: {out_time} us\n")
+        print(f"OUTPUT CSV FILE:\t{filename}")
+
+        if (args.debug):
+            #kernels that took the longest time printed
+            for item in max_kernel_info_list:
+                print(f"KERNEL NAME: {item['name']}\t\t{item['dur']}")
+
+        with open('rocTX_kernel_timing_details.txt', 'w') as f:
+            f.write(
+                "MOST TIME CONSUMING KERNELS IN EACH ITERATION (EXPECTED TO BE SAME KERNEL):\n"
+            )
+            for i in max_kernel_info_list:
+                f.write(f"KERNEL NAME: {i['name']}\t\t{i['dur']}\n")
+        print("KERNEL TIMING DETAILS:\trocTX_kernel_timing_details.txt")
+        print("ALL DATA FROM ALL RUNS:\trocTX_runs_dataframe.csv")
+
+    elif (args.parse):
+        if not (file):
+            raise Exception("JSON PATH IS NOT PROVIDED FOR PARSING.")
+        parse(file)
+    else:
+        raise Exception("PLEASE PROVIDE A COMMAND: RUN, PARSE, CLEAN")
+
+
+if __name__ == "__main__":
+    main()

tools/test_runner.py

-import os
+import os, sys
 import numpy as np
 import argparse
 import onnx
@@ -54,36 +54,112 @@ def read_pb_file(filename):
         tensor.ParseFromString(data_str)
         np_array = numpy_helper.to_array(tensor)
 
-    return np_array
+    return tensor.name, np_array
 
 
-def wrapup_inputs(io_folder, parameter_names):
-    index = 0
+def wrapup_inputs(io_folder, param_names):
     param_map = {}
-    for param_name in parameter_names:
-        file_name = io_folder + '/input_' + str(index) + '.pb'
-        data = read_pb_file(file_name)
-        param_map[param_name] = data
-        index = index + 1
+    data_array = []
+    name_array = []
+    for i in range(len(param_names)):
+        file_name = io_folder + '/input_' + str(i) + '.pb'
+        name, data = read_pb_file(file_name)
+        param_map[name] = data
+        data_array.append(data)
+        if name:
+            name_array.append(name)
+
+    if len(name_array) < len(data_array):
+        param_map = {}
+        for i in range(len(param_names)):
+            param_map[param_names[i]] = data_array[i]
+
+        return param_map
+
+    for name in param_names:
+        if not name in param_map.keys():
+            print("Input {} does not exist!".format(name))
+            sys.exit()
 
     return param_map
 
 
-def read_outputs(io_folder, out_num):
+def read_outputs(io_folder, out_names):
     outputs = []
-    for i in range(out_num):
+    data_array = []
+    name_array = []
+    for i in range(len(out_names)):
         file_name = io_folder + '/output_' + str(i) + '.pb'
-        data = read_pb_file(file_name)
-        outputs.append(data)
+        name, data = read_pb_file(file_name)
+        data_array.append(data)
+        if name:
+            name_array.append(name)
+
+    if len(name_array) < len(data_array):
+        return data_array
+
+    for name in out_names:
+        index = name_array.index(name)
+        outputs.append(data_array[index])
 
     return outputs
 
 
+def model_parameter_names(model_file_name):
+    with open(model_file_name, 'rb') as pfile:
+        data_str = pfile.read()
+        model_proto = onnx.ModelProto()
+        model_proto.ParseFromString(data_str)
+        init_names = set([(i.name) for i in model_proto.graph.initializer])
+        param_names = [
+            input.name for input in model_proto.graph.input
+            if input.name not in init_names
+        ]
+
+        return param_names
+
+
+def model_output_names(model_file_name):
+    with open(model_file_name, 'rb') as pfile:
+        data_str = pfile.read()
+        model_proto = onnx.ModelProto()
+        model_proto.ParseFromString(data_str)
+        output_names = [out.name for out in model_proto.graph.output]
+
+        return output_names
+
+
+def get_input_shapes(sample_case, param_names):
+    param_shape_map = {}
+    name_array = []
+    shape_array = []
+    for i in range(len(param_names)):
+        file_name = sample_case + '/input_' + str(i) + '.pb'
+        name, data = read_pb_file(file_name)
+        param_shape_map[name] = data.shape
+        shape_array.append(data.shape)
+        if name:
+            name_array.append(name)
+
+    if len(name_array) < len(shape_array):
+        param_shape_map = {}
+        for i in range(len(param_names)):
+            param_shape_map[param_names[i]] = shape_array[i]
+
+        return param_shape_map
+
+    for name in param_names:
+        if not name in param_shape_map:
+            print("Input {} does not exist!".format(name))
+            sys.exit()
+
+    return param_shape_map
+
+
 def run_one_case(model, param_map):
     # convert np array to model argument
     pp = {}
     for key, val in param_map.items():
-        print("input = {}".format(val))
         pp[key] = migraphx.argument(val)
 
     # run the model
@@ -106,12 +182,11 @@ def check_correctness(gold_outputs, outputs, rtol=1e-3, atol=1e-3):
     out_num = len(gold_outputs)
     ret = True
     for i in range(out_num):
-        print("Expected value: \n{}".format(gold_outputs[i]))
-        print("Actual value: \n{}".format(outputs[i]))
         if not np.allclose(gold_outputs[i], outputs[i], rtol, atol):
-            print("Output {} is incorrect ...".format(i))
+            print("\nOutput {} is incorrect ...".format(i))
             print("Expected value: \n{}".format(gold_outputs[i]))
-            print("Actual value: \n{}".format(outputs[i]))
+            print("......")
+            print("Actual value: \n{}\n".format(outputs[i]))
             ret = False
 
     return ret
@@ -142,21 +217,32 @@ def main():
     # get model full path
     model_name = get_model_name(test_loc)
     model_path_name = test_loc + '/' + model_name
-    # read and compile model
-    model = migraphx.parse_onnx(model_path_name)
-    param_names = model.get_parameter_names()
-    output_shapes = model.get_output_shapes()
 
-    model.compile(migraphx.get_target(target))
+    # get param names
+    param_names = model_parameter_names(model_path_name)
+
+    # get output names
+    output_names = model_output_names(model_path_name)
 
     # get test cases
     cases = get_test_cases(test_loc)
+    sample_case = test_loc + '/' + cases[0]
+    param_shapes = get_input_shapes(sample_case, param_names)
+    for name, dims in param_shapes.items():
+        print("Input: {}, shape: {}".format(name, dims))
+    print()
+
+    # read and compile model
+    model = migraphx.parse_onnx(model_path_name, map_input_dims=param_shapes)
+    model.compile(migraphx.get_target(target))
+
+    # get test cases
     case_num = len(cases)
     correct_num = 0
     for case_name in cases:
         io_folder = test_loc + '/' + case_name
         input_data = wrapup_inputs(io_folder, param_names)
-        gold_output_data = read_outputs(io_folder, len(output_shapes))
+        gold_outputs = read_outputs(io_folder, output_names)
 
         # if input shape is different from model shape, reload and recompile
         # model
@@ -170,7 +256,7 @@ def main():
         output_data = run_one_case(model, input_data)
 
         # check output correctness
-        ret = check_correctness(gold_output_data, output_data)
+        ret = check_correctness(gold_outputs, output_data)
         if ret:
             correct_num += 1