Merge branch 'rocblas_mlir_fp8' into miopen_fp8

test/verify/quant_conv_1d.cpp

@@ -27,15 +27,16 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct quant_conv_1d : verify_program<quant_conv_1d>
+template <migraphx::shape::type_t DType>
+struct quant_conv_1d : verify_program<quant_conv_1d<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape a_shape{migraphx::shape::int8_type, {2, 3, 4}};
+        migraphx::shape a_shape{DType, {2, 3, 4}};
         auto pa = mm->add_parameter("a", a_shape);
-        migraphx::shape c_shape{migraphx::shape::int8_type, {2, 3, 3}};
+        migraphx::shape c_shape{DType, {2, 3, 3}};
         auto pc = mm->add_parameter("c", c_shape);
         mm->add_instruction(
             migraphx::make_op("quant_convolution",
@@ -45,3 +46,7 @@ struct quant_conv_1d : verify_program<quant_conv_1d>
         return p;
     }
 };
+
+template struct quant_conv_1d<migraphx::shape::int8_type>;
+// MLIR 1D convolution is not supported in MIGraphX yet. Enable this through MIOpen route later.
+// template struct quant_conv_1d<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/quant_conv_2.cpp

@@ -27,17 +27,21 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/op/quant_convolution.hpp>
 
-struct quant_conv_2 : verify_program<quant_conv_2>
+template <migraphx::shape::type_t DType>
+struct quant_conv_2 : verify_program<quant_conv_2<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape a_shape{migraphx::shape::int8_type, {16, 16, 4, 4}};
+        migraphx::shape a_shape{DType, {16, 16, 4, 4}};
         auto pa = mm->add_parameter("a", a_shape);
-        migraphx::shape c_shape{migraphx::shape::int8_type, {16, 16, 3, 3}};
+        migraphx::shape c_shape{DType, {16, 16, 3, 3}};
         auto pc = mm->add_parameter("c", c_shape);
         mm->add_instruction(migraphx::op::quant_convolution{{{0, 0}}, {{1, 1}}, {{1, 1}}}, pa, pc);
         return p;
     }
 };
+
+template struct quant_conv_2<migraphx::shape::int8_type>;
+template struct quant_conv_2<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/quant_conv_padding.cpp

@@ -27,15 +27,16 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct quant_conv_padding : verify_program<quant_conv_padding>
+template <migraphx::shape::type_t DType>
+struct quant_conv_padding : verify_program<quant_conv_padding<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape a_shape{migraphx::shape::int8_type, {2, 3, 4, 4}};
+        migraphx::shape a_shape{DType, {2, 3, 4, 4}};
         auto pa = mm->add_parameter("a", a_shape);
-        migraphx::shape c_shape{migraphx::shape::int8_type, {2, 3, 3, 3}};
+        migraphx::shape c_shape{DType, {2, 3, 3, 3}};
         auto pc = mm->add_parameter("c", c_shape);
         mm->add_instruction(
             migraphx::make_op("quant_convolution", {{"padding", {1, 1}}, {"stride", {1, 1}}}),
@@ -44,3 +45,6 @@ struct quant_conv_padding : verify_program<quant_conv_padding>
         return p;
     }
 };
+
+template struct quant_conv_padding<migraphx::shape::int8_type>;
+template struct quant_conv_padding<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/quant_conv_padding_stride.cpp

@@ -27,15 +27,16 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct quant_conv_padding_stride : verify_program<quant_conv_padding_stride>
+template <migraphx::shape::type_t DType>
+struct quant_conv_padding_stride : verify_program<quant_conv_padding_stride<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape a_shape{migraphx::shape::int8_type, {2, 3, 4, 4}};
+        migraphx::shape a_shape{DType, {2, 3, 4, 4}};
         auto pa = mm->add_parameter("a", a_shape);
-        migraphx::shape c_shape{migraphx::shape::int8_type, {2, 3, 3, 3}};
+        migraphx::shape c_shape{DType, {2, 3, 3, 3}};
         auto pc = mm->add_parameter("c", c_shape);
         mm->add_instruction(
             migraphx::make_op("quant_convolution", {{"padding", {1, 1}}, {"stride", {2, 2}}}),
@@ -45,3 +46,5 @@ struct quant_conv_padding_stride : verify_program<quant_conv_padding_stride>
         return p;
     }
 };
+template struct quant_conv_padding_stride<migraphx::shape::int8_type>;
+template struct quant_conv_padding_stride<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/run_verify.cpp

@@ -142,7 +142,8 @@ std::vector<migraphx::argument> run_verify::run_ref(migraphx::program p,
 {
     migraphx::target t = migraphx::make_target("ref");
     auto_print pp{p, t.name()};
-    compile_check(p, t, c_opts);
+    auto trace_target = migraphx::string_value_of(MIGRAPHX_TRACE_TEST_COMPILE{});
+    compile_check(p, t, c_opts, (trace_target == "ref"));
     return p.eval(std::move(inputs));
 }
 std::pair<migraphx::program, std::vector<migraphx::argument>>

test/verify/test_acosh.cpp

@@ -38,8 +38,8 @@ struct test_acosh : verify_program<test_acosh<CType>>
         migraphx::shape::type_t dtype = migraphx::shape::get_type<CType>();
         migraphx::shape s{dtype, {16}};
         auto x       = mm->add_parameter("x", s);
-        auto min_val = mm->add_literal(migraphx::literal{migraphx::shape{dtype}, {1.1}});
-        auto max_val = mm->add_literal(migraphx::literal{migraphx::shape{dtype}, {100.0}});
+        auto min_val = mm->add_literal(migraphx::literal{migraphx::shape{dtype}, {1.1f}});
+        auto max_val = mm->add_literal(migraphx::literal{migraphx::shape{dtype}, {100.0f}});
         min_val =
             mm->add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", {16}}}), min_val);
         max_val =

test/verify/test_avg_pooling_1d.cpp

@@ -35,7 +35,7 @@ struct test_avg_pooling_1d : verify_program<test_avg_pooling_1d>
         auto* mm = p.get_main_module();
         auto input =
             mm->add_parameter("x", migraphx::shape{migraphx::shape::float_type, {1, 3, 5}});
-        auto op = migraphx::op::pooling{migraphx::op::pooling_mode::average, {0}, {1}, {3}};
+        auto op = migraphx::op::pooling{migraphx::op::pooling_mode::average, {0}, {1}, {3}, {1}};
         mm->add_instruction(op, input);
         return p;
     }

test/verify/test_avg_pooling_3d.cpp

@@ -36,7 +36,7 @@ struct test_avg_pooling_3d : verify_program<test_avg_pooling_3d>
         auto input =
             mm->add_parameter("x", migraphx::shape{migraphx::shape::float_type, {1, 3, 5, 5, 5}});
         auto op = migraphx::op::pooling{
-            migraphx::op::pooling_mode::average, {1, 1, 1}, {3, 3, 3}, {3, 3, 3}};
+            migraphx::op::pooling_mode::average, {1, 1, 1}, {3, 3, 3}, {3, 3, 3}, {1, 1, 1}};
         mm->add_instruction(op, input);
         return p;
     }

test/verify/test_avg_pooling_3d_opt.cpp

@@ -36,7 +36,7 @@ struct test_avg_pooling_3d_opt : verify_program<test_avg_pooling_3d_opt>
         auto input =
             mm->add_parameter("x", migraphx::shape{migraphx::shape::float_type, {4, 2, 3, 3, 3}});
         auto op = migraphx::op::pooling{
-            migraphx::op::pooling_mode::average, {0, 0, 0}, {1, 1, 1}, {3, 3, 3}};
+            migraphx::op::pooling_mode::average, {0, 0, 0}, {1, 1, 1}, {3, 3, 3}, {1, 1, 1}};
         mm->add_instruction(op, input);
         return p;
     }

test/verify/test_avg_pooling_ceil_3d.cpp

@@ -37,7 +37,7 @@ struct test_avg_pooling_ceil_3d : verify_program<test_avg_pooling_ceil_3d>
         auto input =
             mm->add_parameter("x", migraphx::shape{migraphx::shape::float_type, {1, 3, 5, 5, 5}});
         auto op = migraphx::op::pooling{
-            migraphx::op::pooling_mode::average, {1, 1, 1}, {3, 3, 3}, {3, 3, 3}, true};
+            migraphx::op::pooling_mode::average, {1, 1, 1}, {3, 3, 3}, {3, 3, 3}, {1, 1, 1}, true};
         mm->add_instruction(op, input);
         return p;
     }

test/verify/test_avg_pooling_pad.cpp

@@ -36,7 +36,7 @@ struct test_avg_pooling_pad : verify_program<test_avg_pooling_pad>
         auto* mm = p.get_main_module();
         auto input =
             mm->add_parameter("x", migraphx::shape{migraphx::shape::float_type, {1, 3, 7}});
-        auto op = migraphx::op::pooling{migraphx::op::pooling_mode::average, {2}, {1}, {3}};
+        auto op = migraphx::op::pooling{migraphx::op::pooling_mode::average, {2}, {1}, {3}, {1}};
         mm->add_instruction(op, input);
         return p;
     }

test/verify/test_concat_axis_0.cpp

@@ -28,7 +28,6 @@
 #include <migraphx/make_op.hpp>
 
 template <migraphx::shape::type_t DType>
-
 struct test_concat_axis_0 : verify_program<test_concat_axis_0<DType>>
 {
     migraphx::program create_program() const

test/verify/test_concat_pooling.cpp

@@ -47,7 +47,8 @@ struct test_concat_pooling : verify_program<test_concat_pooling>
                                                   {{"mode", migraphx::op::pooling_mode::average},
                                                    {"padding", {0, 0}},
                                                    {"stride", {1, 1}},
-                                                   {"lengths", {8, 8}}}),
+                                                   {"lengths", {8, 8}},
+                                                   {"dilations", {1, 1}}}),
                                 concat_t);
         mm->add_instruction(migraphx::make_op("relu"), pooling);
         return p;

test/verify/test_conv.cpp

@@ -34,7 +34,7 @@ struct test_conv : verify_program<test_conv<DType>>
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        auto input = mm->add_parameter("x", migraphx::shape{DType, {4, 3, 3, 3}});
+        auto input   = mm->add_parameter("x", migraphx::shape{DType, {4, 3, 3, 3}});
         auto weights = mm->add_parameter("w", migraphx::shape{DType, {4, 3, 3, 3}});
         mm->add_instruction(migraphx::make_op("convolution"), input, weights);
         return p;

test/verify/test_conv2.cpp

@@ -34,10 +34,8 @@ struct test_conv2 : verify_program<test_conv2<DType>>
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        auto input =
-            mm->add_parameter("x", migraphx::shape{DType, {1, 512, 28, 28}});
-        auto weights =
-            mm->add_parameter("w", migraphx::shape{DType, {256, 512, 1, 1}});
+        auto input   = mm->add_parameter("x", migraphx::shape{DType, {1, 512, 28, 28}});
+        auto weights = mm->add_parameter("w", migraphx::shape{DType, {256, 512, 1, 1}});
         mm->add_instruction(
             migraphx::make_op("convolution",
                               {{"padding", {0, 0}}, {"stride", {1, 1}}, {"dilation", {1, 1}}}),

test/verify/test_conv_add.cpp

@@ -34,9 +34,9 @@ struct test_conv_add : verify_program<test_conv_add<DType>>
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        auto x   = mm->add_parameter("x", {DType, {1, 8, 4, 4}});
-        auto w   = mm->add_literal(migraphx::generate_literal({DType, {2, 8, 3, 3}}, 1));
-        auto y   = mm->add_parameter("y", {DType, {1, 8, 4, 4}});
+        auto x     = mm->add_parameter("x", {DType, {1, 8, 4, 4}});
+        auto w     = mm->add_literal(migraphx::generate_literal({DType, {2, 8, 3, 3}}, 1));
+        auto y     = mm->add_parameter("y", {DType, {1, 8, 4, 4}});
         auto v     = mm->add_literal(migraphx::generate_literal({DType, {2, 8, 3, 3}}, 2));
         auto conv1 = mm->add_instruction(migraphx::make_op("convolution"), x, w);
         auto conv2 = mm->add_instruction(migraphx::make_op("convolution"), y, v);

test/verify/test_conv_add_relu.cpp

@@ -35,12 +35,10 @@ struct test_conv_add_relu : verify_program<test_conv_add_relu<DType>>
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        auto input =
-            mm->add_parameter("x", migraphx::shape{DType, {4, 3, 3, 3}});
-        auto weights =
-            mm->add_parameter("w", migraphx::shape{DType, {4, 3, 3, 3}});
-        auto bias_literal = migraphx::literal{migraphx::shape{DType, {4}},
-                                              {2.0f, 2.0f, 2.0f, 2.0f}};
+        auto input   = mm->add_parameter("x", migraphx::shape{DType, {4, 3, 3, 3}});
+        auto weights = mm->add_parameter("w", migraphx::shape{DType, {4, 3, 3, 3}});
+        auto bias_literal =
+            migraphx::literal{migraphx::shape{DType, {4}}, {2.0f, 2.0f, 2.0f, 2.0f}};
         auto bias         = mm->add_literal(bias_literal);
         auto conv         = mm->add_instruction(migraphx::make_op("convolution"), input, weights);
         auto bcast_bias   = mm->add_instruction(

test/verify/test_conv_bn.cpp

@@ -54,7 +54,7 @@ struct test_conv_bn : verify_program<test_conv_bn<DType>>
         auto mean     = mm->add_literal(migraphx::abs(migraphx::generate_literal(vars, 3)));
         auto variance = mm->add_literal(migraphx::abs(migraphx::generate_literal(vars, 4)));
 
-        auto rt  = mm->add_literal(migraphx::literal{DType, {0.5}});
+        auto rt = mm->add_literal(migraphx::literal{DType, {0.5}});
 
         auto eps = mm->add_literal(migraphx::literal{DType, {1e-5f}});
         if constexpr((DType) == migraphx::shape::fp8e4m3fnuz_type)

test/verify/test_conv_bn_add.cpp

@@ -74,7 +74,7 @@ struct test_conv_bn_add : verify_program<test_conv_bn_add<DType>>
         auto x                = mm->add_parameter("x", {DType, {1, ichannels, 56, 56}});
         auto w =
             mm->add_literal(migraphx::generate_literal({DType, {ochannels, ichannels, 1, 1}}, 1));
-        auto y     = mm->add_parameter("y", {DType, {1, ichannels, 56, 56}});
+        auto y = mm->add_parameter("y", {DType, {1, ichannels, 56, 56}});
         auto v =
             mm->add_literal(migraphx::generate_literal({DType, {ochannels, ichannels, 1, 1}}, 2));
         auto relu1 = mm->add_instruction(migraphx::make_op("relu"), x);

test/verify/test_conv_bn_relu_pooling.cpp

@@ -81,7 +81,8 @@ struct test_conv_bn_relu_pooling : verify_program<test_conv_bn_relu_pooling<DTyp
                                               {{"mode", migraphx::op::pooling_mode::average},
                                                {"padding", {1, 1}},
                                                {"stride", {2, 2}},
-                                               {"lengths", {3, 3}}}),
+                                               {"lengths", {3, 3}},
+                                               {"dilations", {1, 1}}}),
                             relu);
         return p;
     }