Merge branch 'develop' into navi-reduce

test/verify/test_avg_pooling_3d_opt.cpp

@@ -36,7 +36,7 @@ struct test_avg_pooling_3d_opt : verify_program<test_avg_pooling_3d_opt>
         auto input =
             mm->add_parameter("x", migraphx::shape{migraphx::shape::float_type, {4, 2, 3, 3, 3}});
         auto op = migraphx::op::pooling{
-            migraphx::op::pooling_mode::average, {0, 0, 0}, {1, 1, 1}, {3, 3, 3}};
+            migraphx::op::pooling_mode::average, {0, 0, 0}, {1, 1, 1}, {3, 3, 3}, {1, 1, 1}};
         mm->add_instruction(op, input);
         return p;
     }

test/verify/test_avg_pooling_ceil_3d.cpp

@@ -37,7 +37,7 @@ struct test_avg_pooling_ceil_3d : verify_program<test_avg_pooling_ceil_3d>
         auto input =
             mm->add_parameter("x", migraphx::shape{migraphx::shape::float_type, {1, 3, 5, 5, 5}});
         auto op = migraphx::op::pooling{
-            migraphx::op::pooling_mode::average, {1, 1, 1}, {3, 3, 3}, {3, 3, 3}, true};
+            migraphx::op::pooling_mode::average, {1, 1, 1}, {3, 3, 3}, {3, 3, 3}, {1, 1, 1}, true};
         mm->add_instruction(op, input);
         return p;
     }

test/verify/test_avg_pooling_pad.cpp

@@ -36,7 +36,7 @@ struct test_avg_pooling_pad : verify_program<test_avg_pooling_pad>
         auto* mm = p.get_main_module();
         auto input =
             mm->add_parameter("x", migraphx::shape{migraphx::shape::float_type, {1, 3, 7}});
-        auto op = migraphx::op::pooling{migraphx::op::pooling_mode::average, {2}, {1}, {3}};
+        auto op = migraphx::op::pooling{migraphx::op::pooling_mode::average, {2}, {1}, {3}, {1}};
         mm->add_instruction(op, input);
         return p;
     }

test/verify/test_ceil.cpp

@@ -27,16 +27,21 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_ceil : verify_program<test_ceil>
+template <migraphx::shape::type_t DType>
+struct test_ceil : verify_program<test_ceil<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
 
-        migraphx::shape s{migraphx::shape::double_type, {2, 3, 4, 6}};
+        migraphx::shape s{DType, {2, 3, 4, 6}};
         auto param = mm->add_parameter("x", s);
         mm->add_instruction(migraphx::make_op("ceil"), param);
         return p;
     };
 };
+
+template struct test_ceil<migraphx::shape::float_type>;
+template struct test_ceil<migraphx::shape::half_type>;
+template struct test_ceil<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_concat_axis_0.cpp

@@ -27,16 +27,17 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_concat_axis_0 : verify_program<test_concat_axis_0>
+template <migraphx::shape::type_t DType>
+struct test_concat_axis_0 : verify_program<test_concat_axis_0<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
         int axis = 0;
-        migraphx::shape s0{migraphx::shape::int32_type, {2, 2}};
-        migraphx::shape s1{migraphx::shape::int32_type, {3, 2}};
-        migraphx::shape s2{migraphx::shape::int32_type, {1, 2}};
+        migraphx::shape s0{DType, {2, 2}};
+        migraphx::shape s1{DType, {3, 2}};
+        migraphx::shape s2{DType, {1, 2}};
         auto l0 = mm->add_parameter("x", s0);
         auto l1 = mm->add_parameter("y", s1);
         auto l2 = mm->add_parameter("z", s2);
@@ -44,3 +45,8 @@ struct test_concat_axis_0 : verify_program<test_concat_axis_0>
         return p;
     }
 };
+
+template struct test_concat_axis_0<migraphx::shape::fp8e4m3fnuz_type>;
+template struct test_concat_axis_0<migraphx::shape::half_type>;
+template struct test_concat_axis_0<migraphx::shape::float_type>;
+template struct test_concat_axis_0<migraphx::shape::int32_type>;

test/verify/test_concat_pooling.cpp

@@ -47,7 +47,8 @@ struct test_concat_pooling : verify_program<test_concat_pooling>
                                                   {{"mode", migraphx::op::pooling_mode::average},
                                                    {"padding", {0, 0}},
                                                    {"stride", {1, 1}},
-                                                   {"lengths", {8, 8}}}),
+                                                   {"lengths", {8, 8}},
+                                                   {"dilations", {1, 1}}}),
                                 concat_t);
         mm->add_instruction(migraphx::make_op("relu"), pooling);
         return p;

test/verify/test_conv_bn_relu_pooling.cpp

@@ -76,7 +76,8 @@ struct test_conv_bn_relu_pooling : verify_program<test_conv_bn_relu_pooling>
                                               {{"mode", migraphx::op::pooling_mode::average},
                                                {"padding", {1, 1}},
                                                {"stride", {2, 2}},
-                                               {"lengths", {3, 3}}}),
+                                               {"lengths", {3, 3}},
+                                               {"dilations", {1, 1}}}),
                             relu);
         return p;
     }

test/verify/test_conv_bn_relu_pooling2.cpp

@@ -92,7 +92,8 @@ struct test_conv_bn_relu_pooling2 : verify_program<test_conv_bn_relu_pooling2>
                                               {{"mode", migraphx::op::pooling_mode::average},
                                                {"padding", {1, 1}},
                                                {"stride", {2, 2}},
-                                               {"lengths", {3, 3}}}),
+                                               {"lengths", {3, 3}},
+                                               {"dilations", {1, 1}}}),
                             relu);
         return p;
     }

test/verify/test_cos.cpp

@@ -27,15 +27,20 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_cos : verify_program<test_cos>
+template <migraphx::shape::type_t DType>
+struct test_cos : verify_program<test_cos<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {8}};
+        migraphx::shape s{DType, {8}};
         auto x = mm->add_parameter("x", s);
         mm->add_instruction(migraphx::make_op("cos"), x);
         return p;
     }
 };
+
+template struct test_cos<migraphx::shape::float_type>;
+template struct test_cos<migraphx::shape::half_type>;
+template struct test_cos<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_cosh.cpp

@@ -27,15 +27,20 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_cosh : verify_program<test_cosh>
+template <migraphx::shape::type_t DType>
+struct test_cosh : verify_program<test_cosh<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {16}};
+        migraphx::shape s{DType, {16}};
         auto x = mm->add_parameter("x", s);
         mm->add_instruction(migraphx::make_op("cosh"), x);
         return p;
     }
 };
+
+template struct test_cosh<migraphx::shape::float_type>;
+template struct test_cosh<migraphx::shape::half_type>;
+template struct test_cosh<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_erf.cpp

@@ -27,15 +27,20 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_erf : verify_program<test_erf>
+template <migraphx::shape::type_t DType>
+struct test_erf : verify_program<test_erf<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {2, 3, 4, 6}};
+        migraphx::shape s{DType, {2, 3, 4, 6}};
         auto param = mm->add_parameter("x", s);
         mm->add_instruction(migraphx::make_op("erf"), param);
         return p;
     }
 };
+
+template struct test_erf<migraphx::shape::float_type>;
+template struct test_erf<migraphx::shape::half_type>;
+template struct test_erf<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_exp.cpp

@@ -27,15 +27,20 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_exp : verify_program<test_exp>
+template <migraphx::shape::type_t DType>
+struct test_exp : verify_program<test_exp<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {6}};
+        migraphx::shape s{DType, {6}};
         auto x = mm->add_instruction(migraphx::make_op("abs"), mm->add_parameter("x", s));
         mm->add_instruction(migraphx::make_op("exp"), x);
         return p;
     }
 };
+
+template struct test_exp<migraphx::shape::float_type>;
+template struct test_exp<migraphx::shape::half_type>;
+template struct test_exp<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_floor.cpp

@@ -27,16 +27,21 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_floor : verify_program<test_floor>
+template <migraphx::shape::type_t DType>
+struct test_floor : verify_program<test_floor<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
 
-        migraphx::shape s{migraphx::shape::float_type, {2, 3, 4, 6}};
+        migraphx::shape s{DType, {2, 3, 4, 6}};
         auto param = mm->add_parameter("x", s);
         mm->add_instruction(migraphx::make_op("floor"), param);
         return p;
     };
 };
+
+template struct test_floor<migraphx::shape::float_type>;
+template struct test_floor<migraphx::shape::half_type>;
+template struct test_floor<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_fmod_mod.cpp

@@ -34,40 +34,52 @@
     Adding this because HIP fmod sign changes when y = 0 resulting in nan and -nan not beign
    consistent between ref and gpu implementations.
 */
-migraphx::instruction_ref add_epsilon(migraphx::module& m, migraphx::instruction_ref y)
+migraphx::instruction_ref add_epsilon(migraphx::module& m,
+                                      migraphx::instruction_ref y,
+                                      migraphx::shape::type_t dtype = migraphx::shape::float_type)
 {
-    auto zero = m.add_literal(0.0f);
-    auto eps  = m.add_literal(1e-3f);
+    auto zero = m.add_literal(migraphx::literal{migraphx::shape{dtype}, {0.0f}});
+    auto eps  = m.add_literal(migraphx::literal{migraphx::shape{dtype}, {1e-3f}});
     auto op_y = add_common_op(m, migraphx::make_op("equal"), {y, zero});
     return add_common_op(m, migraphx::make_op("where"), {op_y, eps, y});
 }
 
-struct test_fmod : verify_program<test_fmod>
+template <migraphx::shape::type_t DType>
+struct test_fmod : verify_program<test_fmod<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {64}};
+        migraphx::shape s{DType, {64}};
         auto x        = mm->add_parameter("x", s);
         auto y        = mm->add_parameter("y", s);
-        auto op_where = add_epsilon(*mm, y);
+        auto op_where = add_epsilon(*mm, y, DType);
         mm->add_instruction(migraphx::make_op("fmod"), x, op_where);
         return p;
     }
 };
+template struct test_fmod<migraphx::shape::float_type>;
+template struct test_fmod<migraphx::shape::half_type>;
+template struct test_fmod<migraphx::shape::fp8e4m3fnuz_type>;
 
-struct test_mod : verify_program<test_mod>
+template <migraphx::shape::type_t DType>
+struct test_mod : verify_program<test_mod<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {64}};
+        migraphx::shape s{DType, {64}};
         auto x        = mm->add_parameter("x", s);
         auto y        = mm->add_parameter("y", s);
-        auto op_where = add_epsilon(*mm, y);
+        auto op_where = add_epsilon(*mm, y, DType);
         mm->add_instruction(migraphx::make_op("mod"), x, op_where);
         return p;
     }
 };
+
+template struct test_mod<migraphx::shape::float_type>;
+// TODO: Fix half type test
+// template struct test_mod<migraphx::shape::half_type>;
+template struct test_mod<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_gather.cpp

@@ -27,14 +27,14 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-template <int Axis>
-struct test_gather : verify_program<test_gather<Axis>>
+template <int Axis, migraphx::shape::type_t DType>
+struct test_gather : verify_program<test_gather<Axis, DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {3, 3}};
+        migraphx::shape s{DType, {3, 3}};
         migraphx::shape s_indices{migraphx::shape::int32_type, {2, 2}};
         std::vector<int> indices{1, 2, 2, 1};
         auto a0  = mm->add_parameter("data", s);
@@ -46,6 +46,10 @@ struct test_gather : verify_program<test_gather<Axis>>
 };
 
 // Standard gather test
-template struct test_gather<0>;
+template struct test_gather<0, migraphx::shape::float_type>;
+template struct test_gather<0, migraphx::shape::half_type>;
+template struct test_gather<0, migraphx::shape::fp8e4m3fnuz_type>;
 // Test Negative axis
-template struct test_gather<-2>;
+template struct test_gather<-2, migraphx::shape::float_type>;
+template struct test_gather<-2, migraphx::shape::half_type>;
+template struct test_gather<-2, migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_gathernd_default.cpp

@@ -26,13 +26,14 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_gathernd_default : verify_program<test_gathernd_default>
+template <migraphx::shape::type_t DType>
+struct test_gathernd_default : verify_program<test_gathernd_default<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape ds{migraphx::shape::float_type, {2, 2}};
+        migraphx::shape ds{DType, {2, 2}};
         migraphx::shape is{migraphx::shape::int64_type, {2, 2}};
         std::vector<int64_t> indices{0, 0, 1, 1};
         auto a0 = mm->add_parameter("data", ds);
@@ -41,3 +42,7 @@ struct test_gathernd_default : verify_program<test_gathernd_default>
         return p;
     }
 };
+
+template struct test_gathernd_default<migraphx::shape::float_type>;
+template struct test_gathernd_default<migraphx::shape::half_type>;
+template struct test_gathernd_default<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_isnan_float.cpp → test/verify/test_isnan.cpp

@@ -27,7 +27,8 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_isnan_float : verify_program<test_isnan_float>
+template <migraphx::shape::type_t DType>
+struct test_isnan : verify_program<test_isnan<DType>>
 {
     migraphx::program create_program() const
     {
@@ -40,3 +41,7 @@ struct test_isnan_float : verify_program<test_isnan_float>
         return p;
     }
 };
+
+template struct test_isnan<migraphx::shape::float_type>;
+template struct test_isnan<migraphx::shape::half_type>;
+template struct test_isnan<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_layernorm.cpp

@@ -117,6 +117,19 @@ struct test_layernorm_fp16 : verify_program<test_layernorm_fp16>
     }
 };
 
+struct test_layernorm_fp8 : verify_program<test_layernorm_fp8>
+{
+    migraphx::program create_program() const
+    {
+        migraphx::program p;
+        auto* mm                 = p.get_main_module();
+        std::vector<size_t> dims = {1, 24, 64};
+        auto x = mm->add_parameter("x", migraphx::shape{migraphx::shape::fp8e4m3fnuz_type, dims});
+        add_layernorm(*mm, x, dims);
+        return p;
+    }
+};
+
 struct test_layernorm_eps : verify_program<test_layernorm_eps>
 {
     migraphx::program create_program() const

test/verify/test_literal_limits.cpp

@@ -26,6 +26,7 @@
 #include <migraphx/program.hpp>
 #include <migraphx/make_op.hpp>
 #include <limits>
+#include <type_traits>
 
 template <migraphx::shape::type_t Q, typename T>
 struct test_literal_limits : verify_program<test_literal_limits<Q, T>>
@@ -33,9 +34,13 @@ struct test_literal_limits : verify_program<test_literal_limits<Q, T>>
     migraphx::program create_program() const
     {
         migraphx::program p;
-        auto* mm          = p.get_main_module();
-        auto input_s      = migraphx::shape(Q, {3, 1});
-        auto infinity_val = std::numeric_limits<T>::infinity();
+        auto* mm     = p.get_main_module();
+        auto input_s = migraphx::shape(Q, {3, 1});
+        T infinity_val{0};
+        if constexpr(std::numeric_limits<T>::has_infinity and std::is_floating_point<T>{})
+        {
+            infinity_val = std::numeric_limits<T>::infinity();
+        }
         std::vector<T> s_data{
             infinity_val, static_cast<T>(-infinity_val), std::numeric_limits<T>::quiet_NaN()};
 
@@ -52,3 +57,4 @@ template struct test_literal_limits<migraphx::shape::double_type, double>;
 template struct test_literal_limits<migraphx::shape::half_type, migraphx::half>;
 template struct test_literal_limits<migraphx::shape::int32_type, int32_t>;
 template struct test_literal_limits<migraphx::shape::int8_type, int8_t>;
+template struct test_literal_limits<migraphx::shape::fp8e4m3fnuz_type, migraphx::fp8::fp8e4m3fnuz>;

test/verify/test_log.cpp

@@ -27,15 +27,20 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_log : verify_program<test_log>
+template <migraphx::shape::type_t DType>
+struct test_log : verify_program<test_log<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {6}};
+        migraphx::shape s{DType, {6}};
         auto x = mm->add_instruction(migraphx::make_op("abs"), mm->add_parameter("x", s));
         mm->add_instruction(migraphx::make_op("log"), x);
         return p;
     }
 };
+
+template struct test_log<migraphx::shape::float_type>;
+template struct test_log<migraphx::shape::half_type>;
+template struct test_log<migraphx::shape::fp8e4m3fnuz_type>;