Device kernels using FP8 (#2510)

src/targets/gpu/target.cpp

@@ -110,6 +110,16 @@ std::vector<pass> target::get_passes(migraphx::context& gctx, const compile_opti
     {
         unsupported_fp8_ops.insert("dot");
     }
+    // add all device kernels
+    unsupported_fp8_ops.insert("logsoftmax");
+    unsupported_fp8_ops.insert("nonzero");
+    unsupported_fp8_ops.insert("prefix_scan_sum");
+    unsupported_fp8_ops.insert("scatter_none");
+    unsupported_fp8_ops.insert("topk");
+    unsupported_fp8_ops.insert("rnn_var_sl_shift_output");
+    unsupported_fp8_ops.insert("multinomial");
+    unsupported_fp8_ops.insert("argmax");
+    unsupported_fp8_ops.insert("argmin");
     // clang-format off
     return
     {

test/verify/gemm_2args_mm_8.cpp

@@ -48,5 +48,5 @@ struct gemm_2args_mm_8 : verify_program<gemm_2args_mm_8<DType>>
 };
 
 template struct gemm_2args_mm_8<migraphx::shape::float_type>;
-// template struct gemm_2args_mm_8<migraphx::shape::half_type>;
+// template struct gemm_2args_mm_8<migraphx::shape::half_type>; // fails with CK, issue#2514
 template struct gemm_2args_mm_8<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/gemm_add_broadcast2.cpp

@@ -51,5 +51,5 @@ struct gemm_add_broadcast2 : verify_program<gemm_add_broadcast2<DType>>
 };
 
 template struct gemm_add_broadcast2<migraphx::shape::float_type>;
-// template struct gemm_add_broadcast2<migraphx::shape::half_type>;
+// template struct gemm_add_broadcast2<migraphx::shape::half_type>; // fails with CK, issue#2514
 template struct gemm_add_broadcast2<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_contiguous.cpp

@@ -29,16 +29,20 @@
 
 #include <cassert>
 
-struct test_contiguous : verify_program<test_contiguous>
+template <migraphx::shape::type_t DType>
+struct test_contiguous : verify_program<test_contiguous<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {4, 4, 4, 3}, {48, 4, 1, 16}};
+        migraphx::shape s{DType, {4, 4, 4, 3}, {48, 4, 1, 16}};
         auto x = mm->add_parameter("x", s);
         mm->add_instruction(migraphx::make_op("contiguous"), x);
         assert(p.get_output_shapes().back().standard());
         return p;
     }
 };
+
+template struct test_contiguous<migraphx::shape::float_type>;
+template struct test_contiguous<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_logsoftmax.cpp

@@ -50,3 +50,7 @@ template struct test_logsoftmax<1, migraphx::shape::half_type>;
 template struct test_logsoftmax<0, migraphx::shape::half_type>;
 template struct test_logsoftmax<2, migraphx::shape::half_type>;
 template struct test_logsoftmax<3, migraphx::shape::half_type>;
+template struct test_logsoftmax<0, migraphx::shape::fp8e4m3fnuz_type>;
+template struct test_logsoftmax<1, migraphx::shape::fp8e4m3fnuz_type>;
+template struct test_logsoftmax<2, migraphx::shape::fp8e4m3fnuz_type>;
+template struct test_logsoftmax<3, migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_multinomial.cpp

@@ -27,7 +27,8 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_multinomial : verify_program<test_multinomial>
+template <migraphx::shape::type_t DType>
+struct test_multinomial : verify_program<test_multinomial<DType>>
 {
     migraphx::program create_program() const
     {
@@ -40,10 +41,10 @@ struct test_multinomial : verify_program<test_multinomial>
         std::uniform_real_distribution<> dis(0.0, 1.0);
         std::vector<float> rand_samples(batch_size * sample_size);
         std::generate(rand_samples.begin(), rand_samples.end(), [&]() { return dis(gen); });
-        migraphx::shape rs{migraphx::shape::float_type, {batch_size, sample_size}};
+        migraphx::shape rs{DType, {batch_size, sample_size}};
         auto rs_lit = mm->add_literal(migraphx::literal{rs, rand_samples});
 
-        migraphx::shape s{migraphx::shape::float_type, {batch_size, 5}};
+        migraphx::shape s{DType, {batch_size, 5}};
         auto input = mm->add_parameter("input", s);
 
         auto maxes = mm->add_instruction(migraphx::make_op("reduce_max", {{"axes", {1}}}), input);
@@ -58,3 +59,8 @@ struct test_multinomial : verify_program<test_multinomial>
         return p;
     }
 };
+
+template struct test_multinomial<migraphx::shape::float_type>;
+template struct test_multinomial<migraphx::shape::half_type>;
+// This fails, need to figure out why
+// template struct test_multinomial<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_nonzero.cpp

@@ -27,13 +27,14 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_nonzero : verify_program<test_nonzero>
+template <migraphx::shape::type_t DType>
+struct test_nonzero : verify_program<test_nonzero<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {2, 3, 4, 5}};
+        migraphx::shape s{DType, {2, 3, 4, 5}};
         auto x = mm->add_parameter("data", s);
         auto r = mm->add_instruction(migraphx::make_op("nonzero"), x);
         mm->add_return({r});
@@ -41,3 +42,7 @@ struct test_nonzero : verify_program<test_nonzero>
         return p;
     }
 };
+
+template struct test_nonzero<migraphx::shape::float_type>;
+template struct test_nonzero<migraphx::shape::half_type>;
+template struct test_nonzero<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_nonzero_half.cpp

-/*
- * The MIT License (MIT)
- *
- * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "verify_program.hpp"
-#include <migraphx/program.hpp>
-#include <migraphx/generate.hpp>
-#include <migraphx/make_op.hpp>
-
-struct test_nonzero_half : verify_program<test_nonzero_half>
-{
-    migraphx::program create_program() const
-    {
-        migraphx::program p;
-        auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::half_type, {3, 4, 3, 5}};
-        auto x = mm->add_parameter("data", s);
-        auto r = mm->add_instruction(migraphx::make_op("nonzero"), x);
-        mm->add_return({r});
-
-        return p;
-    }
-};

test/verify/test_prefix_scan_sum_2d.cpp

@@ -23,16 +23,18 @@
  */
 #include "verify_program.hpp"
 #include <migraphx/program.hpp>
+#include <migraphx/shape.hpp>
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_prefix_scan_sum_2d_small : verify_program<test_prefix_scan_sum_2d_small>
+template <migraphx::shape::type_t DType>
+struct test_prefix_scan_sum_2d_small : verify_program<test_prefix_scan_sum_2d_small<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {1}};
+        migraphx::shape s{DType, {1}};
         auto x = mm->add_parameter("x", s);
         auto xb =
             mm->add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", {3, 3}}}), x);
@@ -42,16 +44,25 @@ struct test_prefix_scan_sum_2d_small : verify_program<test_prefix_scan_sum_2d_sm
     }
 };
 
-struct test_prefix_scan_sum_2d_large : verify_program<test_prefix_scan_sum_2d_large>
+template struct test_prefix_scan_sum_2d_small<migraphx::shape::float_type>;
+template struct test_prefix_scan_sum_2d_small<migraphx::shape::half_type>;
+template struct test_prefix_scan_sum_2d_small<migraphx::shape::fp8e4m3fnuz_type>;
+
+template <migraphx::shape::type_t DType>
+struct test_prefix_scan_sum_2d_large : verify_program<test_prefix_scan_sum_2d_large<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {3, 1000}};
+        migraphx::shape s{DType, {3, 1000}};
         auto x = mm->add_parameter("x", s);
         mm->add_instruction(
             migraphx::make_op("prefix_scan_sum", {{"axis", 1}, {"exclusive", false}}), x);
         return p;
     }
 };
+
+template struct test_prefix_scan_sum_2d_large<migraphx::shape::float_type>;
+template struct test_prefix_scan_sum_2d_large<migraphx::shape::half_type>;
+template struct test_prefix_scan_sum_2d_large<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_reverse.cpp

@@ -26,16 +26,21 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_reverse : verify_program<test_reverse>
+template <migraphx::shape::type_t DType>
+struct test_reverse : verify_program<test_reverse<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {4, 16}};
+        migraphx::shape s{DType, {4, 16}};
         auto a0                   = mm->add_parameter("data", s);
         std::vector<int64_t> axis = {0};
         mm->add_instruction(migraphx::make_op("reverse", {{"axes", axis}}), a0);
         return p;
     }
 };
+
+template struct test_reverse<migraphx::shape::float_type>;
+template struct test_reverse<migraphx::shape::half_type>;
+template struct test_reverse<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_rnn_sql_1.cpp

@@ -31,7 +31,8 @@
 
 #include <migraphx/op/common.hpp>
 
-struct test_rnn_sql_1 : verify_program<test_rnn_sql_1>
+template <migraphx::shape::type_t DType>
+struct test_rnn_sql_1 : verify_program<test_rnn_sql_1<DType>>
 {
     migraphx::program create_program() const
     {
@@ -44,12 +45,12 @@ struct test_rnn_sql_1 : verify_program<test_rnn_sql_1>
 
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape in_shape{migraphx::shape::float_type, {seq_len, batch_size, input_size}};
-        migraphx::shape w_shape{migraphx::shape::float_type, {num_dirct, hidden_size, input_size}};
-        migraphx::shape r_shape{migraphx::shape::float_type, {num_dirct, hidden_size, hidden_size}};
-        migraphx::shape b_shape{migraphx::shape::float_type, {num_dirct, 2 * hidden_size}};
+        migraphx::shape in_shape{DType, {seq_len, batch_size, input_size}};
+        migraphx::shape w_shape{DType, {num_dirct, hidden_size, input_size}};
+        migraphx::shape r_shape{DType, {num_dirct, hidden_size, hidden_size}};
+        migraphx::shape b_shape{DType, {num_dirct, 2 * hidden_size}};
         migraphx::shape s_shape{migraphx::shape::int32_type, {batch_size}};
-        migraphx::shape ih_shape{migraphx::shape::float_type, {num_dirct, batch_size, hidden_size}};
+        migraphx::shape ih_shape{DType, {num_dirct, batch_size, hidden_size}};
 
         auto seq  = mm->add_parameter("seq", in_shape);
         auto w    = mm->add_parameter("w", w_shape);
@@ -81,3 +82,7 @@ struct test_rnn_sql_1 : verify_program<test_rnn_sql_1>
     }
     std::string section() const { return "rnn"; }
 };
+
+template struct test_rnn_sql_1<migraphx::shape::float_type>;
+template struct test_rnn_sql_1<migraphx::shape::half_type>;
+template struct test_rnn_sql_1<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_scatter0.cpp

@@ -27,16 +27,17 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_scatter0 : verify_program<test_scatter0>
+template <migraphx::shape::type_t DType>
+struct test_scatter0 : verify_program<test_scatter0<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape sd{migraphx::shape::float_type, {3, 3}};
+        migraphx::shape sd{DType, {3, 3}};
         migraphx::shape si{migraphx::shape::int32_type, {2, 3}};
         std::vector<int> vi = {1, 0, 2, 0, 2, 1};
-        migraphx::shape su{migraphx::shape::float_type, {2, 3}};
+        migraphx::shape su{DType, {2, 3}};
 
         auto pd = mm->add_parameter("data", sd);
         auto li = mm->add_literal(migraphx::literal{si, vi});
@@ -47,3 +48,7 @@ struct test_scatter0 : verify_program<test_scatter0>
         return p;
     }
 };
+
+template struct test_scatter0<migraphx::shape::float_type>;
+template struct test_scatter0<migraphx::shape::half_type>;
+template struct test_scatter0<migraphx::shape::fp8e4m3fnuz_type>;

test/verify/test_topk_0.cpp

@@ -27,13 +27,14 @@
 #include <migraphx/generate.hpp>
 #include <migraphx/make_op.hpp>
 
-struct test_topk_0 : verify_program<test_topk_0>
+template <migraphx::shape::type_t DType>
+struct test_topk_0 : verify_program<test_topk_0<DType>>
 {
     migraphx::program create_program() const
     {
         migraphx::program p;
         auto* mm = p.get_main_module();
-        migraphx::shape s{migraphx::shape::float_type, {3, 5}};
+        migraphx::shape s{DType, {3, 5}};
         auto data = mm->add_parameter("data", s);
         auto r    = mm->add_instruction(
             migraphx::make_op("topk", {{"axis", 1}, {"k", 4}, {"largest", 1}}), data);
@@ -43,3 +44,7 @@ struct test_topk_0 : verify_program<test_topk_0>
         return p;
     }
 };
+
+template struct test_topk_0<migraphx::shape::float_type>;
+template struct test_topk_0<migraphx::shape::half_type>;
+template struct test_topk_0<migraphx::shape::fp8e4m3fnuz_type>;