Merge branch 'develop' into aosewski/gemm_tile_loop

test/elementwise_normalization/CMakeLists.txt

-add_custom_target(test_elementwise_normalization)
-
-add_gtest_executable(test_elementwise_layernorm_fp16 test_elementwise_layernorm_fp16.cpp)
-
-target_link_libraries(test_elementwise_layernorm_fp16 PRIVATE utility device_elementwise_normalization_instance)
-
-add_dependencies(test_elementwise_normalization test_elementwise_layernorm_fp16)
+if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
+  add_custom_target(test_elementwise_normalization)
+  add_gtest_executable(test_elementwise_layernorm_fp16 test_elementwise_layernorm_fp16.cpp)
+  target_link_libraries(test_elementwise_layernorm_fp16 PRIVATE utility device_elementwise_normalization_instance)
+  add_dependencies(test_elementwise_normalization test_elementwise_layernorm_fp16)
+endif()
\ No newline at end of file

test/gemm_layernorm/CMakeLists.txt

@@ -2,10 +2,12 @@ list(APPEND gpu_list gfx908 gfx90a gfx940 gfx941 gfx942)
 set(target 0)
 foreach(gpu IN LISTS GPU_TARGETS)
  if(gpu IN_LIST gpu_list AND target EQUAL 0)
+  if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
    add_custom_target(test_gemm_layernorm)
    add_gtest_executable(test_gemm_add_relu_add_layernorm_fp16 test_gemm_add_relu_add_layernorm_fp16.cpp)
    target_link_libraries(test_gemm_add_relu_add_layernorm_fp16 PRIVATE utility device_gemm_add_relu_add_layernorm_instance)
    add_dependencies(test_gemm_layernorm test_gemm_add_relu_add_layernorm_fp16)
    set(target 1)
+  endif()
  endif()
 endforeach()

test/gemm_reduce/CMakeLists.txt

-add_test_executable(test_gemm_reduce_fp16 gemm_reduce_fp16.cpp)
-target_link_libraries(test_gemm_reduce_fp16 PRIVATE utility)
-target_link_libraries(test_gemm_reduce_fp16 PRIVATE device_gemm_reduce_instance)
+if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
+  add_test_executable(test_gemm_reduce_fp16 gemm_reduce_fp16.cpp)
+  target_link_libraries(test_gemm_reduce_fp16 PRIVATE utility)
+  target_link_libraries(test_gemm_reduce_fp16 PRIVATE device_gemm_reduce_instance)
+endif()
\ No newline at end of file

test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight.cpp

@@ -100,6 +100,9 @@ TYPED_TEST(TestGroupedConvndBwdWeight1d, Test1D)
     this->conv_params.push_back({1, 2, 128, 128, 256, {1}, {14}, {2}, {1}, {0}, {0}});
     this->conv_params.push_back({1, 2, 32, 128, 256, {3}, {28}, {1}, {1}, {1}, {1}});
     this->conv_params.push_back({1, 2, 128, 128, 256, {1}, {3}, {1}, {1}, {0}, {0}});
+    this->conv_params.push_back({1, 1, 1, 1, 32, {3}, {32}, {1}, {1}, {1}, {1}});
+    this->conv_params.push_back({1, 1, 1, 64, 3, {3}, {32}, {1}, {1}, {1}, {1}});
+    this->conv_params.push_back({1, 1, 1, 1, 1, {3}, {32}, {1}, {1}, {1}, {1}});
     this->Run();
 }
 
@@ -112,6 +115,9 @@ TYPED_TEST(TestGroupedConvndBwdWeight2d, Test2D)
         {2, 2, 4, 128, 256, {3, 3}, {14, 14}, {1, 1}, {1, 1}, {1, 1}, {1, 1}});
     this->conv_params.push_back(
         {2, 2, 128, 128, 256, {1, 1}, {3, 3}, {1, 1}, {1, 1}, {0, 0}, {0, 0}});
+    this->conv_params.push_back({2, 1, 1, 1, 32, {3, 3}, {32, 32}, {1, 1}, {1, 1}, {1, 1}, {1, 1}});
+    this->conv_params.push_back({2, 1, 1, 64, 3, {3, 3}, {32, 32}, {1, 1}, {1, 1}, {1, 1}, {1, 1}});
+    this->conv_params.push_back({2, 1, 1, 1, 1, {3, 3}, {32, 32}, {1, 1}, {1, 1}, {1, 1}, {1, 1}});
     this->Run();
 }
 
@@ -124,5 +130,11 @@ TYPED_TEST(TestGroupedConvndBwdWeight3d, Test3D)
         {3, 2, 2, 128, 256, {3, 3, 3}, {14, 14, 3}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}});
     this->conv_params.push_back(
         {3, 2, 32, 128, 256, {1, 1, 1}, {3, 3, 3}, {1, 1, 1}, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}});
+    this->conv_params.push_back(
+        {3, 1, 1, 1, 32, {3, 3, 3}, {32, 32, 32}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}});
+    this->conv_params.push_back(
+        {3, 1, 1, 64, 3, {3, 3, 3}, {32, 32, 32}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}});
+    this->conv_params.push_back(
+        {3, 1, 1, 1, 1, {3, 3, 3}, {32, 32, 32}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}});
     this->Run();
 }

test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight_interface.cpp

@@ -70,10 +70,11 @@ class TestGroupedConvndBwdWeight : public ::testing::Test
             ck::utils::conv::make_output_host_tensor_descriptor_g_n_k_wos_packed<OutLayout>(
                 conv_param);
 
-        std::array<ck::index_t, NDimSpatial> input_spatial_lengths{};
-        std::array<ck::index_t, NDimSpatial> filter_spatial_lengths{};
-        std::array<ck::index_t, NDimSpatial> output_spatial_lengths{};
+        std::array<ck::index_t, NDimSpatial + 3> input_lengths{};
+        std::array<ck::index_t, NDimSpatial + 3> filter_lengths{};
+        std::array<ck::index_t, NDimSpatial + 3> output_lengths{};
         std::array<ck::index_t, NDimSpatial + 3> input_strides{};
+        std::array<ck::index_t, NDimSpatial + 3> weights_strides{};
         std::array<ck::index_t, NDimSpatial + 3> output_strides{};
         std::array<ck::index_t, NDimSpatial> conv_filter_strides{};
         std::array<ck::index_t, NDimSpatial> conv_filter_dilations{};
@@ -82,10 +83,11 @@ class TestGroupedConvndBwdWeight : public ::testing::Test
 
         auto range_copy = [](const auto& from, auto to) { std::copy(begin(from), end(from), to); };
 
-        range_copy(conv_param.input_spatial_lengths_, begin(input_spatial_lengths));
-        range_copy(conv_param.filter_spatial_lengths_, begin(filter_spatial_lengths));
-        range_copy(conv_param.output_spatial_lengths_, begin(output_spatial_lengths));
+        range_copy(in_g_n_c_wis_desc.GetLengths(), begin(input_lengths));
         range_copy(in_g_n_c_wis_desc.GetStrides(), begin(input_strides));
+        range_copy(wei_g_k_c_xs_desc.GetLengths(), begin(filter_lengths));
+        range_copy(wei_g_k_c_xs_desc.GetStrides(), begin(weights_strides));
+        range_copy(out_g_n_k_wos_desc.GetLengths(), begin(output_lengths));
         range_copy(out_g_n_k_wos_desc.GetStrides(), begin(output_strides));
         range_copy(conv_param.conv_filter_strides_, begin(conv_filter_strides));
         range_copy(conv_param.conv_filter_dilations_, begin(conv_filter_dilations));
@@ -97,14 +99,11 @@ class TestGroupedConvndBwdWeight : public ::testing::Test
         auto argument = conv.MakeArgument(nullptr,
                                           nullptr,
                                           nullptr,
-                                          conv_param.G_,
-                                          conv_param.N_,
-                                          conv_param.K_,
-                                          conv_param.C_,
-                                          input_spatial_lengths,
-                                          filter_spatial_lengths,
-                                          output_spatial_lengths,
+                                          input_lengths,
                                           input_strides,
+                                          filter_lengths,
+                                          weights_strides,
+                                          output_lengths,
                                           output_strides,
                                           conv_filter_strides,
                                           conv_filter_dilations,

test/grouped_convnd_fwd/grouped_convnd_fwd.cpp

@@ -22,6 +22,8 @@ TEST_F(TestGroupedConvNdFwd, GroupedConv1dFwdGNWC)
     conv_params.push_back({1, 2, 128, 128, 256, {1}, {14}, {2}, {1}, {0}, {0}});
     conv_params.push_back({1, 2, 128, 128, 256, {3}, {28}, {1}, {1}, {1}, {1}});
     conv_params.push_back({1, 2, 128, 128, 256, {1}, {3}, {1}, {1}, {0}, {0}});
+    conv_params.push_back({1, 1, 1, 1, 32, {3}, {32}, {1}, {1}, {1}, {1}});
+    conv_params.push_back({1, 1, 1, 64, 3, {3}, {32}, {1}, {1}, {1}, {1}});
 
     for(auto& param : conv_params)
     {
@@ -96,6 +98,9 @@ TEST_F(TestGroupedConvNdFwd, GroupedConv2dFwdGNHWC)
     conv_params.push_back({2, 2, 128, 128, 256, {1, 1}, {7, 7}, {2, 2}, {1, 1}, {0, 0}, {0, 0}});
     conv_params.push_back({2, 2, 128, 128, 256, {3, 3}, {14, 14}, {1, 1}, {1, 1}, {1, 1}, {1, 1}});
     conv_params.push_back({2, 2, 128, 128, 256, {1, 1}, {3, 3}, {1, 1}, {1, 1}, {0, 0}, {0, 0}});
+    conv_params.push_back({2, 1, 1, 1, 32, {3, 3}, {32, 32}, {1, 1}, {1, 1}, {1, 1}, {1, 1}});
+    conv_params.push_back({2, 1, 1, 64, 3, {3, 3}, {32, 32}, {1, 1}, {1, 1}, {1, 1}, {1, 1}});
+    conv_params.push_back({2, 1, 1, 1, 1, {3, 3}, {32, 32}, {1, 1}, {1, 1}, {1, 1}, {1, 1}});
 
     for(auto& param : conv_params)
     {
@@ -173,6 +178,12 @@ TEST_F(TestGroupedConvNdFwd, GroupedConv3dFwdGNDHWC)
         {3, 2, 128, 128, 256, {3, 3, 3}, {14, 14, 3}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}});
     conv_params.push_back(
         {3, 2, 128, 128, 256, {1, 1, 1}, {3, 3, 3}, {1, 1, 1}, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}});
+    conv_params.push_back(
+        {3, 1, 1, 1, 32, {3, 3, 3}, {32, 32, 32}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}});
+    this->conv_params.push_back(
+        {3, 1, 1, 64, 3, {3, 3, 3}, {32, 32, 32}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}});
+    conv_params.push_back(
+        {3, 1, 1, 1, 1, {3, 3, 3}, {32, 32, 32}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}});
 
     for(auto& param : conv_params)
     {
@@ -247,6 +258,9 @@ TEST_F(TestGroupedConvNdFwd, GroupedConv2dFwdNHWGC)
     conv_params.push_back({2, 2, 128, 128, 256, {1, 1}, {7, 7}, {2, 2}, {1, 1}, {0, 0}, {0, 0}});
     conv_params.push_back({2, 2, 128, 128, 256, {3, 3}, {14, 14}, {1, 1}, {1, 1}, {1, 1}, {1, 1}});
     conv_params.push_back({2, 2, 128, 128, 256, {1, 1}, {3, 3}, {1, 1}, {1, 1}, {0, 0}, {0, 0}});
+    conv_params.push_back({2, 1, 1, 1, 32, {3, 3}, {32, 32}, {1, 1}, {1, 1}, {1, 1}, {1, 1}});
+    conv_params.push_back({2, 1, 1, 64, 3, {3, 3}, {32, 32}, {1, 1}, {1, 1}, {1, 1}, {1, 1}});
+    conv_params.push_back({2, 1, 1, 1, 1, {3, 3}, {32, 32}, {1, 1}, {1, 1}, {1, 1}, {1, 1}});
 
     for(auto& param : conv_params)
     {
@@ -255,7 +269,7 @@ TEST_F(TestGroupedConvNdFwd, GroupedConv2dFwdNHWGC)
         // fp16
         pass = ck::profiler::profile_grouped_conv_fwd_impl<2,
                                                            ck::tensor_layout::convolution::NHWGC,
-                                                           ck::tensor_layout::convolution::KYXGC,
+                                                           ck::tensor_layout::convolution::GKYXC,
                                                            ck::tensor_layout::convolution::NHWGK,
                                                            ck::half_t,
                                                            ck::half_t,

test/grouped_gemm/CMakeLists.txt

+if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
 list(APPEND gpu_list gfx908 gfx90a gfx940 gfx941 gfx942)
 set(target 0)
 foreach(gpu IN LISTS GPU_TARGETS)
@@ -12,3 +13,4 @@ foreach(gpu IN LISTS GPU_TARGETS)
    set(target 1)
  endif()
 endforeach()
+endif()

test/normalization/CMakeLists.txt

-add_custom_target(test_normalization)
-
-add_gtest_executable(test_layernorm2d_fp32 test_layernorm2d_fp32.cpp)
-add_gtest_executable(test_layernorm2d_fp16 test_layernorm2d_fp16.cpp)
-add_gtest_executable(test_groupnorm_fp16 test_groupnorm_fp16.cpp)
-add_gtest_executable(test_groupnorm_fp32 test_groupnorm_fp32.cpp)
-
-target_link_libraries(test_layernorm2d_fp32 PRIVATE utility device_normalization_instance)
-target_link_libraries(test_layernorm2d_fp16 PRIVATE utility device_normalization_instance)
-target_link_libraries(test_groupnorm_fp16 PRIVATE utility device_normalization_instance)
-target_link_libraries(test_groupnorm_fp32 PRIVATE utility device_normalization_instance)
-
-add_dependencies(test_normalization test_layernorm2d_fp32)
-add_dependencies(test_normalization test_layernorm2d_fp16)
-add_dependencies(test_normalization test_groupnorm_fp16)
-add_dependencies(test_normalization test_groupnorm_fp32)
+if(DTYPES MATCHES "fp16" OR DTYPES MATCHES "fp32" OR NOT DEFINED DTYPES)
+  add_custom_target(test_normalization)
+endif()
+if(DTYPES MATCHES "fp32" OR NOT DEFINED DTYPES)
+  add_gtest_executable(test_layernorm2d_fp32 test_layernorm2d_fp32.cpp)
+  add_gtest_executable(test_groupnorm_fp32 test_groupnorm_fp32.cpp)
+  target_link_libraries(test_layernorm2d_fp32 PRIVATE utility device_normalization_instance)
+  target_link_libraries(test_groupnorm_fp32 PRIVATE utility device_normalization_instance)
+  add_dependencies(test_normalization test_layernorm2d_fp32)
+  add_dependencies(test_normalization test_groupnorm_fp32)
+endif()
+if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
+  add_gtest_executable(test_layernorm2d_fp16 test_layernorm2d_fp16.cpp)
+  add_gtest_executable(test_groupnorm_fp16 test_groupnorm_fp16.cpp)
+  target_link_libraries(test_layernorm2d_fp16 PRIVATE utility device_normalization_instance)
+  target_link_libraries(test_groupnorm_fp16 PRIVATE utility device_normalization_instance)
+  add_dependencies(test_normalization test_layernorm2d_fp16)
+  add_dependencies(test_normalization test_groupnorm_fp16)
+endif()

test/pool_fwd/test_avg_pool2d_fwd.cpp

@@ -41,8 +41,12 @@ class TestAvgPool2dFwd : public ::testing::Test
     }
 };
 
+#ifdef __fp16__
 using KernelTypes =
     ::testing::Types<std::tuple<F16, F16, F32, I32>, std::tuple<F32, F32, F32, I32>>;
+#else
+using KernelTypes = ::testing::Types<std::tuple<F32, F32, F32, I32>>;
+#endif
 
 TYPED_TEST_SUITE(TestAvgPool2dFwd, KernelTypes);
 TYPED_TEST(TestAvgPool2dFwd, Test_Pool)

test/pool_fwd/test_avg_pool3d_fwd.cpp

@@ -40,10 +40,12 @@ class TestAvgPool3dFwd : public ::testing::Test
         }
     }
 };
-
+#ifdef __fp16__
 using KernelTypes =
     ::testing::Types<std::tuple<F16, F16, F32, I32>, std::tuple<F32, F32, F32, I32>>;
-
+#else
+using KernelTypes = ::testing::Types<std::tuple<F32, F32, F32, I32>>;
+#endif
 TYPED_TEST_SUITE(TestAvgPool3dFwd, KernelTypes);
 TYPED_TEST(TestAvgPool3dFwd, Test_Pool)
 {

test/pool_fwd/test_max_pool2d_fwd.cpp

@@ -59,10 +59,12 @@ class TestMaxPool2dFwd : public ::testing::Test
         }
     }
 };
-
+#ifdef __fp16__
 using KernelTypes =
     ::testing::Types<std::tuple<F16, F16, F16, I32>, std::tuple<F32, F32, F32, I32>>;
-
+#else
+using KernelTypes = ::testing::Types<std::tuple<F32, F32, F32, I32>>;
+#endif
 TYPED_TEST_SUITE(TestMaxPool2dFwd, KernelTypes);
 TYPED_TEST(TestMaxPool2dFwd, Test_Pool)
 {

test/pool_fwd/test_max_pool3d_fwd.cpp

@@ -60,8 +60,12 @@ class TestMaxPool3dFwd : public ::testing::Test
     }
 };
 
+#ifdef __fp16__
 using KernelTypes =
-    ::testing::Types<std::tuple<F16, F16, F16, I32>, std::tuple<F32, F32, F32, I32>>;
+    ::testing::Types<std::tuple<F16, F16, F32, I32>, std::tuple<F32, F32, F32, I32>>;
+#else
+using KernelTypes = ::testing::Types<std::tuple<F32, F32, F32, I32>>;
+#endif
 
 TYPED_TEST_SUITE(TestMaxPool3dFwd, KernelTypes);
 TYPED_TEST(TestMaxPool3dFwd, Test_Pool)

test/softmax/test_softmax_rank3.cpp

@@ -10,8 +10,9 @@
 
 template <ck::index_t N>
 using I = ck::Number<N>;
-
+#ifdef __fp16__
 using F16 = ck::half_t;
+#endif
 using F32 = float;
 
 template <typename Tuple>
@@ -22,7 +23,9 @@ class TestSoftmax : public ck::TestSoftmax<Tuple>
 // clang-format off
 using KernelTypes = ::testing::Types<
     //         InDataType, AccDataType, OutDataType, Rank
+#ifdef __fp16__
     std::tuple<       F16,         F32,         F16,    I<3>>,
+#endif
     std::tuple<       F32,         F32,         F32,    I<3>>
     >;
 // clang-format on

test/softmax/test_softmax_rank4.cpp

@@ -10,8 +10,9 @@
 
 template <ck::index_t N>
 using I = ck::Number<N>;
-
+#ifdef __fp16__
 using F16 = ck::half_t;
+#endif
 using F32 = float;
 
 template <typename Tuple>
@@ -22,7 +23,9 @@ class TestSoftmax : public ck::TestSoftmax<Tuple>
 // clang-format off
 using KernelTypes = ::testing::Types<
     //         InDataType, AccDataType, OutDataType, Rank
+#ifdef __fp16__
     std::tuple<       F16,         F32,         F16,    I<4>>,
+#endif
     std::tuple<       F32,         F32,         F32,    I<4>>
     >;
 // clang-format on