kernarg load latency optimization for mi300

example/53_gemv_splitk/CMakeLists.txt

@@ -6,6 +6,7 @@ foreach(gpu IN LISTS GPU_TARGETS)
    add_example_executable(example_gemv_splitk_fp16 gemv_splitk_fp16.cpp)
    add_dependencies(example_gemv_splitk
                 example_gemv_splitk_fp16)
+   set_source_files_properties(gemv_splitk_fp16.cpp PROPERTIES COMPILE_OPTIONS "-DKERNARG_PRELOAD;-Wno-gnu-line-marker;-gline-tables-only;-mllvm;--amdgpu-kernarg-preload-count=16")
    set(target 1)
   endif()
 endforeach()

example/54_tall_and_skinny_gemm_splitk/CMakeLists.txt

@@ -6,6 +6,7 @@ foreach(gpu IN LISTS GPU_TARGETS)
     add_example_executable(example_tall_and_skinny_gemm_splitk_fp16 tall_and_skinny_gemm_splitk_fp16.cpp)
     add_dependencies(example_tall_and_skinny_gemm_splitk
                 example_tall_and_skinny_gemm_splitk_fp16)
+                set_source_files_properties(tall_and_skinny_gemm_splitk_fp16.cpp PROPERTIES COMPILE_OPTIONS "-DKERNARG_PRELOAD;-Wno-gnu-line-marker;-gline-tables-only;-mllvm;--amdgpu-kernarg-preload-count=16")            
     set(target 1)
  endif()
  endforeach()
\ No newline at end of file

include/ck/host_utility/kernel_launch.hpp

@@ -9,8 +9,9 @@
 #include "ck/stream_config.hpp"
 #include "ck/host_utility/hip_check_error.hpp"
 
+#ifndef KERNARG_PRELOAD
 template <typename... Args, typename F>
-float launch_and_time_kernel(const StreamConfig& stream_config,
+float launch_and_time_kernel(const StreamConfig &stream_config,
                              F kernel,
                              dim3 grid_dim,
                              dim3 block_dim,
@@ -18,7 +19,7 @@ float launch_and_time_kernel(const StreamConfig& stream_config,
                              Args... args)
 {
 #if CK_TIME_KERNEL
-    if(stream_config.time_kernel_)
+    if (stream_config.time_kernel_)
     {
 #if DEBUG_LOG
         printf("%s: grid_dim {%d, %d, %d}, block_dim {%d, %d, %d} \n",
@@ -48,7 +49,7 @@ float launch_and_time_kernel(const StreamConfig& stream_config,
         hip_check_error(hipDeviceSynchronize());
         hip_check_error(hipEventRecord(start, stream_config.stream_id_));
 
-        for(int i = 0; i < nrepeat; ++i)
+        for (int i = 0; i < nrepeat; ++i)
         {
             kernel<<<grid_dim, block_dim, lds_byte, stream_config.stream_id_>>>(args...);
             hip_check_error(hipGetLastError());
@@ -78,8 +79,83 @@ float launch_and_time_kernel(const StreamConfig& stream_config,
 #endif
 }
 
+#else
+template <typename... Args, typename F>
+float launch_and_time_kernel(const StreamConfig &stream_config,
+                             F kernel,
+                             dim3 grid_dim,
+                             dim3 block_dim,
+                             std::size_t lds_byte,
+                             Args... args)
+{
+    // Args* args1;
+    // hipGetErrorString(hipMalloc(&args1, sizeof(Args)));
+    // hip_check_error(hipMemcpy(args1, &args, sizeof(Args), hipMemcpyHostToDevice));
+#if CK_TIME_KERNEL
+    if (stream_config.time_kernel_)
+    {
+#if DEBUG_LOG
+        printf("%s: grid_dim {%d, %d, %d}, block_dim {%d, %d, %d} \n",
+               __func__,
+               grid_dim.x,
+               grid_dim.y,
+               grid_dim.z,
+               block_dim.x,
+               block_dim.y,
+               block_dim.z);
+
+        printf("Warm up 1 time\n");
+#endif
+        //
+        // warm up
+        const int nrepeat = 1000;
+        for (auto i = 0; i < nrepeat; i++)
+            hipLaunchKernelGGL(kernel, grid_dim, block_dim, lds_byte, stream_config.stream_id_,
+                               args...);
+        hip_check_error(hipGetLastError());
+
+#if DEBUG_LOG
+        printf("Start running %d times...\n", nrepeat);
+#endif
+        hipEvent_t start, stop;
+        float total_time = 0;
+
+        hip_check_error(hipEventCreate(&start));
+        hip_check_error(hipEventCreate(&stop));
+
+        hip_check_error(hipDeviceSynchronize());
+
+        hip_check_error(hipEventRecord(start, stream_config.stream_id_));
+
+        for (int i = 0; i < nrepeat; ++i)
+            hipLaunchKernelGGL(kernel, grid_dim, block_dim, lds_byte, stream_config.stream_id_,
+                               args...);
+        // hip_check_error(hipGetLastError());
+
+        hip_check_error(hipEventRecord(stop, stream_config.stream_id_));
+        hip_check_error(hipEventSynchronize(stop));
+
+        hip_check_error(hipEventElapsedTime(&total_time, start, stop));
+        return total_time / nrepeat;
+    }
+    else
+    {
+        kernel<<<grid_dim, block_dim, lds_byte, stream_config.stream_id_>>>(
+            args...);
+        hip_check_error(hipGetLastError());
+
+        return 0;
+    }
+#else
+    kernel<<<grid_dim, block_dim, lds_byte, stream_config.stream_id_>>>(args...);
+    hip_check_error(hipGetLastError());
+
+    return 0;
+#endif
+}
+#endif
 template <typename... Args, typename F, typename PreProcessFunc>
-float launch_and_time_kernel_with_preprocess(const StreamConfig& stream_config,
+float launch_and_time_kernel_with_preprocess(const StreamConfig &stream_config,
                                              PreProcessFunc preprocess,
                                              F kernel,
                                              dim3 grid_dim,
@@ -88,7 +164,7 @@ float launch_and_time_kernel_with_preprocess(const StreamConfig& stream_config,
                                              Args... args)
 {
 #if CK_TIME_KERNEL
-    if(stream_config.time_kernel_)
+    if (stream_config.time_kernel_)
     {
 #if DEBUG_LOG
         printf("%s: grid_dim {%d, %d, %d}, block_dim {%d, %d, %d} \n",
@@ -119,7 +195,7 @@ float launch_and_time_kernel_with_preprocess(const StreamConfig& stream_config,
         hip_check_error(hipDeviceSynchronize());
         hip_check_error(hipEventRecord(start, stream_config.stream_id_));
 
-        for(int i = 0; i < nrepeat; ++i)
+        for (int i = 0; i < nrepeat; ++i)
         {
             preprocess();
             kernel<<<grid_dim, block_dim, lds_byte, stream_config.stream_id_>>>(args...);