Merge pull request #260 from ROCm/merge_from_public

Merge from public

example/ck_tile/12_smoothquant/example_smoothquant.cpp

@@ -35,7 +35,8 @@ auto create_args(int argc, char* argv[])
     ck_tile::ArgParser arg_parser;
     arg_parser.insert("m", "3328", "m dimension")
         .insert("n", "4096", "n dimension")
-        .insert("stride", "-1", "stride per row, if -1 then equal to n")
+        .insert("x_stride", "-1", "input stride per row, if -1 then equal to n")
+        .insert("y_stride", "-1", "output stride per row, if -1 then equal to n")
         .insert("e", "1e-5", "epsilon")
         .insert("v", "1", "cpu validation or not")
         .insert("prec", "fp16", "precision")
@@ -49,11 +50,14 @@ auto create_args(int argc, char* argv[])
 template <typename DataType>
 bool run(const ck_tile::ArgParser& arg_parser)
 {
-    ck_tile::index_t m      = arg_parser.get_int("m");
-    ck_tile::index_t n      = arg_parser.get_int("n");
-    ck_tile::index_t stride = arg_parser.get_int("stride");
-    if(stride < 0)
-        stride = n;
+    ck_tile::index_t m        = arg_parser.get_int("m");
+    ck_tile::index_t n        = arg_parser.get_int("n");
+    ck_tile::index_t x_stride = arg_parser.get_int("x_stride");
+    if(x_stride < 0)
+        x_stride = n;
+    ck_tile::index_t y_stride = arg_parser.get_int("y_stride");
+    if(y_stride < 0)
+        y_stride = n;
     std::string data_type = arg_parser.get_str("prec");
     int do_validation     = arg_parser.get_int("v");
     int warmup            = arg_parser.get_int("warmup");
@@ -68,14 +72,14 @@ bool run(const ck_tile::ArgParser& arg_parser)
     using ComputeDataType = float;
 
     // host verify
-    ck_tile::HostTensor<XDataType> x_host({m, n}, {stride, 1});
+    ck_tile::HostTensor<XDataType> x_host({m, n}, {x_stride, 1});
     ck_tile::HostTensor<XScaleDataType> xscale_host({n});
 
     ck_tile::HostTensor<YScaleDataType> yscale_host_ref({m}, {1});
     ck_tile::HostTensor<YScaleDataType> yscale_host_dev({m}, {1});
 
-    ck_tile::HostTensor<QYDataType> qy_host_ref({m, n}, {stride, 1});
-    ck_tile::HostTensor<QYDataType> qy_host_dev({m, n}, {stride, 1});
+    ck_tile::HostTensor<QYDataType> qy_host_ref({m, n}, {y_stride, 1});
+    ck_tile::HostTensor<QYDataType> qy_host_dev({m, n}, {y_stride, 1});
 
     ck_tile::FillUniformDistribution<XDataType>{-.5f, .5f}(x_host);
     ck_tile::FillUniformDistribution<XScaleDataType>{1e-3, .5f}(xscale_host);
@@ -116,7 +120,8 @@ bool run(const ck_tile::ArgParser& arg_parser)
                                       qy_buf.GetDeviceBuffer(),
                                       m,
                                       n,
-                                      stride};
+                                      x_stride,
+                                      y_stride};
 
     auto kargs = Kernel::MakeKargs(args);
 
@@ -133,7 +138,7 @@ bool run(const ck_tile::ArgParser& arg_parser)
     if(do_validation)
     {
         using YDataType = ComputeDataType;
-        ck_tile::HostTensor<ComputeDataType> y_host({m, n}, {stride, 1});
+        ck_tile::HostTensor<ComputeDataType> y_host({m, n}, {y_stride, 1});
         // smooth outlier
         {
             auto f = [&](auto n_) {
@@ -183,7 +188,7 @@ bool run(const ck_tile::ArgParser& arg_parser)
             qy_buf.FromDevice(qy_host_dev.data());
             auto [rtol, atol] = get_elimit<QYDataType>();
 
-            if(stride == n)
+            if(y_stride == n)
             {
                 pass = ck_tile::check_err(qy_host_dev,
                                           qy_host_ref,
@@ -195,10 +200,12 @@ bool run(const ck_tile::ArgParser& arg_parser)
             {
                 for(int i_r = 0; i_r < m; i_r++)
                 {
-                    std::vector<QYDataType> qy_host_dev_row(qy_host_dev.begin() + i_r * stride,
-                                                            qy_host_dev.begin() + i_r * stride + n);
-                    std::vector<QYDataType> qy_host_ref_row(qy_host_ref.begin() + i_r * stride,
-                                                            qy_host_ref.begin() + i_r * stride + n);
+                    std::vector<QYDataType> qy_host_dev_row(qy_host_dev.begin() + i_r * y_stride,
+                                                            qy_host_dev.begin() + i_r * y_stride +
+                                                                n);
+                    std::vector<QYDataType> qy_host_ref_row(qy_host_ref.begin() + i_r * y_stride,
+                                                            qy_host_ref.begin() + i_r * y_stride +
+                                                                n);
                     pass &= ck_tile::check_err(qy_host_dev_row,
                                                qy_host_ref_row,
                                                std::string("qy[") + std::to_string(i_r) +
@@ -210,8 +217,9 @@ bool run(const ck_tile::ArgParser& arg_parser)
         }
 
         std::cout << "[" << data_type << "]"
-                  << " m:" << m << ", n:" << n << ", stride:" << stride
-                  << ", valid:" << (pass ? "y" : "n") << std::flush << std::endl;
+                  << " m:" << m << ", n:" << n << ", x_stride:" << x_stride
+                  << ", y_stride:" << y_stride << ", valid:" << (pass ? "y" : "n") << std::flush
+                  << std::endl;
     }
 
     return pass;

example/ck_tile/12_smoothquant/smoothquant.cpp

@@ -33,7 +33,8 @@ auto create_args(int argc, char* argv[])
     ck_tile::ArgParser arg_parser;
     arg_parser.insert("m", "3328", "m dimension")
         .insert("n", "4096", "n dimension")
-        .insert("stride", "-1", "stride per row, if -1 then equal to n")
+        .insert("x_stride", "-1", "input stride per row, if -1 then equal to n")
+        .insert("y_stride", "-1", "output stride per row, if -1 then equal to n")
         .insert("v", "1", "cpu validation or not")
         .insert("kname", "1", "print kernel name or not")
         .insert("prec", "fp16", "precision")
@@ -47,18 +48,21 @@ auto create_args(int argc, char* argv[])
 template <typename DataType>
 bool run(const ck_tile::ArgParser& arg_parser)
 {
-    ck_tile::index_t m      = arg_parser.get_int("m");
-    ck_tile::index_t n      = arg_parser.get_int("n");
-    ck_tile::index_t stride = arg_parser.get_int("stride");
-    if(stride < 0)
-        stride = n;
+    ck_tile::index_t m        = arg_parser.get_int("m");
+    ck_tile::index_t n        = arg_parser.get_int("n");
+    ck_tile::index_t x_stride = arg_parser.get_int("x_stride");
+    if(x_stride < 0)
+        x_stride = n;
+    ck_tile::index_t y_stride = arg_parser.get_int("y_stride");
+    if(y_stride < 0)
+        y_stride = n;
     std::string data_type = arg_parser.get_str("prec");
     int kname             = arg_parser.get_int("kname");
     int do_validation     = arg_parser.get_int("v");
     int warmup            = arg_parser.get_int("warmup");
     int repeat            = arg_parser.get_int("repeat");
 
-    assert(stride >= n);
+    assert(x_stride >= n);
 
     using TypeConfig = SmoothquantTypeConfig<DataType>;
 
@@ -69,14 +73,14 @@ bool run(const ck_tile::ArgParser& arg_parser)
     using ComputeDataType = typename TypeConfig::ComputeDataType;
 
     // host verify
-    ck_tile::HostTensor<XDataType> x_host({m, n}, {stride, 1});
+    ck_tile::HostTensor<XDataType> x_host({m, n}, {x_stride, 1});
     ck_tile::HostTensor<XScaleDataType> xscale_host({n});
 
     ck_tile::HostTensor<YScaleDataType> yscale_host_ref({m}, {1});
     ck_tile::HostTensor<YScaleDataType> yscale_host_dev({m}, {1});
 
-    ck_tile::HostTensor<QYDataType> qy_host_ref({m, n}, {stride, 1});
-    ck_tile::HostTensor<QYDataType> qy_host_dev({m, n}, {stride, 1});
+    ck_tile::HostTensor<QYDataType> qy_host_ref({m, n}, {y_stride, 1});
+    ck_tile::HostTensor<QYDataType> qy_host_dev({m, n}, {y_stride, 1});
 
     ck_tile::FillUniformDistribution<XDataType>{-.5f, .5f}(x_host);
     ck_tile::FillUniformDistribution<XScaleDataType>{1e-3, .5f}(xscale_host);
@@ -90,7 +94,8 @@ bool run(const ck_tile::ArgParser& arg_parser)
     xscale_buf.ToDevice(xscale_host.data());
 
     std::cout << "[" << data_type << "]"
-              << " m:" << m << ", n:" << n << ", stride:" << stride << std::flush;
+              << " m:" << m << ", n:" << n << ", x_stride:" << x_stride << ", y_stride:" << y_stride
+              << std::flush;
 
     smoothquant_traits traits{data_type};
 
@@ -100,7 +105,8 @@ bool run(const ck_tile::ArgParser& arg_parser)
                           qy_buf.GetDeviceBuffer(),
                           m,
                           n,
-                          stride};
+                          x_stride,
+                          y_stride};
 
     float ave_time = smoothquant(
         traits, args, ck_tile::stream_config{nullptr, true, kname ? 1 : 0, warmup, repeat});
@@ -116,7 +122,7 @@ bool run(const ck_tile::ArgParser& arg_parser)
     if(do_validation)
     {
         using YDataType = ComputeDataType;
-        ck_tile::HostTensor<ComputeDataType> y_host({m, n}, {stride, 1});
+        ck_tile::HostTensor<ComputeDataType> y_host({m, n}, {y_stride, 1});
         // smooth outlier
         {
             auto f = [&](auto n_) {
@@ -166,7 +172,7 @@ bool run(const ck_tile::ArgParser& arg_parser)
             qy_buf.FromDevice(qy_host_dev.data());
             auto [rtol, atol] = get_elimit<QYDataType>();
 
-            if(stride == n)
+            if(y_stride == n)
             {
                 pass = ck_tile::check_err(qy_host_dev,
                                           qy_host_ref,
@@ -178,10 +184,12 @@ bool run(const ck_tile::ArgParser& arg_parser)
             {
                 for(int i_r = 0; i_r < m; i_r++)
                 {
-                    std::vector<QYDataType> qy_host_dev_row(qy_host_dev.begin() + i_r * stride,
-                                                            qy_host_dev.begin() + i_r * stride + n);
-                    std::vector<QYDataType> qy_host_ref_row(qy_host_ref.begin() + i_r * stride,
-                                                            qy_host_ref.begin() + i_r * stride + n);
+                    std::vector<QYDataType> qy_host_dev_row(qy_host_dev.begin() + i_r * y_stride,
+                                                            qy_host_dev.begin() + i_r * y_stride +
+                                                                n);
+                    std::vector<QYDataType> qy_host_ref_row(qy_host_ref.begin() + i_r * y_stride,
+                                                            qy_host_ref.begin() + i_r * y_stride +
+                                                                n);
                     pass &= ck_tile::check_err(qy_host_dev_row,
                                                qy_host_ref_row,
                                                std::string("qy[") + std::to_string(i_r) +

include/ck/config.h.in

@@ -111,6 +111,22 @@
 #cmakedefine CK_USE_WMMA @CK_USE_WMMA@
 #endif
 
+#ifndef CK_USE_GFX94
+#cmakedefine CK_USE_GFX94 @CK_USE_GFX94@
+#endif
+
+#ifndef DCK_USE_OCP_FP8
+#cmakedefine DCK_USE_OCP_FP8 @DCK_USE_OCP_FP8@
+#endif
+
+#ifndef CK_USE_FNUZ_FP8
+#cmakedefine CK_USE_FNUZ_FP8 @CK_USE_FNUZ_FP8@
+#endif
+
+#ifndef CK_USE_FP8_ON_UNSUPPORTED_ARCH
+#cmakedefine CK_USE_FP8_ON_UNSUPPORTED_ARCH @CK_USE_FP8_ON_UNSUPPORTED_ARCH@
+#endif
+
 // clang-format on
 
 #endif // CK_CONFIG_H_IN

include/ck/tensor_operation/gpu/device/device_base.hpp

@@ -5,6 +5,8 @@
 
 #include <string>
 #include <sstream>
+#include <regex>
+#include <optional>
 
 #include "ck/stream_config.hpp"
 
@@ -12,6 +14,34 @@ namespace ck {
 namespace tensor_operation {
 namespace device {
 
+#define GET_OBJECT_NAME_IMLP                                                  \
+    std::optional<std::string> GetObjectName() const override                 \
+    {                                                                         \
+        std::string str = __PRETTY_FUNCTION__;                                \
+        static std::regex obj_name_expr{"<std::string> (.*)::GetObjectName"}; \
+        std::smatch match;                                                    \
+        if(!std::regex_search(str, match, obj_name_expr))                     \
+        {                                                                     \
+            return str;                                                       \
+        }                                                                     \
+        return std::string(match[1]) + ';';                                   \
+    }
+
+#define GET_TEMPLATE_INFO_IMPL                                  \
+    std::optional<std::string> GetTemplateInfo() const override \
+    {                                                           \
+        std::string str = __PRETTY_FUNCTION__;                  \
+        static std::regex template_expr{"\\[(.*)\\]"};          \
+        std::smatch match;                                      \
+        if(!std::regex_search(str, match, template_expr))       \
+        {                                                       \
+            return std::nullopt;                                \
+        }                                                       \
+        return std::string(match[1]);                           \
+    }
+
+#define REGISTER_EXTRA_PRINTING_METHODS GET_OBJECT_NAME_IMLP GET_TEMPLATE_INFO_IMPL
+
 struct BaseArgument
 {
     BaseArgument()                    = default;
@@ -48,6 +78,10 @@ struct BaseOperator
 
     virtual std::string GetTypeIdName() const { return typeid(*this).name(); }
 
+    virtual std::optional<std::string> GetObjectName() const { return std::nullopt; }
+
+    virtual std::optional<std::string> GetTemplateInfo() const { return std::nullopt; }
+
     virtual std::string GetTypeIdHashCode() const
     {
         std::ostringstream oss;

include/ck/tensor_operation/gpu/device/device_batched_gemm_multi_d.hpp

@@ -89,7 +89,8 @@ struct DeviceBatchedGemmV2MultiD : public BaseOperator
                         index_t BatchStrideE,
                         AElementwiseOperation a_element_op,
                         BElementwiseOperation b_element_op,
-                        CDEElementwiseOperation cde_element_op) = 0;
+                        CDEElementwiseOperation cde_element_op,
+                        index_t KBatch) = 0;
 
     virtual std::unique_ptr<BaseInvoker> MakeInvokerPointer() = 0;
 };

include/ck/tensor_operation/gpu/device/impl/device_batched_gemm_multiple_d_xdl_cshuffle_v3.hpp

@@ -41,12 +41,15 @@ __global__ void
     __shared__ char p_shared[GridwiseGemm::GetSharedMemoryNumberOfByte()];
 
     const index_t g_idx = blockIdx.z % karg.Batch;
+    const index_t k_idx = blockIdx.z / karg.Batch;
 
     const auto a_batch_offset  = karg.compute_ptr_offset_of_batch.GetAPtrOffset(g_idx);
     const auto b_batch_offset  = karg.compute_ptr_offset_of_batch.GetBPtrOffset(g_idx);
     const auto ds_batch_offset = karg.compute_ptr_offset_of_batch.GetDsPtrOffset(g_idx);
     const auto c_batch_offset  = karg.compute_ptr_offset_of_batch.GetCPtrOffset(g_idx);
 
+    auto splitk_batch_offset = typename GridwiseGemm::SplitKBatchOffset(karg, k_idx);
+
     // populate pointer, desc for Ds
     static_for<0, GridwiseGemm::NumDTensor, 1>{}([&](auto i) {
         // D pointer
@@ -54,8 +57,8 @@ __global__ void
     });
 
     GridwiseGemm::template Run<HasMainKBlockLoop, CGlobalMemoryDataOperation, TailNum>(
-        karg.p_a_grid + a_batch_offset,
-        karg.p_b_grid + b_batch_offset,
+        karg.p_a_grid + a_batch_offset + splitk_batch_offset.a_k_split_offset,
+        karg.p_b_grid + b_batch_offset + splitk_batch_offset.b_k_split_offset,
         karg.p_ds_grid,
         karg.p_c_grid + c_batch_offset,
         p_shared,
@@ -87,12 +90,15 @@ __global__ void
     __shared__ char p_shared_1[GridwiseGemm::GetSharedMemoryNumberOfByte()];
 
     const index_t g_idx = blockIdx.z % karg.Batch;
+    const index_t k_idx = blockIdx.z / karg.Batch;
 
     const auto a_batch_offset  = karg.compute_ptr_offset_of_batch.GetAPtrOffset(g_idx);
     const auto b_batch_offset  = karg.compute_ptr_offset_of_batch.GetBPtrOffset(g_idx);
     const auto ds_batch_offset = karg.compute_ptr_offset_of_batch.GetDsPtrOffset(g_idx);
     const auto c_batch_offset  = karg.compute_ptr_offset_of_batch.GetCPtrOffset(g_idx);
 
+    auto splitk_batch_offset = typename GridwiseGemm::SplitKBatchOffset(karg, k_idx);
+
     // populate pointer, desc for Ds
     static_for<0, GridwiseGemm::NumDTensor, 1>{}([&](auto i) {
         // D pointer
@@ -100,8 +106,8 @@ __global__ void
     });
 
     GridwiseGemm::template Run_2Lds<HasMainKBlockLoop, CGlobalMemoryDataOperation, TailNum>(
-        karg.p_a_grid + a_batch_offset,
-        karg.p_b_grid + b_batch_offset,
+        karg.p_a_grid + a_batch_offset + splitk_batch_offset.a_k_split_offset,
+        karg.p_b_grid + b_batch_offset + splitk_batch_offset.b_k_split_offset,
         karg.p_ds_grid,
         karg.p_c_grid + c_batch_offset,
         p_shared_0,
@@ -303,7 +309,8 @@ struct DeviceBatchedGemmMultiD_Xdl_CShuffle_V3
                  index_t Batch_,
                  AElementwiseOperation a_element_op_,
                  BElementwiseOperation b_element_op_,
-                 CElementwiseOperation c_element_op_)
+                 CElementwiseOperation c_element_op_,
+                 index_t KBatch_)
             : GridwiseGemm::Argument{p_a_grid_,
                                      p_b_grid_,
                                      p_ds_grid_,
@@ -315,7 +322,7 @@ struct DeviceBatchedGemmMultiD_Xdl_CShuffle_V3
                                      StrideB_,
                                      StrideDs_,
                                      StrideE_,
-                                     1,
+                                     KBatch_,
                                      a_element_op_,
                                      b_element_op_,
                                      c_element_op_},
@@ -336,13 +343,14 @@ struct DeviceBatchedGemmMultiD_Xdl_CShuffle_V3
                 arg.Print();
             }
 
-            if(!GridwiseGemm::CheckValidity(arg) || arg.KBatch > 1)
+            if(!GridwiseGemm::CheckValidity(arg))
             {
                 throw std::runtime_error("wrong! GridwiseGemm has invalid setting");
             }
 
             index_t gdx, gdy, gdz;
-            std::tie(gdx, gdy, gdz) = GridwiseGemm::CalculateGridSize(arg.M, arg.N, arg.Batch);
+            std::tie(gdx, gdy, gdz) =
+                GridwiseGemm::CalculateGridSize(arg.M, arg.N, arg.Batch * arg.KBatch);
 
             float ave_time = 0;
 
@@ -387,10 +395,11 @@ struct DeviceBatchedGemmMultiD_Xdl_CShuffle_V3
                         rotating_mem.Next();
                         // clear c mem
                         if(arg_.KBatch > 1)
-                            hipGetErrorString(hipMemsetAsync(arg_.p_c_grid,
-                                                             0,
-                                                             arg_.M * arg_.N * sizeof(CDataType),
-                                                             stream_config.stream_id_));
+                            hipGetErrorString(
+                                hipMemsetAsync(arg_.p_c_grid,
+                                               0,
+                                               arg.Batch * arg_.M * arg_.N * sizeof(CDataType),
+                                               stream_config.stream_id_));
                     };
 
                     ave_time = ck::utility::launch_and_time_kernel_with_preprocess<false>(
@@ -889,7 +898,8 @@ struct DeviceBatchedGemmMultiD_Xdl_CShuffle_V3
                              index_t BatchStrideE,
                              AElementwiseOperation a_element_op,
                              BElementwiseOperation b_element_op,
-                             CElementwiseOperation c_element_op)
+                             CElementwiseOperation c_element_op,
+                             index_t KBatch = 1)
     {
         return Argument{static_cast<const ADataType*>(p_a),
                         static_cast<const BDataType*>(p_b),
@@ -909,7 +919,8 @@ struct DeviceBatchedGemmMultiD_Xdl_CShuffle_V3
                         Batch,
                         a_element_op,
                         b_element_op,
-                        c_element_op};
+                        c_element_op,
+                        KBatch};
     }
 
     static auto MakeInvoker() { return Invoker{}; }
@@ -934,7 +945,8 @@ struct DeviceBatchedGemmMultiD_Xdl_CShuffle_V3
                         index_t BatchStrideE,
                         AElementwiseOperation a_element_op,
                         BElementwiseOperation b_element_op,
-                        CElementwiseOperation c_element_op) override
+                        CElementwiseOperation c_element_op,
+                        index_t KBatch = 1) override
     {
         return std::make_unique<Argument>(static_cast<const ADataType*>(p_a),
                                           static_cast<const BDataType*>(p_b),
@@ -954,7 +966,8 @@ struct DeviceBatchedGemmMultiD_Xdl_CShuffle_V3
                                           Batch,
                                           a_element_op,
                                           b_element_op,
-                                          c_element_op);
+                                          c_element_op,
+                                          KBatch);
     }
 
     // polymorphic

include/ck/tensor_operation/gpu/device/impl/device_gemm_xdl_cshuffle_v3.hpp

@@ -729,6 +729,7 @@ struct DeviceGemm_Xdl_CShuffleV3 : public DeviceGemmV2<ALayout,
 
         return str.str();
     }
+    REGISTER_EXTRA_PRINTING_METHODS
 };
 
 } // namespace device

include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d.hpp

@@ -41,7 +41,7 @@ __global__ void
 #if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx9__))
     __shared__ char p_shared[GridwiseGemm::GetSharedMemoryNumberOfByte()];
 
-    auto splitk_batch_offset = typename GridwiseGemm::SplitKBatchOffset(karg);
+    auto splitk_batch_offset = typename GridwiseGemm::SplitKBatchOffset(karg, blockIdx.z);
 
     GridwiseGemm::template Run<HasMainKBlockLoop, CGlobalMemoryDataOperation, TailNum>(
         karg.p_a_grid + splitk_batch_offset.a_k_split_offset,
@@ -76,7 +76,7 @@ __global__ void
     __shared__ char p_shared_0[GridwiseGemm::GetSharedMemoryNumberOfByte()];
     __shared__ char p_shared_1[GridwiseGemm::GetSharedMemoryNumberOfByte()];
 
-    auto splitk_batch_offset = typename GridwiseGemm::SplitKBatchOffset(karg);
+    auto splitk_batch_offset = typename GridwiseGemm::SplitKBatchOffset(karg, blockIdx.z);
 
     GridwiseGemm::template Run_2Lds<HasMainKBlockLoop, CGlobalMemoryDataOperation, TailNum>(
         karg.p_a_grid + splitk_batch_offset.a_k_split_offset,
@@ -639,27 +639,27 @@ struct GridwiseGemmMultiD_xdl_cshuffle_v3
 
     struct SplitKBatchOffset
     {
-        __device__ SplitKBatchOffset(Argument& karg)
+        __device__ SplitKBatchOffset(Argument& karg, index_t k_id)
         {
             if constexpr(is_same_v<tensor_layout::gemm::RowMajor, ALayout>)
             {
-                a_k_split_offset = blockIdx.z * karg.KRead;
+                a_k_split_offset = k_id * karg.KRead;
             }
             else if constexpr(is_same_v<tensor_layout::gemm::ColumnMajor, ALayout>)
             {
-                a_k_split_offset = blockIdx.z * karg.KRead * karg.StrideA;
+                a_k_split_offset = k_id * karg.KRead * karg.StrideA;
             }
 
             if constexpr(is_same_v<tensor_layout::gemm::RowMajor, BLayout>)
             {
-                b_k_split_offset = blockIdx.z * karg.KRead * karg.StrideB;
+                b_k_split_offset = k_id * karg.KRead * karg.StrideB;
             }
             else if constexpr(is_same_v<tensor_layout::gemm::ColumnMajor, BLayout>)
             {
-                b_k_split_offset = blockIdx.z * karg.KRead;
+                b_k_split_offset = k_id * karg.KRead;
             }
 
-            if(blockIdx.z < static_cast<uint32_t>(karg.KBatch - 1))
+            if(k_id < karg.KBatch - 1)
             {
                 karg.K = karg.KRead;
             }

include/ck/utility/amd_ck_fp8.hpp

@@ -18,6 +18,20 @@
 #define CK_USE_OCP_FP8 0
 #endif
 
+namespace {
+// https://en.cppreference.com/w/cpp/types/conditional
+template <bool B, class T, class F>
+struct conditional
+{
+    using type = T;
+};
+template <class T, class F>
+struct conditional<false, T, F>
+{
+    using type = F;
+};
+} // namespace
+
 namespace ck {
 
 using f8_fnuz_t  = _BitInt(8);
@@ -191,11 +205,10 @@ __host__ __device__ static inline T cast_from_f8(fp8_storage_t x)
         }
     }
 
-    typename __hip_internal::conditional<
+    typename conditional<
         sizeof(T) == 2,
         unsigned short int,
-        typename __hip_internal::conditional<sizeof(T) == 4, unsigned int, unsigned long long>::
-            type>::type retval;
+        typename conditional<sizeof(T) == 4, unsigned int, unsigned long long>::type>::type retval;
 
     if constexpr(we == 5 && is_half && !is_fnuz)
     {
@@ -538,11 +551,10 @@ __host__ __device__ static inline fp8_storage_t cast_to_f8(T _x, unsigned int rn
 
     constexpr int mfmt = (sizeof(T) == 8) ? 52 : ((sizeof(T) == 4) ? 23 : 10);
 
-    using T_bitwise = typename __hip_internal::conditional<
+    using T_bitwise = typename conditional<
         sizeof(T) == 2,
         unsigned short int,
-        typename __hip_internal::conditional<sizeof(T) == 4, unsigned int, unsigned long long>::
-            type>::type;
+        typename conditional<sizeof(T) == 4, unsigned int, unsigned long long>::type>::type;
     T_bitwise x_bitwise = bit_cast<T_bitwise>(_x);
 
     unsigned long long x{x_bitwise};

include/ck/utility/math_v2.hpp

@@ -611,7 +611,7 @@ inline __device__ int8_t neg<int8_t>(int8_t x)
 template <>
 inline __device__ half_t neg<half_t>(half_t x)
 {
-    return __hneg(x);
+    return __hneg(static_cast<__half>(x));
 };
 
 template <typename T>

include/ck_tile/README.md

@@ -45,5 +45,8 @@ our implementation of different device operators.
 **[ops/epilogue]**  
 epilogue part of our kernel. We may extend this epilogue part to let users to build their own cutomized epilogues.
 
+**[ref]**  
+reference implementation of cpu or gpu. This folder is supposed to include a specific header on demand.
+
 ## examples
 currently we put all ck_tile related example under [/example/ck_tile](/example/ck_tile/) folder. Please check each example's subfolder.

include/ck_tile/core.hpp

@@ -54,6 +54,7 @@
 #include "ck_tile/core/tensor/tile_window_linear.hpp"
 #include "ck_tile/core/tensor/tile_window_utils.hpp"
 #include "ck_tile/core/tensor/update_tile.hpp"
+#include "ck_tile/core/utility/amd_address_space.hpp"
 #include "ck_tile/core/utility/bit_cast.hpp"
 #include "ck_tile/core/utility/functional.hpp"
 #include "ck_tile/core/utility/functional_with_tuple.hpp"

include/ck_tile/ops/flatmm.hpp

@@ -5,6 +5,7 @@
 
 #include "ck_tile/ops/flatmm/block/flatmm_32x512x128_1x4x1_16x16x32.hpp"
 #include "ck_tile/ops/flatmm/block/flatmm_sn_32x128x512_1x4x1_16x16x32.hpp"
+#include "ck_tile/ops/flatmm/block/flatmm_sn_32x128x512_1x4x1_16x16x32_itl.hpp"
 #include "ck_tile/ops/flatmm/block/flatmm_uk_config.hpp"
 #include "ck_tile/ops/common/generic_2d_block_shape.hpp"
 #include "ck_tile/ops/common/tensor_layout.hpp"

include/ck_tile/ops/flatmm/block/flatmm_sn_32x128x512_1x4x1_16x16x32_itl.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2024, Advanced Micro Devices, Inc. All rights reserved.
+
+#pragma once
+
+#include "ck_tile/core.hpp"
+#include "ck_tile/ops/gemm/warp/warp_gemm.hpp"
+#include "ck_tile/ops/flatmm/block/flatmm_uk_config.hpp"
+#include "ck_tile/ops/flatmm/block/flatmm_sn_32x128x512_1x4x1_16x16x32.hpp"
+
+namespace ck_tile {
+
+// "S"tream update output along "N"
+// A in smem, B load from global
+// require 4 wave, occupancy=1c
+
+struct FlatmmSn_32x128x512_1x4x1_16x16x32_BF16_itl : public FlatmmSn_32x128x512_1x4x1_16x16x32_Base
+{
+    using BDataType = bf16_t;
+    using ODataType = bf16_t;
+
+    // TODO: need paired with tile_window_linear!
+    // TODO: need call init_raw() before call this function!
+    // template <typename AWindow, typename BWindow, typename OWindow, typename ScaleTensor>
+    template <typename BRes,
+              typename BCoords,
+              typename ORes,
+              typename OCoords,
+              typename OFlags,
+              typename ScaleTensor>
+    CK_TILE_DEVICE auto
+    operator()(const BRes& res_b,
+               const BCoords& cached_coords_b,
+               const ORes& res_o,
+               const OCoords& cached_coords_o,
+               const OFlags& o_flags, // this should be in sgpr
+               CK_TILE_LDS_ADDR void* smem,
+               index_t n, // loop along n dim
+               const ScaleTensor& scale_,
+               index_t tile_offset_b, // stride b is fixed to blockKr * blockW, but still can adjust
+               index_t tile_offset_o)
+    {
+        static_assert(BCoords::size() == 8); // 8
+        static_assert(OCoords::size() == 8);
+
+        const index_t tile_stride_b_bytes = tile_offset_b * sizeof(BDataType);
+        const index_t tile_stride_o_bytes = tile_offset_o * sizeof(ODataType);
+
+        static_assert(ScaleTensor::size() == 2);
+        float s0 = scale_[number<0>{}];
+        float s1 = scale_[number<1>{}];
+
+        // index_t loop_cnt = n / Block_N;
+
+        register float v_c0 asm("v64");
+        register float v_c1 asm("v65");
+        register float v_c2 asm("v66");
+        register float v_c3 asm("v67");
+        register float v_c4 asm("v68");
+        register float v_c5 asm("v69");
+        register float v_c6 asm("v70");
+        register float v_c7 asm("v71");
+        register float v_c8 asm("v72");
+        register float v_c9 asm("v73");
+        register float v_c10 asm("v74");
+        register float v_c11 asm("v75");
+        register float v_c12 asm("v76");
+        register float v_c13 asm("v77");
+        register float v_c14 asm("v78");
+        register float v_c15 asm("v79");
+        register float v_c16 asm("v80");
+        register float v_c17 asm("v81");
+        register float v_c18 asm("v82");
+        register float v_c19 asm("v83");
+        register float v_c20 asm("v84");
+        register float v_c21 asm("v85");
+        register float v_c22 asm("v86");
+        register float v_c23 asm("v87");
+        register float v_c24 asm("v88");
+        register float v_c25 asm("v89");
+        register float v_c26 asm("v90");
+        register float v_c27 asm("v91");
+        register float v_c28 asm("v92");
+        register float v_c29 asm("v93");
+        register float v_c30 asm("v94");
+        register float v_c31 asm("v95");
+        int32_t nan_hi = 0x7fff0000;
+        int32_t nan_lo = 0x00007fff;
+
+        // in smem, the layout is  M0(2)*K0(128)*M1(16)*K1(4)
+        // every threads need 8xK in contiguous register
+        // ... and every wave need the same data
+        int lane_id  = threadIdx.x % 64;
+        int sld_y_os = (lane_id % 16) * 4 + (lane_id / 16) * 128;
+        sld_y_os *= 2;
+
+        //                    y     y     p     p      p      y
+        // reg before shfl  M0(2)*N0(2)*Nl(4)*Nw(4)*Mw(16)*Nv(4)
+        // but order is N0*M0*Nv
+        // in LDS we need store as
+        //          M0(2)* N0(2) *  Nl(4) * Nw(4) * (Mw(16)*Nv(4) + 4)
+        //             y    y       wave-id  lid/16  lid%16   v
+        // sst(v3) = (v0/16*34 + v0%16 * 2 + wid*136) * 4
+        int sfl_sst = (threadIdx.x % 16 * 4) + (threadIdx.x / 16) * (64 + 4);
+        sfl_sst *= 2;
+
+        // from LDS we need load as
+        //          M0(2)*    N0(2) *  Nl(4) * Nw(4) * (Mw(16)         *  Nv(4) + 4)
+        //        ( 2 issue)    (rem 32-lane)        (4 wave*4issue)   2lane*1ussue(pk2)
+        // sld(v4) = v0/2 *34*4  + v0 % 2 *4 + wid*2 *4
+        int sfl_sld = (lane_id % 2) * 2 + (lane_id / 2) * (64 + 4) + (threadIdx.x / 64) * 4;
+        sfl_sld *= 2;
+
+        // B nr->kr
+        // clang-format off
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Winline-asm"
+        asm volatile(
+#define CK_TILE_FLATMM_UK_MFMA CK_TILE_FLATMM_UK_MFMA_BF16
+#include "uk/flatmm_sn_uk_gfx9_32x128x512_1x4x1_16x16x16_itl.inc"
+#undef CK_TILE_FLATMM_UK_MFMA
+            :[smem_]"+r"(smem),
+            // [s_loop_cnt]"+s"(loop_cnt),
+            [s_loop_cnt]"+s"(n),
+                [c0]"+v" (v_c0),
+                [c1]"+v" (v_c1),
+                [c2]"+v" (v_c2),
+                [c3]"+v" (v_c3),
+                [c4]"+v" (v_c4),
+                [c5]"+v" (v_c5),
+                [c6]"+v" (v_c6),
+                [c7]"+v" (v_c7),
+                [c8]"+v" (v_c8),
+                [c9]"+v" (v_c9),
+                [c10]"+v"(v_c10),
+                [c11]"+v"(v_c11),
+                [c12]"+v"(v_c12),
+                [c13]"+v"(v_c13),
+                [c14]"+v"(v_c14),
+                [c15]"+v"(v_c15),
+                [c16]"+v"(v_c16),
+                [c17]"+v"(v_c17),
+                [c18]"+v"(v_c18),
+                [c19]"+v"(v_c19),
+                [c20]"+v"(v_c20),
+                [c21]"+v"(v_c21),
+                [c22]"+v"(v_c22),
+                [c23]"+v"(v_c23),
+                [c24]"+v"(v_c24),
+                [c25]"+v"(v_c25),
+                [c26]"+v"(v_c26),
+                [c27]"+v"(v_c27),
+                [c28]"+v"(v_c28),
+                [c29]"+v"(v_c29),
+                [c30]"+v"(v_c30),
+                [c31]"+v"(v_c31)
+            :
+            [sld_a_base]"n"(0),
+            [shfl_base]"n"(0),
+            [v_sld_y_os]"v"(sld_y_os),
+            [v_sfl_sld]"v"(sfl_sld),
+            [v_sfl_sst]"v"(sfl_sst),
+            [s_res_o0]"s"(res_o[0]),
+                [s_res_o1]"s"(res_o[1]),
+                //[s_res_o2]"s"(res_o[2]),
+                //[s_res_o3]"s"(res_o[3]),
+                [s_res_b0]"s"(res_b[0]),
+                [s_res_b1]"s"(res_b[1]),
+                [s_res_b2]"s"(res_b[2]),
+                [s_res_b3]"s"(res_b[3]),
+                [v_os_o0]"v"(static_cast<index_t>(cached_coords_o[number<0>{}] * sizeof(ODataType))),
+                [v_os_o1]"v"(static_cast<index_t>(cached_coords_o[number<1>{}] * sizeof(ODataType))),
+                [v_os_o2]"v"(static_cast<index_t>(cached_coords_o[number<2>{}] * sizeof(ODataType))),
+                [v_os_o3]"v"(static_cast<index_t>(cached_coords_o[number<3>{}] * sizeof(ODataType))),
+                [v_os_o4]"v"(static_cast<index_t>(cached_coords_o[number<4>{}] * sizeof(ODataType))),
+                [v_os_o5]"v"(static_cast<index_t>(cached_coords_o[number<5>{}] * sizeof(ODataType))),
+                [v_os_o6]"v"(static_cast<index_t>(cached_coords_o[number<6>{}] * sizeof(ODataType))),
+                [v_os_o7]"v"(static_cast<index_t>(cached_coords_o[number<7>{}] * sizeof(ODataType))),
+                [v_os_b0]"v"(static_cast<index_t>(cached_coords_b[number<0>{}] * sizeof(BDataType))),
+                [v_os_b1]"v"(static_cast<index_t>(cached_coords_b[number<1>{}] * sizeof(BDataType))),
+                [v_os_b2]"v"(static_cast<index_t>(cached_coords_b[number<2>{}] * sizeof(BDataType))),
+                [v_os_b3]"v"(static_cast<index_t>(cached_coords_b[number<3>{}] * sizeof(BDataType))),
+                [v_os_b4]"v"(static_cast<index_t>(cached_coords_b[number<4>{}] * sizeof(BDataType))),
+                [v_os_b5]"v"(static_cast<index_t>(cached_coords_b[number<5>{}] * sizeof(BDataType))),
+                [v_os_b6]"v"(static_cast<index_t>(cached_coords_b[number<6>{}] * sizeof(BDataType))),
+                [v_os_b7]"v"(static_cast<index_t>(cached_coords_b[number<7>{}] * sizeof(BDataType))),
+
+                [s_tile_os_o]"s"(tile_stride_o_bytes),
+                [s_tile_os_b]"s"(tile_stride_b_bytes),
+                [scale_0]"v"(s0),
+                [scale_1]"v"(s1),
+                [v_nan_lo]"v"(nan_lo),
+                [v_nan_hi]"v"(nan_hi),
+                [s_execflag_0]"s"(o_flags[number<0>{}]),
+                [s_execflag_1]"s"(o_flags[number<1>{}]),
+                [s_execflag_2]"s"(o_flags[number<2>{}]),
+                [s_execflag_3]"s"(o_flags[number<3>{}]),
+                [s_execflag_4]"s"(o_flags[number<4>{}]),
+                [s_execflag_5]"s"(o_flags[number<5>{}]),
+                [s_execflag_6]"s"(o_flags[number<6>{}]),
+                [s_execflag_7]"s"(o_flags[number<7>{}])
+            :
+          "memory", "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8", "a9",
+          "a10", "a11", "a12", "a13", "a14", "a15", "a16", "a17", "a18", "a19",
+          "a20", "a21", "a22", "a23", "a24", "a25", "a26", "a27", "a28", "a29",
+          "a30", "a31", "a32", "a33", "a34", "a35", "a36", "a37", "a38", "a39",
+          "a40", "a41", "a42", "a43", "a44", "a45", "a46", "a47", "a48", "a49",
+          "a50", "a51", "a52", "a53", "a54", "a55", "a56", "a57", "a58", "a59",
+          "a60", "a61", "a62", "a63", "a64", "a65", "a66", "a67", "a68", "a69",
+          "a70", "a71", "a72", "a73", "a74", "a75", "a76", "a77", "a78", "a79",
+          "a80", "a81", "a82", "a83", "a84", "a85", "a86", "a87", "a88", "a89",
+          "a90", "a91", "a92", "a93", "a94", "a95", "a96", "a97", "a98", "a99",
+          "a100", "a101", "a102", "a103", "a104", "a105", "a106", "a107",
+          "a108", "a109", "a110", "a111", "a112", "a113", "a114", "a115",
+          "a116", "a117", "a118", "a119", "a120", "a121", "a122", "a123",
+          "a124", "a125", "a126", "a127", "a128", "a129", "a130", "a131",
+          "a132", "a133", "a134", "a135", "a136", "a137", "a138", "a139",
+          "a140", "a141", "a142", "a143", "a144", "a145", "a146", "a147",
+          "a148", "a149", "a150", "a151", "a152", "a153", "a154", "a155",
+          "a156", "a157", "a158", "a159", "a160", "a161", "a162", "a163",
+          "a164", "a165", "a166", "a167", "a168", "a169", "a170", "a171",
+          "a172", "a173", "a174", "a175", "a176", "a177", "a178", "a179",
+          "a180", "a181", "a182", "a183", "a184", "a185", "a186", "a187",
+          "a188", "a189", "a190", "a191", "a192", "a193", "a194", "a195",
+          "a196", "a197", "a198", "a199", "a200", "a201", "a202", "a203",
+          "a204", "a205", "a206", "a207", "a208", "a209", "a210", "a211",
+          "a212", "a213", "a214", "a215", "a216", "a217", "a218", "a219",
+          "a220", "a221", "a222", "a223", "a224", "a225", "a226", "a227",
+          "a228", "a229", "a230", "a231", "a232", "a233", "a234", "a235",
+          "a236", "a237", "a238", "a239", "a240", "a241", "a242", "a243",
+          "a244", "a245", "a246", "a247", "a248", "a249", "a250", "a251",
+          "a252", "a253", "a254", "a255", 
+          "s8", "s9", "s12", "s13", "s14", "s15", "s38", "s39", "s52", "s86",
+          "s36", "s37","s59","s80",
+          "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",
+          "v50", "v54", "v55",
+          "v64","v65","v66","v67","v68","v69","v70","v71",
+          "v72","v73","v74","v75","v76","v77","v78","v79",
+          "v80","v81","v82","v83","v84","v85","v86","v87",
+          "v88","v89","v90","v91","v92","v93","v94","v95",
+          "v128", "v129", "v130", "v131",
+          "v132", "v133", "v134", "v135", "v136", "v137", "v138", "v139",
+          "v140", "v141", "v142", "v143", "v144", "v145", "v146", "v147",
+          "v148", "v149", "v150", "v151", "v152", "v153", "v154", "v155",
+          "v156", "v157", "v158", "v159", "v160", "v161", "v162", "v163",
+          "v164", "v165", "v166", "v167", "v168", "v169", "v170", "v171",
+          "v172", "v173", "v174", "v175", "v176", "v177", "v178", "v179",
+          "v180", "v181", "v182", "v183", "v184", "v185", "v186", "v187",
+          "v188", "v189", "v190", "v191", "v192", "v193", "v194", "v195",
+          "v196", "v197", "v198", "v199", "v200", "v201", "v202", "v203",
+          "v204", "v205", "v206", "v207", "v208", "v209", "v210", "v211",
+          "v212", "v213", "v214", "v215", "v216", "v217", "v218", "v219",
+          "v220", "v221", "v222", "v223", "v224", "v225", "v226", "v227",
+          "v228", "v229", "v230", "v231", "v232", "v233", "v234", "v235",
+          "v236", "v237", "v238", "v239", "v240", "v241", "v242", "v243",
+          "v244", "v245", "v246", "v247", "v248", "v249", "v250", "v251",
+          "v252", "v253", "v254", "v255"
+        );
+#pragma clang diagnostic pop
+        // clang-format on
+    }
+};
+
+struct FlatmmSn_32x128x512_1x4x1_16x16x32_FP16_itl : public FlatmmSn_32x128x512_1x4x1_16x16x32_Base
+{
+    using BDataType = bf16_t;
+    using ODataType = bf16_t;
+
+    // TODO: need paired with tile_window_linear!
+    // TODO: need call init_raw() before call this function!
+    // template <typename AWindow, typename BWindow, typename OWindow, typename ScaleTensor>
+    template <typename BRes,
+              typename BCoords,
+              typename ORes,
+              typename OCoords,
+              typename OFlags,
+              typename ScaleTensor>
+    CK_TILE_DEVICE auto
+    operator()(const BRes& res_b,
+               const BCoords& cached_coords_b,
+               const ORes& res_o,
+               const OCoords& cached_coords_o,
+               const OFlags& o_flags, // this should be in sgpr
+               CK_TILE_LDS_ADDR void* smem,
+               index_t n, // loop along n dim
+               const ScaleTensor& scale_,
+               index_t tile_offset_b, // stride b is fixed to blockKr * blockW, but still can adjust
+               index_t tile_offset_o)
+    {
+        static_assert(BCoords::size() == 8); // 8
+        static_assert(OCoords::size() == 8);
+
+        const index_t tile_stride_b_bytes = tile_offset_b * sizeof(BDataType);
+        const index_t tile_stride_o_bytes = tile_offset_o * sizeof(ODataType);
+
+        static_assert(ScaleTensor::size() == 2);
+        float s0 = scale_[number<0>{}];
+        float s1 = scale_[number<1>{}];
+
+        // index_t loop_cnt = n / Block_N;
+
+        register float v_c0 asm("v64");
+        register float v_c1 asm("v65");
+        register float v_c2 asm("v66");
+        register float v_c3 asm("v67");
+        register float v_c4 asm("v68");
+        register float v_c5 asm("v69");
+        register float v_c6 asm("v70");
+        register float v_c7 asm("v71");
+        register float v_c8 asm("v72");
+        register float v_c9 asm("v73");
+        register float v_c10 asm("v74");
+        register float v_c11 asm("v75");
+        register float v_c12 asm("v76");
+        register float v_c13 asm("v77");
+        register float v_c14 asm("v78");
+        register float v_c15 asm("v79");
+        register float v_c16 asm("v80");
+        register float v_c17 asm("v81");
+        register float v_c18 asm("v82");
+        register float v_c19 asm("v83");
+        register float v_c20 asm("v84");
+        register float v_c21 asm("v85");
+        register float v_c22 asm("v86");
+        register float v_c23 asm("v87");
+        register float v_c24 asm("v88");
+        register float v_c25 asm("v89");
+        register float v_c26 asm("v90");
+        register float v_c27 asm("v91");
+        register float v_c28 asm("v92");
+        register float v_c29 asm("v93");
+        register float v_c30 asm("v94");
+        register float v_c31 asm("v95");
+        int32_t nan_hi = 0x7fff0000;
+        int32_t nan_lo = 0x00007fff;
+
+        // in smem, the layout is  M0(2)*K0(128)*M1(16)*K1(4)
+        // every threads need 8xK in contiguous register
+        // ... and every wave need the same data
+        int lane_id  = threadIdx.x % 64;
+        int sld_y_os = (lane_id % 16) * 4 + (lane_id / 16) * 128;
+        sld_y_os *= 2;
+
+        //                    y     y     p     p      p      y
+        // reg before shfl  M0(2)*N0(2)*Nl(4)*Nw(4)*Mw(16)*Nv(4)
+        // but order is N0*M0*Nv
+        // in LDS we need store as
+        //          M0(2)* N0(2) *  Nl(4) * Nw(4) * (Mw(16)*Nv(4) + 4)
+        //             y    y       wave-id  lid/16  lid%16   v
+        // sst(v3) = (v0/16*34 + v0%16 * 2 + wid*136) * 4
+        int sfl_sst = (threadIdx.x % 16 * 4) + (threadIdx.x / 16) * (64 + 4);
+        sfl_sst *= 2;
+
+        // from LDS we need load as
+        //          M0(2)*    N0(2) *  Nl(4) * Nw(4) * (Mw(16)         *  Nv(4) + 4)
+        //        ( 2 issue)    (rem 32-lane)        (4 wave*4issue)   2lane*1ussue(pk2)
+        // sld(v4) = v0/2 *34*4  + v0 % 2 *4 + wid*2 *4
+        int sfl_sld = (lane_id % 2) * 2 + (lane_id / 2) * (64 + 4) + (threadIdx.x / 64) * 4;
+        sfl_sld *= 2;
+
+        // B nr->kr
+        // clang-format off
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Winline-asm"
+        asm volatile(
+#define CK_TILE_FLATMM_UK_MFMA CK_TILE_FLATMM_UK_MFMA_FP16
+#include "uk/flatmm_sn_uk_gfx9_32x128x512_1x4x1_16x16x16_itl.inc"
+#undef CK_TILE_FLATMM_UK_MFMA
+            :[smem_]"+r"(smem),
+            [s_loop_cnt]"+s"(n),
+                [c0]"+v" (v_c0),
+                [c1]"+v" (v_c1),
+                [c2]"+v" (v_c2),
+                [c3]"+v" (v_c3),
+                [c4]"+v" (v_c4),
+                [c5]"+v" (v_c5),
+                [c6]"+v" (v_c6),
+                [c7]"+v" (v_c7),
+                [c8]"+v" (v_c8),
+                [c9]"+v" (v_c9),
+                [c10]"+v"(v_c10),
+                [c11]"+v"(v_c11),
+                [c12]"+v"(v_c12),
+                [c13]"+v"(v_c13),
+                [c14]"+v"(v_c14),
+                [c15]"+v"(v_c15),
+                [c16]"+v"(v_c16),
+                [c17]"+v"(v_c17),
+                [c18]"+v"(v_c18),
+                [c19]"+v"(v_c19),
+                [c20]"+v"(v_c20),
+                [c21]"+v"(v_c21),
+                [c22]"+v"(v_c22),
+                [c23]"+v"(v_c23),
+                [c24]"+v"(v_c24),
+                [c25]"+v"(v_c25),
+                [c26]"+v"(v_c26),
+                [c27]"+v"(v_c27),
+                [c28]"+v"(v_c28),
+                [c29]"+v"(v_c29),
+                [c30]"+v"(v_c30),
+                [c31]"+v"(v_c31)
+            :
+            [sld_a_base]"n"(0),
+            [shfl_base]"n"(0),
+            [v_sld_y_os]"v"(sld_y_os),
+            [v_sfl_sld]"v"(sfl_sld),
+            [v_sfl_sst]"v"(sfl_sst),
+            [s_res_o0]"s"(res_o[0]),
+                [s_res_o1]"s"(res_o[1]),
+                //[s_res_o2]"s"(res_o[2]),
+                //[s_res_o3]"s"(res_o[3]),
+                [s_res_b0]"s"(res_b[0]),
+                [s_res_b1]"s"(res_b[1]),
+                [s_res_b2]"s"(res_b[2]),
+                [s_res_b3]"s"(res_b[3]),
+                [v_os_o0]"v"(static_cast<index_t>(cached_coords_o[number<0>{}] * sizeof(ODataType))),
+                [v_os_o1]"v"(static_cast<index_t>(cached_coords_o[number<1>{}] * sizeof(ODataType))),
+                [v_os_o2]"v"(static_cast<index_t>(cached_coords_o[number<2>{}] * sizeof(ODataType))),
+                [v_os_o3]"v"(static_cast<index_t>(cached_coords_o[number<3>{}] * sizeof(ODataType))),
+                [v_os_o4]"v"(static_cast<index_t>(cached_coords_o[number<4>{}] * sizeof(ODataType))),
+                [v_os_o5]"v"(static_cast<index_t>(cached_coords_o[number<5>{}] * sizeof(ODataType))),
+                [v_os_o6]"v"(static_cast<index_t>(cached_coords_o[number<6>{}] * sizeof(ODataType))),
+                [v_os_o7]"v"(static_cast<index_t>(cached_coords_o[number<7>{}] * sizeof(ODataType))),
+                [v_os_b0]"v"(static_cast<index_t>(cached_coords_b[number<0>{}] * sizeof(BDataType))),
+                [v_os_b1]"v"(static_cast<index_t>(cached_coords_b[number<1>{}] * sizeof(BDataType))),
+                [v_os_b2]"v"(static_cast<index_t>(cached_coords_b[number<2>{}] * sizeof(BDataType))),
+                [v_os_b3]"v"(static_cast<index_t>(cached_coords_b[number<3>{}] * sizeof(BDataType))),
+                [v_os_b4]"v"(static_cast<index_t>(cached_coords_b[number<4>{}] * sizeof(BDataType))),
+                [v_os_b5]"v"(static_cast<index_t>(cached_coords_b[number<5>{}] * sizeof(BDataType))),
+                [v_os_b6]"v"(static_cast<index_t>(cached_coords_b[number<6>{}] * sizeof(BDataType))),
+                [v_os_b7]"v"(static_cast<index_t>(cached_coords_b[number<7>{}] * sizeof(BDataType))),
+
+                [s_tile_os_o]"s"(tile_stride_o_bytes),
+                [s_tile_os_b]"s"(tile_stride_b_bytes),
+                [scale_0]"v"(s0),
+                [scale_1]"v"(s1),
+                [v_nan_lo]"v"(nan_lo),
+                [v_nan_hi]"v"(nan_hi),
+                [s_execflag_0]"s"(o_flags[number<0>{}]),
+                [s_execflag_1]"s"(o_flags[number<1>{}]),
+                [s_execflag_2]"s"(o_flags[number<2>{}]),
+                [s_execflag_3]"s"(o_flags[number<3>{}]),
+                [s_execflag_4]"s"(o_flags[number<4>{}]),
+                [s_execflag_5]"s"(o_flags[number<5>{}]),
+                [s_execflag_6]"s"(o_flags[number<6>{}]),
+                [s_execflag_7]"s"(o_flags[number<7>{}])
+            :
+          "memory", "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8", "a9",
+          "a10", "a11", "a12", "a13", "a14", "a15", "a16", "a17", "a18", "a19",
+          "a20", "a21", "a22", "a23", "a24", "a25", "a26", "a27", "a28", "a29",
+          "a30", "a31", "a32", "a33", "a34", "a35", "a36", "a37", "a38", "a39",
+          "a40", "a41", "a42", "a43", "a44", "a45", "a46", "a47", "a48", "a49",
+          "a50", "a51", "a52", "a53", "a54", "a55", "a56", "a57", "a58", "a59",
+          "a60", "a61", "a62", "a63", "a64", "a65", "a66", "a67", "a68", "a69",
+          "a70", "a71", "a72", "a73", "a74", "a75", "a76", "a77", "a78", "a79",
+          "a80", "a81", "a82", "a83", "a84", "a85", "a86", "a87", "a88", "a89",
+          "a90", "a91", "a92", "a93", "a94", "a95", "a96", "a97", "a98", "a99",
+          "a100", "a101", "a102", "a103", "a104", "a105", "a106", "a107",
+          "a108", "a109", "a110", "a111", "a112", "a113", "a114", "a115",
+          "a116", "a117", "a118", "a119", "a120", "a121", "a122", "a123",
+          "a124", "a125", "a126", "a127", "a128", "a129", "a130", "a131",
+          "a132", "a133", "a134", "a135", "a136", "a137", "a138", "a139",
+          "a140", "a141", "a142", "a143", "a144", "a145", "a146", "a147",
+          "a148", "a149", "a150", "a151", "a152", "a153", "a154", "a155",
+          "a156", "a157", "a158", "a159", "a160", "a161", "a162", "a163",
+          "a164", "a165", "a166", "a167", "a168", "a169", "a170", "a171",
+          "a172", "a173", "a174", "a175", "a176", "a177", "a178", "a179",
+          "a180", "a181", "a182", "a183", "a184", "a185", "a186", "a187",
+          "a188", "a189", "a190", "a191", "a192", "a193", "a194", "a195",
+          "a196", "a197", "a198", "a199", "a200", "a201", "a202", "a203",
+          "a204", "a205", "a206", "a207", "a208", "a209", "a210", "a211",
+          "a212", "a213", "a214", "a215", "a216", "a217", "a218", "a219",
+          "a220", "a221", "a222", "a223", "a224", "a225", "a226", "a227",
+          "a228", "a229", "a230", "a231", "a232", "a233", "a234", "a235",
+          "a236", "a237", "a238", "a239", "a240", "a241", "a242", "a243",
+          "a244", "a245", "a246", "a247", "a248", "a249", "a250", "a251",
+          "a252", "a253", "a254", "a255", 
+          "s8", "s9", "s12", "s13", "s14", "s15", "s38", "s39", "s52", "s86",
+          "s36", "s37","s59","s80",
+          "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",
+          "v50", "v54", "v55",
+          "v64","v65","v66","v67","v68","v69","v70","v71",
+          "v72","v73","v74","v75","v76","v77","v78","v79",
+          "v80","v81","v82","v83","v84","v85","v86","v87",
+          "v88","v89","v90","v91","v92","v93","v94","v95",
+          "v128", "v129", "v130", "v131",
+          "v132", "v133", "v134", "v135", "v136", "v137", "v138", "v139",
+          "v140", "v141", "v142", "v143", "v144", "v145", "v146", "v147",
+          "v148", "v149", "v150", "v151", "v152", "v153", "v154", "v155",
+          "v156", "v157", "v158", "v159", "v160", "v161", "v162", "v163",
+          "v164", "v165", "v166", "v167", "v168", "v169", "v170", "v171",
+          "v172", "v173", "v174", "v175", "v176", "v177", "v178", "v179",
+          "v180", "v181", "v182", "v183", "v184", "v185", "v186", "v187",
+          "v188", "v189", "v190", "v191", "v192", "v193", "v194", "v195",
+          "v196", "v197", "v198", "v199", "v200", "v201", "v202", "v203",
+          "v204", "v205", "v206", "v207", "v208", "v209", "v210", "v211",
+          "v212", "v213", "v214", "v215", "v216", "v217", "v218", "v219",
+          "v220", "v221", "v222", "v223", "v224", "v225", "v226", "v227",
+          "v228", "v229", "v230", "v231", "v232", "v233", "v234", "v235",
+          "v236", "v237", "v238", "v239", "v240", "v241", "v242", "v243",
+          "v244", "v245", "v246", "v247", "v248", "v249", "v250", "v251",
+          "v252", "v253", "v254", "v255"
+        );
+#pragma clang diagnostic pop
+        // clang-format on
+    }
+};
+
+} // namespace ck_tile

include/ck_tile/ops/flatmm/block/uk/flatmm_sn_uk_gfx9_32x128x512_1x4x1_16x16x16.inc

@@ -3,610 +3,815 @@
 #endif
 
 #if CK_TILE_FLATMM_UK_MFMA == CK_TILE_FLATMM_UK_MFMA_BF16
-#   define _UK_MFMA_ "v_mfma_f32_16x16x16_bf16"
+#define _UK_MFMA_ "v_mfma_f32_16x16x16_bf16"
 
-#   define _UK_PK_CVT_(x0_, x1_, y_)                           \
-    "  v_cmp_u_f32   s[36:37], " x0_ ", " x0_ "          \n"   \
-    "  v_add3_u32    v50, " x0_ ", %[v_nan_lo], 1        \n"   \
-    "  v_cndmask_b32  v54, v50, %[v_nan_hi], s[36:37]    \n"   \
-    "  v_cmp_u_f32   s[36:37], " x1_ ", " x1_ "          \n"   \
-    "  v_add3_u32    v50, " x1_ ", %[v_nan_lo], 1        \n"   \
-    "  v_cndmask_b32  v55, v50, %[v_nan_hi], s[36:37]    \n"   \
+#define _UK_PK_CVT_(x0_, x1_, y_)                            \
+    "  v_cmp_u_f32   s[36:37], " x0_ ", " x0_ "          \n" \
+    "  v_add3_u32    v50, " x0_ ", %[v_nan_lo], 1        \n" \
+    "  v_cndmask_b32  v54, v50, %[v_nan_hi], s[36:37]    \n" \
+    "  v_cmp_u_f32   s[36:37], " x1_ ", " x1_ "          \n" \
+    "  v_add3_u32    v50, " x1_ ", %[v_nan_lo], 1        \n" \
+    "  v_cndmask_b32  v55, v50, %[v_nan_hi], s[36:37]    \n" \
     "  v_perm_b32    " y_ ", v55, v54, s52               \n"
 
-#   define _UK_ATOMIC_ADD_ "global_atomic_pk_add_bf16"
+#define _UK_ATOMIC_ADD_ "global_atomic_pk_add_bf16"
 
 #elif CK_TILE_FLATMM_UK_MFMA == CK_TILE_FLATMM_UK_MFMA_FP16
 #define _UK_MFMA_ "v_mfma_f32_16x16x16_f16"
 
-#   define _UK_PK_CVT_(x0_, x1_, y_)                \
-    "  v_cvt_f16_f32  v54, " x0_ "  \n"             \
-    "  v_cvt_f16_f32  v55, " x1_ "  \n"             \
+#define _UK_PK_CVT_(x0_, x1_, y_)       \
+    "  v_cvt_f16_f32  v54, " x0_ "  \n" \
+    "  v_cvt_f16_f32  v55, " x1_ "  \n" \
     "  v_pack_b32_f16 " y_ ", v54, v55  \n"
 
-#   define _UK_ATOMIC_ADD_ "global_atomic_pk_add_f16"
+#define _UK_ATOMIC_ADD_ "global_atomic_pk_add_f16"
 
 #endif
 
-
 ";-------------------------------------------------------------\n"
-" s_mov_b32 s52, 0x07060302 ; v_perm\n"
-" s_mov_b64 s[38:39], exec ; save current exec\n"
-" s_mov_b32 s8,    %[s_res_o0] \n"
-" s_mov_b32 s9,    %[s_res_o1] \n"
-" s_mov_b32 s12,    %[s_res_b0] \n"
-" s_mov_b32 s13,    %[s_res_b1] \n"
-" s_mov_b32 s14,    %[s_res_b2] \n"
-" s_mov_b32 s15,    %[s_res_b3] \n"
-" ds_read_b64   v[128:129], %[v_sld_y_os] offset:0 + %[sld_a_base]                       \n"
-" ds_read_b64   v[130:131], %[v_sld_y_os] offset:128 + %[sld_a_base]                     \n"
-" ds_read_b64   v[132:133], %[v_sld_y_os] offset:1024 + %[sld_a_base]                    \n"
-" ds_read_b64   v[134:135], %[v_sld_y_os] offset:1152 + %[sld_a_base]                    \n"
-" ds_read_b64   v[136:137], %[v_sld_y_os] offset:2048 + %[sld_a_base]                    \n"
-" ds_read_b64   v[138:139], %[v_sld_y_os] offset:2176 + %[sld_a_base]                    \n"
-" ds_read_b64   v[140:141], %[v_sld_y_os] offset:3072 + %[sld_a_base]                    \n"
-" ds_read_b64   v[142:143], %[v_sld_y_os] offset:3200 + %[sld_a_base]                    \n"
-" ds_read_b64   v[144:145], %[v_sld_y_os] offset:4096 + %[sld_a_base]                    \n"
-" ds_read_b64   v[146:147], %[v_sld_y_os] offset:4224 + %[sld_a_base]                    \n"
-" ds_read_b64   v[148:149], %[v_sld_y_os] offset:5120 + %[sld_a_base]                    \n"
-" ds_read_b64   v[150:151], %[v_sld_y_os] offset:5248 + %[sld_a_base]                    \n"
-" ds_read_b64   v[152:153], %[v_sld_y_os] offset:6144 + %[sld_a_base]                    \n"
-" ds_read_b64   v[154:155], %[v_sld_y_os] offset:6272 + %[sld_a_base]                    \n"
-" ds_read_b64   v[156:157], %[v_sld_y_os] offset:7168 + %[sld_a_base]                    \n"
-" ds_read_b64   v[158:159], %[v_sld_y_os] offset:7296 + %[sld_a_base]                    \n"
-" ds_read_b64   v[160:161], %[v_sld_y_os] offset:8192 + %[sld_a_base]                    \n"
-" ds_read_b64   v[162:163], %[v_sld_y_os] offset:8320 + %[sld_a_base]                    \n"
-" ds_read_b64   v[164:165], %[v_sld_y_os] offset:9216 + %[sld_a_base]                    \n"
-" ds_read_b64   v[166:167], %[v_sld_y_os] offset:9344 + %[sld_a_base]                    \n"
-" ds_read_b64   v[168:169], %[v_sld_y_os] offset:10240 + %[sld_a_base]                    \n"
-" ds_read_b64   v[170:171], %[v_sld_y_os] offset:10368 + %[sld_a_base]                    \n"
-" ds_read_b64   v[172:173], %[v_sld_y_os] offset:11264 + %[sld_a_base]                    \n"
-" ds_read_b64   v[174:175], %[v_sld_y_os] offset:11392 + %[sld_a_base]                    \n"
-" ds_read_b64   v[176:177], %[v_sld_y_os] offset:12288 + %[sld_a_base]                    \n"
-" ds_read_b64   v[178:179], %[v_sld_y_os] offset:12416 + %[sld_a_base]                    \n"
-" ds_read_b64   v[180:181], %[v_sld_y_os] offset:13312 + %[sld_a_base]                    \n"
-" ds_read_b64   v[182:183], %[v_sld_y_os] offset:13440 + %[sld_a_base]                    \n"
-" ds_read_b64   v[184:185], %[v_sld_y_os] offset:14336 + %[sld_a_base]                    \n"
-" ds_read_b64   v[186:187], %[v_sld_y_os] offset:14464 + %[sld_a_base]                    \n"
-" ds_read_b64   v[188:189], %[v_sld_y_os] offset:15360 + %[sld_a_base]                    \n"
-" ds_read_b64   v[190:191], %[v_sld_y_os] offset:15488 + %[sld_a_base]                    \n"
-" ds_read_b64   v[192:193], %[v_sld_y_os] offset:16384 + %[sld_a_base]                    \n"
-" ds_read_b64   v[194:195], %[v_sld_y_os] offset:16512 + %[sld_a_base]                    \n"
-" ds_read_b64   v[196:197], %[v_sld_y_os] offset:17408 + %[sld_a_base]                    \n"
-" ds_read_b64   v[198:199], %[v_sld_y_os] offset:17536 + %[sld_a_base]                    \n"
-" ds_read_b64   v[200:201], %[v_sld_y_os] offset:18432 + %[sld_a_base]                    \n"
-" ds_read_b64   v[202:203], %[v_sld_y_os] offset:18560 + %[sld_a_base]                    \n"
-" ds_read_b64   v[204:205], %[v_sld_y_os] offset:19456 + %[sld_a_base]                    \n"
-" ds_read_b64   v[206:207], %[v_sld_y_os] offset:19584 + %[sld_a_base]                    \n"
-" ds_read_b64   v[208:209], %[v_sld_y_os] offset:20480 + %[sld_a_base]                    \n"
-" ds_read_b64   v[210:211], %[v_sld_y_os] offset:20608 + %[sld_a_base]                    \n"
-" ds_read_b64   v[212:213], %[v_sld_y_os] offset:21504 + %[sld_a_base]                    \n"
-" ds_read_b64   v[214:215], %[v_sld_y_os] offset:21632 + %[sld_a_base]                    \n"
-" ds_read_b64   v[216:217], %[v_sld_y_os] offset:22528 + %[sld_a_base]                    \n"
-" ds_read_b64   v[218:219], %[v_sld_y_os] offset:22656 + %[sld_a_base]                    \n"
-" ds_read_b64   v[220:221], %[v_sld_y_os] offset:23552 + %[sld_a_base]                    \n"
-" ds_read_b64   v[222:223], %[v_sld_y_os] offset:23680 + %[sld_a_base]                    \n"
-" ds_read_b64   v[224:225], %[v_sld_y_os] offset:24576 + %[sld_a_base]                    \n"
-" ds_read_b64   v[226:227], %[v_sld_y_os] offset:24704 + %[sld_a_base]                    \n"
-" ds_read_b64   v[228:229], %[v_sld_y_os] offset:25600 + %[sld_a_base]                    \n"
-" ds_read_b64   v[230:231], %[v_sld_y_os] offset:25728 + %[sld_a_base]                    \n"
-" ds_read_b64   v[232:233], %[v_sld_y_os] offset:26624 + %[sld_a_base]                    \n"
-" ds_read_b64   v[234:235], %[v_sld_y_os] offset:26752 + %[sld_a_base]                    \n"
-" ds_read_b64   v[236:237], %[v_sld_y_os] offset:27648 + %[sld_a_base]                    \n"
-" ds_read_b64   v[238:239], %[v_sld_y_os] offset:27776 + %[sld_a_base]                    \n"
-" ds_read_b64   v[240:241], %[v_sld_y_os] offset:28672 + %[sld_a_base]                    \n"
-" ds_read_b64   v[242:243], %[v_sld_y_os] offset:28800 + %[sld_a_base]                    \n"
-" ds_read_b64   v[244:245], %[v_sld_y_os] offset:29696 + %[sld_a_base]                    \n"
-" ds_read_b64   v[246:247], %[v_sld_y_os] offset:29824 + %[sld_a_base]                    \n"
-" ds_read_b64   v[248:249], %[v_sld_y_os] offset:30720 + %[sld_a_base]                    \n"
-" ds_read_b64   v[250:251], %[v_sld_y_os] offset:30848 + %[sld_a_base]                    \n"
-" ds_read_b64   v[252:253], %[v_sld_y_os] offset:31744 + %[sld_a_base]                    \n"
-" ds_read_b64   v[254:255], %[v_sld_y_os] offset:31872 + %[sld_a_base]                    \n"
-"  s_waitcnt 0                    \n"
-"  buffer_load_dwordx4  acc[0:3], %[v_os_b0], s[12:15], 0 offen \n"
-"  buffer_load_dwordx4  acc[4:7], %[v_os_b0], s[12:15], 0 offen offset:1024 \n"
-"  buffer_load_dwordx4  acc[8:11], %[v_os_b0], s[12:15], 0 offen offset:2048 \n"
-"  buffer_load_dwordx4  acc[12:15], %[v_os_b0], s[12:15], 0 offen offset:3072 \n"
-"  buffer_load_dwordx4  acc[16:19], %[v_os_b1], s[12:15], 0 offen \n"
-"  buffer_load_dwordx4  acc[20:23], %[v_os_b1], s[12:15], 0 offen offset:1024 \n"
-"  buffer_load_dwordx4  acc[24:27], %[v_os_b1], s[12:15], 0 offen offset:2048 \n"
-"  buffer_load_dwordx4  acc[28:31], %[v_os_b1], s[12:15], 0 offen offset:3072 \n"
-"  buffer_load_dwordx4  acc[32:35], %[v_os_b2], s[12:15], 0 offen \n"
-"  buffer_load_dwordx4  acc[36:39], %[v_os_b2], s[12:15], 0 offen offset:1024 \n"
-"  buffer_load_dwordx4  acc[40:43], %[v_os_b2], s[12:15], 0 offen offset:2048 \n"
-"  buffer_load_dwordx4  acc[44:47], %[v_os_b2], s[12:15], 0 offen offset:3072 \n"
-"  buffer_load_dwordx4  acc[48:51], %[v_os_b3], s[12:15], 0 offen \n"
-"  buffer_load_dwordx4  acc[52:55], %[v_os_b3], s[12:15], 0 offen offset:1024 \n"
-"  buffer_load_dwordx4  acc[56:59], %[v_os_b3], s[12:15], 0 offen offset:2048 \n"
-"  buffer_load_dwordx4  acc[60:63], %[v_os_b3], s[12:15], 0 offen offset:3072 \n"
-"  buffer_load_dwordx4  acc[64:67], %[v_os_b4], s[12:15], 0 offen \n"
-"  buffer_load_dwordx4  acc[68:71], %[v_os_b4], s[12:15], 0 offen offset:1024 \n"
-"  buffer_load_dwordx4  acc[72:75], %[v_os_b4], s[12:15], 0 offen offset:2048 \n"
-"  buffer_load_dwordx4  acc[76:79], %[v_os_b4], s[12:15], 0 offen offset:3072 \n"
-"  buffer_load_dwordx4  acc[80:83], %[v_os_b5], s[12:15], 0 offen \n"
-"  buffer_load_dwordx4  acc[84:87], %[v_os_b5], s[12:15], 0 offen offset:1024 \n"
-"  buffer_load_dwordx4  acc[88:91], %[v_os_b5], s[12:15], 0 offen offset:2048 \n"
-"  buffer_load_dwordx4  acc[92:95], %[v_os_b5], s[12:15], 0 offen offset:3072 \n"
-"  buffer_load_dwordx4  acc[96:99], %[v_os_b6], s[12:15], 0 offen \n"
-"  buffer_load_dwordx4  acc[100:103], %[v_os_b6], s[12:15], 0 offen offset:1024 \n"
-"  buffer_load_dwordx4  acc[104:107], %[v_os_b6], s[12:15], 0 offen offset:2048 \n"
-"  buffer_load_dwordx4  acc[108:111], %[v_os_b6], s[12:15], 0 offen offset:3072 \n"
-"  buffer_load_dwordx4  acc[112:115], %[v_os_b7], s[12:15], 0 offen \n"
-"  buffer_load_dwordx4  acc[116:119], %[v_os_b7], s[12:15], 0 offen offset:1024 \n"
-"  buffer_load_dwordx4  acc[120:123], %[v_os_b7], s[12:15], 0 offen offset:2048 \n"
-"  buffer_load_dwordx4  acc[124:127], %[v_os_b7], s[12:15], 0 offen offset:3072 \n"
-"  s_cmp_gt_i32  %[s_loop_cnt] 1             ; move b with cond \n"
-"  s_cselect_b32 s86, %[s_tile_os_b], 0                          \n"
-"  s_add_u32     s12, s86, s12                                  \n"
-"  s_addc_u32    s13, 0, s13                                    \n"
-"  s_waitcnt 0                    \n"
-"L_start%=:                    \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-"  s_barrier                                             \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[0:1], v[128:129], 0 \n"
-"  buffer_load_dwordx4  acc[128:131], %[v_os_b0], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[2:3], v[130:131], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[4:5], v[132:133], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[6:7], v[134:135], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[8:9], v[136:137], [%[c0], %[c1], %[c2], %[c3]] \n"
-"  buffer_load_dwordx4  acc[132:135], %[v_os_b0], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[10:11], v[138:139], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[12:13], v[140:141], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[14:15], v[142:143], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[0:1], v[192:193], 0 \n"
-"  buffer_load_dwordx4  acc[136:139], %[v_os_b0], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[2:3], v[194:195], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[4:5], v[196:197], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[6:7], v[198:199], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[8:9], v[200:201], [%[c4], %[c5], %[c6], %[c7]] \n"
-"  buffer_load_dwordx4  acc[140:143], %[v_os_b0], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[10:11], v[202:203], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[12:13], v[204:205], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[14:15], v[206:207], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[16:17], v[128:129], 0 \n"
-"  buffer_load_dwordx4  acc[144:147], %[v_os_b1], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[18:19], v[130:131], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[20:21], v[132:133], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[22:23], v[134:135], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[24:25], v[136:137], [%[c8], %[c9], %[c10], %[c11]] \n"
-"  buffer_load_dwordx4  acc[148:151], %[v_os_b1], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[26:27], v[138:139], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[28:29], v[140:141], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[30:31], v[142:143], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[16:17], v[192:193], 0 \n"
-"  buffer_load_dwordx4  acc[152:155], %[v_os_b1], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[18:19], v[194:195], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[20:21], v[196:197], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[22:23], v[198:199], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[24:25], v[200:201], [%[c12], %[c13], %[c14], %[c15]] \n"
-"  buffer_load_dwordx4  acc[156:159], %[v_os_b1], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[26:27], v[202:203], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[28:29], v[204:205], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[30:31], v[206:207], [%[c12], %[c13], %[c14], %[c15]] \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[32:33], v[144:145], [%[c0], %[c1], %[c2], %[c3]] \n"
-"  buffer_load_dwordx4  acc[160:163], %[v_os_b2], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[34:35], v[146:147], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[36:37], v[148:149], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[38:39], v[150:151], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[40:41], v[152:153], [%[c0], %[c1], %[c2], %[c3]] \n"
-"  buffer_load_dwordx4  acc[164:167], %[v_os_b2], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[42:43], v[154:155], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[44:45], v[156:157], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[46:47], v[158:159], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[32:33], v[208:209], [%[c4], %[c5], %[c6], %[c7]] \n"
-"  buffer_load_dwordx4  acc[168:171], %[v_os_b2], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[34:35], v[210:211], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[36:37], v[212:213], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[38:39], v[214:215], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[40:41], v[216:217], [%[c4], %[c5], %[c6], %[c7]] \n"
-"  buffer_load_dwordx4  acc[172:175], %[v_os_b2], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[42:43], v[218:219], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[44:45], v[220:221], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[46:47], v[222:223], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[48:49], v[144:145], [%[c8], %[c9], %[c10], %[c11]] \n"
-"  buffer_load_dwordx4  acc[176:179], %[v_os_b3], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[50:51], v[146:147], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[52:53], v[148:149], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[54:55], v[150:151], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[56:57], v[152:153], [%[c8], %[c9], %[c10], %[c11]] \n"
-"  buffer_load_dwordx4  acc[180:183], %[v_os_b3], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[58:59], v[154:155], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[60:61], v[156:157], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[62:63], v[158:159], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[48:49], v[208:209], [%[c12], %[c13], %[c14], %[c15]] \n"
-"  buffer_load_dwordx4  acc[184:187], %[v_os_b3], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[50:51], v[210:211], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[52:53], v[212:213], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[54:55], v[214:215], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[56:57], v[216:217], [%[c12], %[c13], %[c14], %[c15]] \n"
-"  buffer_load_dwordx4  acc[188:191], %[v_os_b3], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[58:59], v[218:219], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[60:61], v[220:221], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[62:63], v[222:223], [%[c12], %[c13], %[c14], %[c15]] \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[64:65], v[160:161], [%[c0], %[c1], %[c2], %[c3]] \n"
-"  buffer_load_dwordx4  acc[192:195], %[v_os_b4], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[66:67], v[162:163], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[68:69], v[164:165], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[70:71], v[166:167], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[72:73], v[168:169], [%[c0], %[c1], %[c2], %[c3]] \n"
-"  buffer_load_dwordx4  acc[196:199], %[v_os_b4], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[74:75], v[170:171], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[76:77], v[172:173], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[78:79], v[174:175], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[64:65], v[224:225], [%[c4], %[c5], %[c6], %[c7]] \n"
-"  buffer_load_dwordx4  acc[200:203], %[v_os_b4], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[66:67], v[226:227], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[68:69], v[228:229], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[70:71], v[230:231], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[72:73], v[232:233], [%[c4], %[c5], %[c6], %[c7]] \n"
-"  buffer_load_dwordx4  acc[204:207], %[v_os_b4], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[74:75], v[234:235], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[76:77], v[236:237], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[78:79], v[238:239], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[80:81], v[160:161], [%[c8], %[c9], %[c10], %[c11]] \n"
-"  buffer_load_dwordx4  acc[208:211], %[v_os_b5], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[82:83], v[162:163], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[84:85], v[164:165], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[86:87], v[166:167], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[88:89], v[168:169], [%[c8], %[c9], %[c10], %[c11]] \n"
-"  buffer_load_dwordx4  acc[212:215], %[v_os_b5], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[90:91], v[170:171], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[92:93], v[172:173], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[94:95], v[174:175], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[80:81], v[224:225], [%[c12], %[c13], %[c14], %[c15]] \n"
-"  buffer_load_dwordx4  acc[216:219], %[v_os_b5], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[82:83], v[226:227], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[84:85], v[228:229], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[86:87], v[230:231], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[88:89], v[232:233], [%[c12], %[c13], %[c14], %[c15]] \n"
-"  buffer_load_dwordx4  acc[220:223], %[v_os_b5], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[90:91], v[234:235], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[92:93], v[236:237], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[94:95], v[238:239], [%[c12], %[c13], %[c14], %[c15]] \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[96:97], v[176:177], [%[c0], %[c1], %[c2], %[c3]] \n"
-"  buffer_load_dwordx4  acc[224:227], %[v_os_b6], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[98:99], v[178:179], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[100:101], v[180:181], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[102:103], v[182:183], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[104:105], v[184:185], [%[c0], %[c1], %[c2], %[c3]] \n"
-"  buffer_load_dwordx4  acc[228:231], %[v_os_b6], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[106:107], v[186:187], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[108:109], v[188:189], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[110:111], v[190:191], [%[c0], %[c1], %[c2], %[c3]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[96:97], v[240:241], [%[c4], %[c5], %[c6], %[c7]] \n"
-"  buffer_load_dwordx4  acc[232:235], %[v_os_b6], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[98:99], v[242:243], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[100:101], v[244:245], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[102:103], v[246:247], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[104:105], v[248:249], [%[c4], %[c5], %[c6], %[c7]] \n"
-"  buffer_load_dwordx4  acc[236:239], %[v_os_b6], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[106:107], v[250:251], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[108:109], v[252:253], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[110:111], v[254:255], [%[c4], %[c5], %[c6], %[c7]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[112:113], v[176:177], [%[c8], %[c9], %[c10], %[c11]] \n"
-"  buffer_load_dwordx4  acc[240:243], %[v_os_b7], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[114:115], v[178:179], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[116:117], v[180:181], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[118:119], v[182:183], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[120:121], v[184:185], [%[c8], %[c9], %[c10], %[c11]] \n"
-"  buffer_load_dwordx4  acc[244:247], %[v_os_b7], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[122:123], v[186:187], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[124:125], v[188:189], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[126:127], v[190:191], [%[c8], %[c9], %[c10], %[c11]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[112:113], v[240:241], [%[c12], %[c13], %[c14], %[c15]] \n"
-"  buffer_load_dwordx4  acc[248:251], %[v_os_b7], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[114:115], v[242:243], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[116:117], v[244:245], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[118:119], v[246:247], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[120:121], v[248:249], [%[c12], %[c13], %[c14], %[c15]] \n"
-"  buffer_load_dwordx4  acc[252:255], %[v_os_b7], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[122:123], v[250:251], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[124:125], v[252:253], [%[c12], %[c13], %[c14], %[c15]] \n"
-_UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[126:127], v[254:255], [%[c12], %[c13], %[c14], %[c15]]\n"
-"  v_mul_f32     %[c0], %[scale_0], %[c0]                            \n"
-"  v_mul_f32     %[c1], %[scale_0], %[c1]                            \n"
-"  v_mul_f32     %[c2], %[scale_0], %[c2]                            \n"
-"  v_mul_f32     %[c3], %[scale_0], %[c3]                            \n"
-"  v_mul_f32     %[c4], %[scale_1], %[c4]                            \n"
-"  v_mul_f32     %[c5], %[scale_1], %[c5]                            \n"
-"  v_mul_f32     %[c6], %[scale_1], %[c6]                            \n"
-"  v_mul_f32     %[c7], %[scale_1], %[c7]                            \n"
-"  v_mul_f32     %[c8], %[scale_0], %[c8]                            \n"
-"  v_mul_f32     %[c9], %[scale_0], %[c9]                            \n"
-"  v_mul_f32     %[c10], %[scale_0], %[c10]                            \n"
-"  v_mul_f32     %[c11], %[scale_0], %[c11]                            \n"
-"  v_mul_f32     %[c12], %[scale_1], %[c12]                            \n"
-"  v_mul_f32     %[c13], %[scale_1], %[c13]                            \n"
-"  v_mul_f32     %[c14], %[scale_1], %[c14]                            \n"
-"  v_mul_f32     %[c15], %[scale_1], %[c15]                            \n"
-_UK_PK_CVT_("%[c0]",  "%[c1]",  "%[c0]")
-_UK_PK_CVT_("%[c2]",  "%[c3]",  "%[c1]")
-_UK_PK_CVT_("%[c4]",  "%[c5]",  "%[c2]")
-_UK_PK_CVT_("%[c6]",  "%[c7]",  "%[c3]")
-_UK_PK_CVT_("%[c8]",  "%[c9]",  "%[c4]")
-_UK_PK_CVT_("%[c10]", "%[c11]", "%[c5]")
-_UK_PK_CVT_("%[c12]", "%[c13]", "%[c6]")
-_UK_PK_CVT_("%[c14]", "%[c15]", "%[c7]")
-"  ;------------------------------  \n"
-"  ds_write_b64  %[v_sfl_sst], [%[c0],%[c1]] offset:0    + %[shfl_base]               \n"
-"  ds_write_b64  %[v_sfl_sst], [%[c2],%[c3]] offset:4352 + %[shfl_base]               \n"
-"  ds_write_b64  %[v_sfl_sst], [%[c4],%[c5]] offset:2176 + %[shfl_base]               \n"
-"  ds_write_b64  %[v_sfl_sst], [%[c6],%[c7]] offset:6528 + %[shfl_base]               \n"
-"  s_waitcnt     lgkmcnt(0)                              \n"
-"  s_barrier                                             \n"
-"  ds_read_b32   %[c0], %[v_sfl_sld] offset:0    + %[shfl_base]                    \n"
-"  ds_read_b32   %[c1], %[v_sfl_sld] offset:32   + %[shfl_base]                    \n"
-"  ds_read_b32   %[c2], %[v_sfl_sld] offset:64   + %[shfl_base]                    \n"
-"  ds_read_b32   %[c3], %[v_sfl_sld] offset:96   + %[shfl_base]                    \n"
-"  ds_read_b32   %[c4], %[v_sfl_sld] offset:4352 + %[shfl_base]                    \n"
-"  ds_read_b32   %[c5], %[v_sfl_sld] offset:4384 + %[shfl_base]                    \n"
-"  ds_read_b32   %[c6], %[v_sfl_sld] offset:4416 + %[shfl_base]                    \n"
-"  ds_read_b32   %[c7], %[v_sfl_sld] offset:4448 + %[shfl_base]                    \n"
-"  s_waitcnt     lgkmcnt(0)                              \n"
-"  s_mov_b64     exec, %[s_execflag_0]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o0], %[c0], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_1]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o1], %[c1], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_2]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o2], %[c2], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_3]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o3], %[c3], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_4]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o4], %[c4], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_5]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o5], %[c5], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_6]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o6], %[c6], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_7]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o7], %[c7], s[8:9]  \n"
-"  s_mov_b64     exec, s[38:39]                           \n"
-"  s_sub_i32     %[s_loop_cnt], %[s_loop_cnt], 1     ; k--      \n"
-"  s_cmp_gt_i32  %[s_loop_cnt] 0                                \n"
-"  s_cbranch_scc0 L_end%=                                       \n"
-"  s_cmp_gt_i32  %[s_loop_cnt] 1             ; move b with cond \n"
-"  s_cselect_b32 s86, %[s_tile_os_b], 0                          \n"
-"  s_add_u32     s12, s86, s12                                  \n"
-"  s_addc_u32    s13, 0, s13                                    \n"
-"  s_add_u32     s8, %[s_tile_os_o], s8                             \n"
-"  s_addc_u32    s9, 0, s9                               \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-"  s_barrier                                             \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[128:129], v[128:129], 0 \n"
-"  buffer_load_dwordx4  acc[0:3], %[v_os_b0], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[130:131], v[130:131], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[132:133], v[132:133], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[134:135], v[134:135], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[136:137], v[136:137], [%[c16],%[c17],%[c18],%[c19]] \n"
-"  buffer_load_dwordx4  acc[4:7], %[v_os_b0], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[138:139], v[138:139], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[140:141], v[140:141], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[142:143], v[142:143], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[128:129], v[192:193], 0 \n"
-"  buffer_load_dwordx4  acc[8:11], %[v_os_b0], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[130:131], v[194:195], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[132:133], v[196:197], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[134:135], v[198:199], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[136:137], v[200:201], [%[c20],%[c21],%[c22],%[c23]] \n"
-"  buffer_load_dwordx4  acc[12:15], %[v_os_b0], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[138:139], v[202:203], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[140:141], v[204:205], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[142:143], v[206:207], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[144:145], v[128:129], 0 \n"
-"  buffer_load_dwordx4  acc[16:19], %[v_os_b1], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[146:147], v[130:131], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[148:149], v[132:133], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[150:151], v[134:135], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[152:153], v[136:137], [%[c24],%[c25],%[c26],%[c27]] \n"
-"  buffer_load_dwordx4  acc[20:23], %[v_os_b1], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[154:155], v[138:139], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[156:157], v[140:141], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[158:159], v[142:143], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[144:145], v[192:193], 0 \n"
-"  buffer_load_dwordx4  acc[24:27], %[v_os_b1], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[146:147], v[194:195], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[148:149], v[196:197], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[150:151], v[198:199], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[152:153], v[200:201], [%[c28],%[c29],%[c30],%[c31]] \n"
-"  buffer_load_dwordx4  acc[28:31], %[v_os_b1], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[154:155], v[202:203], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[156:157], v[204:205], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[158:159], v[206:207], [%[c28],%[c29],%[c30],%[c31]] \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[160:161], v[144:145], [%[c16],%[c17],%[c18],%[c19]] \n"
-"  buffer_load_dwordx4  acc[32:35], %[v_os_b2], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[162:163], v[146:147], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[164:165], v[148:149], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[166:167], v[150:151], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[168:169], v[152:153], [%[c16],%[c17],%[c18],%[c19]] \n"
-"  buffer_load_dwordx4  acc[36:39], %[v_os_b2], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[170:171], v[154:155], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[172:173], v[156:157], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[174:175], v[158:159], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[160:161], v[208:209], [%[c20],%[c21],%[c22],%[c23]] \n"
-"  buffer_load_dwordx4  acc[40:43], %[v_os_b2], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[162:163], v[210:211], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[164:165], v[212:213], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[166:167], v[214:215], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[168:169], v[216:217], [%[c20],%[c21],%[c22],%[c23]] \n"
-"  buffer_load_dwordx4  acc[44:47], %[v_os_b2], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[170:171], v[218:219], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[172:173], v[220:221], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[174:175], v[222:223], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[176:177], v[144:145], [%[c24],%[c25],%[c26],%[c27]] \n"
-"  buffer_load_dwordx4  acc[48:51], %[v_os_b3], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[178:179], v[146:147], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[180:181], v[148:149], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[182:183], v[150:151], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[184:185], v[152:153], [%[c24],%[c25],%[c26],%[c27]] \n"
-"  buffer_load_dwordx4  acc[52:55], %[v_os_b3], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[186:187], v[154:155], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[188:189], v[156:157], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[190:191], v[158:159], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[176:177], v[208:209], [%[c28],%[c29],%[c30],%[c31]] \n"
-"  buffer_load_dwordx4  acc[56:59], %[v_os_b3], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[178:179], v[210:211], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[180:181], v[212:213], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[182:183], v[214:215], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[184:185], v[216:217], [%[c28],%[c29],%[c30],%[c31]] \n"
-"  buffer_load_dwordx4  acc[60:63], %[v_os_b3], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[186:187], v[218:219], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[188:189], v[220:221], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[190:191], v[222:223], [%[c28],%[c29],%[c30],%[c31]] \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[192:193], v[160:161], [%[c16],%[c17],%[c18],%[c19]] \n"
-"  buffer_load_dwordx4  acc[64:67], %[v_os_b4], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[194:195], v[162:163], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[196:197], v[164:165], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[198:199], v[166:167], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[200:201], v[168:169], [%[c16],%[c17],%[c18],%[c19]] \n"
-"  buffer_load_dwordx4  acc[68:71], %[v_os_b4], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[202:203], v[170:171], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[204:205], v[172:173], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[206:207], v[174:175], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[192:193], v[224:225], [%[c20],%[c21],%[c22],%[c23]] \n"
-"  buffer_load_dwordx4  acc[72:75], %[v_os_b4], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[194:195], v[226:227], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[196:197], v[228:229], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[198:199], v[230:231], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[200:201], v[232:233], [%[c20],%[c21],%[c22],%[c23]] \n"
-"  buffer_load_dwordx4  acc[76:79], %[v_os_b4], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[202:203], v[234:235], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[204:205], v[236:237], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[206:207], v[238:239], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[208:209], v[160:161], [%[c24],%[c25],%[c26],%[c27]] \n"
-"  buffer_load_dwordx4  acc[80:83], %[v_os_b5], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[210:211], v[162:163], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[212:213], v[164:165], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[214:215], v[166:167], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[216:217], v[168:169], [%[c24],%[c25],%[c26],%[c27]] \n"
-"  buffer_load_dwordx4  acc[84:87], %[v_os_b5], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[218:219], v[170:171], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[220:221], v[172:173], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[222:223], v[174:175], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[208:209], v[224:225], [%[c28],%[c29],%[c30],%[c31]] \n"
-"  buffer_load_dwordx4  acc[88:91], %[v_os_b5], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[210:211], v[226:227], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[212:213], v[228:229], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[214:215], v[230:231], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[216:217], v[232:233], [%[c28],%[c29],%[c30],%[c31]] \n"
-"  buffer_load_dwordx4  acc[92:95], %[v_os_b5], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[218:219], v[234:235], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[220:221], v[236:237], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[222:223], v[238:239], [%[c28],%[c29],%[c30],%[c31]] \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[224:225], v[176:177], [%[c16],%[c17],%[c18],%[c19]] \n"
-"  buffer_load_dwordx4  acc[96:99], %[v_os_b6], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[226:227], v[178:179], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[228:229], v[180:181], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[230:231], v[182:183], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[232:233], v[184:185], [%[c16],%[c17],%[c18],%[c19]] \n"
-"  buffer_load_dwordx4  acc[100:103], %[v_os_b6], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[234:235], v[186:187], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[236:237], v[188:189], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[238:239], v[190:191], [%[c16],%[c17],%[c18],%[c19]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[224:225], v[240:241], [%[c20],%[c21],%[c22],%[c23]] \n"
-"  buffer_load_dwordx4  acc[104:107], %[v_os_b6], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[226:227], v[242:243], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[228:229], v[244:245], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[230:231], v[246:247], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[232:233], v[248:249], [%[c20],%[c21],%[c22],%[c23]] \n"
-"  buffer_load_dwordx4  acc[108:111], %[v_os_b6], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[234:235], v[250:251], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[236:237], v[252:253], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c20],%[c21],%[c22],%[c23]], acc[238:239], v[254:255], [%[c20],%[c21],%[c22],%[c23]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[240:241], v[176:177], [%[c24],%[c25],%[c26],%[c27]] \n"
-"  buffer_load_dwordx4  acc[112:115], %[v_os_b7], s[12:15], 0 offen \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[242:243], v[178:179], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[244:245], v[180:181], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[246:247], v[182:183], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[248:249], v[184:185], [%[c24],%[c25],%[c26],%[c27]] \n"
-"  buffer_load_dwordx4  acc[116:119], %[v_os_b7], s[12:15], 0 offen offset:1024 \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[250:251], v[186:187], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[252:253], v[188:189], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[254:255], v[190:191], [%[c24],%[c25],%[c26],%[c27]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[240:241], v[240:241], [%[c28],%[c29],%[c30],%[c31]] \n"
-"  buffer_load_dwordx4  acc[120:123], %[v_os_b7], s[12:15], 0 offen offset:2048 \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[242:243], v[242:243], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[244:245], v[244:245], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[246:247], v[246:247], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[248:249], v[248:249], [%[c28],%[c29],%[c30],%[c31]] \n"
-"  buffer_load_dwordx4  acc[124:127], %[v_os_b7], s[12:15], 0 offen offset:3072 \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[250:251], v[250:251], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[252:253], v[252:253], [%[c28],%[c29],%[c30],%[c31]] \n"
-_UK_MFMA_ "  [%[c28],%[c29],%[c30],%[c31]], acc[254:255], v[254:255], [%[c28],%[c29],%[c30],%[c31]]\n"
-"  v_mul_f32     %[c16], %[scale_0], %[c16]                            \n"
-"  v_mul_f32     %[c17], %[scale_0], %[c17]                            \n"
-"  v_mul_f32     %[c18], %[scale_0], %[c18]                            \n"
-"  v_mul_f32     %[c19], %[scale_0], %[c19]                            \n"
-"  v_mul_f32     %[c20], %[scale_1], %[c20]                            \n"
-"  v_mul_f32     %[c21], %[scale_1], %[c21]                            \n"
-"  v_mul_f32     %[c22], %[scale_1], %[c22]                            \n"
-"  v_mul_f32     %[c23], %[scale_1], %[c23]                            \n"
-"  v_mul_f32     %[c24], %[scale_0], %[c24]                            \n"
-"  v_mul_f32     %[c25], %[scale_0], %[c25]                            \n"
-"  v_mul_f32     %[c26], %[scale_0], %[c26]                            \n"
-"  v_mul_f32     %[c27], %[scale_0], %[c27]                            \n"
-"  v_mul_f32     %[c28], %[scale_1], %[c28]                            \n"
-"  v_mul_f32     %[c29], %[scale_1], %[c29]                            \n"
-"  v_mul_f32     %[c30], %[scale_1], %[c30]                            \n"
-"  v_mul_f32     %[c31], %[scale_1], %[c31]                            \n"
+    " s_mov_b32 s52, 0x07060302 ; v_perm\n"
+    " s_mov_b64 s[38:39], exec ; save current exec\n"
+    " s_mov_b32 s8,    %[s_res_o0] \n"
+    " s_mov_b32 s9,    %[s_res_o1] \n"
+    " s_mov_b32 s12,    %[s_res_b0] \n"
+    " s_mov_b32 s13,    %[s_res_b1] \n"
+    " s_mov_b32 s14,    %[s_res_b2] \n"
+    " s_mov_b32 s15,    %[s_res_b3] \n"
+    " ds_read_b64   v[128:129], %[v_sld_y_os] offset:0 + %[sld_a_base]                       \n"
+    " ds_read_b64   v[130:131], %[v_sld_y_os] offset:128 + %[sld_a_base]                     \n"
+    " ds_read_b64   v[132:133], %[v_sld_y_os] offset:1024 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[134:135], %[v_sld_y_os] offset:1152 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[136:137], %[v_sld_y_os] offset:2048 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[138:139], %[v_sld_y_os] offset:2176 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[140:141], %[v_sld_y_os] offset:3072 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[142:143], %[v_sld_y_os] offset:3200 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[144:145], %[v_sld_y_os] offset:4096 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[146:147], %[v_sld_y_os] offset:4224 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[148:149], %[v_sld_y_os] offset:5120 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[150:151], %[v_sld_y_os] offset:5248 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[152:153], %[v_sld_y_os] offset:6144 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[154:155], %[v_sld_y_os] offset:6272 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[156:157], %[v_sld_y_os] offset:7168 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[158:159], %[v_sld_y_os] offset:7296 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[160:161], %[v_sld_y_os] offset:8192 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[162:163], %[v_sld_y_os] offset:8320 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[164:165], %[v_sld_y_os] offset:9216 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[166:167], %[v_sld_y_os] offset:9344 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[168:169], %[v_sld_y_os] offset:10240 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[170:171], %[v_sld_y_os] offset:10368 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[172:173], %[v_sld_y_os] offset:11264 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[174:175], %[v_sld_y_os] offset:11392 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[176:177], %[v_sld_y_os] offset:12288 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[178:179], %[v_sld_y_os] offset:12416 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[180:181], %[v_sld_y_os] offset:13312 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[182:183], %[v_sld_y_os] offset:13440 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[184:185], %[v_sld_y_os] offset:14336 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[186:187], %[v_sld_y_os] offset:14464 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[188:189], %[v_sld_y_os] offset:15360 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[190:191], %[v_sld_y_os] offset:15488 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[192:193], %[v_sld_y_os] offset:16384 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[194:195], %[v_sld_y_os] offset:16512 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[196:197], %[v_sld_y_os] offset:17408 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[198:199], %[v_sld_y_os] offset:17536 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[200:201], %[v_sld_y_os] offset:18432 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[202:203], %[v_sld_y_os] offset:18560 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[204:205], %[v_sld_y_os] offset:19456 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[206:207], %[v_sld_y_os] offset:19584 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[208:209], %[v_sld_y_os] offset:20480 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[210:211], %[v_sld_y_os] offset:20608 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[212:213], %[v_sld_y_os] offset:21504 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[214:215], %[v_sld_y_os] offset:21632 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[216:217], %[v_sld_y_os] offset:22528 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[218:219], %[v_sld_y_os] offset:22656 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[220:221], %[v_sld_y_os] offset:23552 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[222:223], %[v_sld_y_os] offset:23680 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[224:225], %[v_sld_y_os] offset:24576 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[226:227], %[v_sld_y_os] offset:24704 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[228:229], %[v_sld_y_os] offset:25600 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[230:231], %[v_sld_y_os] offset:25728 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[232:233], %[v_sld_y_os] offset:26624 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[234:235], %[v_sld_y_os] offset:26752 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[236:237], %[v_sld_y_os] offset:27648 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[238:239], %[v_sld_y_os] offset:27776 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[240:241], %[v_sld_y_os] offset:28672 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[242:243], %[v_sld_y_os] offset:28800 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[244:245], %[v_sld_y_os] offset:29696 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[246:247], %[v_sld_y_os] offset:29824 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[248:249], %[v_sld_y_os] offset:30720 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[250:251], %[v_sld_y_os] offset:30848 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[252:253], %[v_sld_y_os] offset:31744 + %[sld_a_base]                    \n"
+    " ds_read_b64   v[254:255], %[v_sld_y_os] offset:31872 + %[sld_a_base]                    \n"
+    "  s_waitcnt 0                    \n"
+    "  buffer_load_dwordx4  acc[0:3], %[v_os_b0], s[12:15], 0 offen \n"
+    "  buffer_load_dwordx4  acc[4:7], %[v_os_b0], s[12:15], 0 offen offset:1024 \n"
+    "  buffer_load_dwordx4  acc[8:11], %[v_os_b0], s[12:15], 0 offen offset:2048 \n"
+    "  buffer_load_dwordx4  acc[12:15], %[v_os_b0], s[12:15], 0 offen offset:3072 \n"
+    "  buffer_load_dwordx4  acc[16:19], %[v_os_b1], s[12:15], 0 offen \n"
+    "  buffer_load_dwordx4  acc[20:23], %[v_os_b1], s[12:15], 0 offen offset:1024 \n"
+    "  buffer_load_dwordx4  acc[24:27], %[v_os_b1], s[12:15], 0 offen offset:2048 \n"
+    "  buffer_load_dwordx4  acc[28:31], %[v_os_b1], s[12:15], 0 offen offset:3072 \n"
+    "  buffer_load_dwordx4  acc[32:35], %[v_os_b2], s[12:15], 0 offen \n"
+    "  buffer_load_dwordx4  acc[36:39], %[v_os_b2], s[12:15], 0 offen offset:1024 \n"
+    "  buffer_load_dwordx4  acc[40:43], %[v_os_b2], s[12:15], 0 offen offset:2048 \n"
+    "  buffer_load_dwordx4  acc[44:47], %[v_os_b2], s[12:15], 0 offen offset:3072 \n"
+    "  buffer_load_dwordx4  acc[48:51], %[v_os_b3], s[12:15], 0 offen \n"
+    "  buffer_load_dwordx4  acc[52:55], %[v_os_b3], s[12:15], 0 offen offset:1024 \n"
+    "  buffer_load_dwordx4  acc[56:59], %[v_os_b3], s[12:15], 0 offen offset:2048 \n"
+    "  buffer_load_dwordx4  acc[60:63], %[v_os_b3], s[12:15], 0 offen offset:3072 \n"
+    "  buffer_load_dwordx4  acc[64:67], %[v_os_b4], s[12:15], 0 offen \n"
+    "  buffer_load_dwordx4  acc[68:71], %[v_os_b4], s[12:15], 0 offen offset:1024 \n"
+    "  buffer_load_dwordx4  acc[72:75], %[v_os_b4], s[12:15], 0 offen offset:2048 \n"
+    "  buffer_load_dwordx4  acc[76:79], %[v_os_b4], s[12:15], 0 offen offset:3072 \n"
+    "  buffer_load_dwordx4  acc[80:83], %[v_os_b5], s[12:15], 0 offen \n"
+    "  buffer_load_dwordx4  acc[84:87], %[v_os_b5], s[12:15], 0 offen offset:1024 \n"
+    "  buffer_load_dwordx4  acc[88:91], %[v_os_b5], s[12:15], 0 offen offset:2048 \n"
+    "  buffer_load_dwordx4  acc[92:95], %[v_os_b5], s[12:15], 0 offen offset:3072 \n"
+    "  buffer_load_dwordx4  acc[96:99], %[v_os_b6], s[12:15], 0 offen \n"
+    "  buffer_load_dwordx4  acc[100:103], %[v_os_b6], s[12:15], 0 offen offset:1024 \n"
+    "  buffer_load_dwordx4  acc[104:107], %[v_os_b6], s[12:15], 0 offen offset:2048 \n"
+    "  buffer_load_dwordx4  acc[108:111], %[v_os_b6], s[12:15], 0 offen offset:3072 \n"
+    "  buffer_load_dwordx4  acc[112:115], %[v_os_b7], s[12:15], 0 offen \n"
+    "  buffer_load_dwordx4  acc[116:119], %[v_os_b7], s[12:15], 0 offen offset:1024 \n"
+    "  buffer_load_dwordx4  acc[120:123], %[v_os_b7], s[12:15], 0 offen offset:2048 \n"
+    "  buffer_load_dwordx4  acc[124:127], %[v_os_b7], s[12:15], 0 offen offset:3072 \n"
+    "  s_cmp_gt_i32  %[s_loop_cnt] 1             ; move b with cond \n"
+    "  s_cselect_b32 s86, %[s_tile_os_b], 0                          \n"
+    "  s_add_u32     s12, s86, s12                                  \n"
+    "  s_addc_u32    s13, 0, s13                                    \n"
+    "  s_waitcnt 0                    \n"
+    "L_start%=:                    \n"
+    "  s_waitcnt     vmcnt(32)                               \n"
+    "  s_barrier                                             \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[0:1], v[128:129], 0 \n"
+    "  buffer_load_dwordx4  acc[128:131], %[v_os_b0], s[12:15], 0 offen \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[2:3], v[130:131], [%[c0], %[c1], %[c2], %[c3]] "
+    "\n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[4:5], v[132:133], [%[c0], %[c1], %[c2], "
+    "%[c3]] \n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[6:7], v[134:135], "
+    "[%[c0], %[c1], %[c2], %[c3]] \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[8:9], v[136:137], [%[c0], %[c1], %[c2], %[c3]] \n"
+    "  buffer_load_dwordx4  acc[132:135], %[v_os_b0], s[12:15], 0 offen offset:1024 \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[10:11], v[138:139], [%[c0], %[c1], %[c2], %[c3]] "
+    "\n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[12:13], v[140:141], [%[c0], %[c1], %[c2], "
+    "%[c3]] \n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[14:15], v[142:143], "
+    "[%[c0], %[c1], %[c2], %[c3]] \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[0:1], v[192:193], 0 \n"
+    "  buffer_load_dwordx4  acc[136:139], %[v_os_b0], s[12:15], 0 offen offset:2048 \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[2:3], v[194:195], [%[c4], %[c5], %[c6], %[c7]] "
+    "\n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[4:5], v[196:197], [%[c4], %[c5], %[c6], "
+    "%[c7]] \n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[6:7], v[198:199], "
+    "[%[c4], %[c5], %[c6], %[c7]] \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[8:9], v[200:201], [%[c4], %[c5], %[c6], %[c7]] \n"
+    "  buffer_load_dwordx4  acc[140:143], %[v_os_b0], s[12:15], 0 offen offset:3072 \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[10:11], v[202:203], [%[c4], %[c5], %[c6], %[c7]] "
+    "\n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[12:13], v[204:205], [%[c4], %[c5], %[c6], "
+    "%[c7]] \n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[14:15], v[206:207], "
+    "[%[c4], %[c5], %[c6], %[c7]] \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[16:17], v[128:129], 0 \n"
+    "  buffer_load_dwordx4  acc[144:147], %[v_os_b1], s[12:15], 0 offen \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[18:19], v[130:131], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[20:21], v[132:133], [%[c8], %[c9], "
+    "%[c10], %[c11]] \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[22:23], v[134:135], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[24:25], v[136:137], [%[c8], %[c9], %[c10], %[c11]] \n"
+    "  buffer_load_dwordx4  acc[148:151], %[v_os_b1], s[12:15], 0 offen offset:1024 \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[26:27], v[138:139], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[28:29], v[140:141], [%[c8], %[c9], "
+    "%[c10], %[c11]] \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[30:31], v[142:143], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[16:17], v[192:193], 0 \n"
+    "  buffer_load_dwordx4  acc[152:155], %[v_os_b1], s[12:15], 0 offen offset:2048 \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[18:19], v[194:195], [%[c12], %[c13], %[c14], %[c15]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[20:21], v[196:197], [%[c12], %[c13], "
+    "%[c14], %[c15]] \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[22:23], v[198:199], [%[c12], %[c13], %[c14], %[c15]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[24:25], v[200:201], [%[c12], %[c13], "
+    "%[c14], %[c15]] \n"
+    "  buffer_load_dwordx4  acc[156:159], %[v_os_b1], s[12:15], 0 offen offset:3072 \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[26:27], v[202:203], [%[c12], %[c13], %[c14], %[c15]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[28:29], v[204:205], [%[c12], %[c13], "
+    "%[c14], %[c15]] \n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[30:31], "
+    "v[206:207], [%[c12], %[c13], %[c14], %[c15]] \n"
+    "  s_waitcnt     vmcnt(32)                               \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[32:33], v[144:145], [%[c0], %[c1], %[c2], %[c3]] \n"
+    "  buffer_load_dwordx4  acc[160:163], %[v_os_b2], s[12:15], 0 offen \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[34:35], v[146:147], [%[c0], %[c1], %[c2], %[c3]] "
+    "\n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[36:37], v[148:149], [%[c0], %[c1], %[c2], "
+    "%[c3]] \n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[38:39], v[150:151], "
+    "[%[c0], %[c1], %[c2], %[c3]] \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[40:41], v[152:153], [%[c0], %[c1], %[c2], %[c3]] \n"
+    "  buffer_load_dwordx4  acc[164:167], %[v_os_b2], s[12:15], 0 offen offset:1024 \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[42:43], v[154:155], [%[c0], %[c1], %[c2], %[c3]] "
+    "\n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[44:45], v[156:157], [%[c0], %[c1], %[c2], "
+    "%[c3]] \n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[46:47], v[158:159], "
+    "[%[c0], %[c1], %[c2], %[c3]] \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[32:33], v[208:209], [%[c4], %[c5], %[c6], %[c7]] \n"
+    "  buffer_load_dwordx4  acc[168:171], %[v_os_b2], s[12:15], 0 offen offset:2048 \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[34:35], v[210:211], [%[c4], %[c5], %[c6], %[c7]] "
+    "\n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[36:37], v[212:213], [%[c4], %[c5], %[c6], "
+    "%[c7]] \n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[38:39], v[214:215], "
+    "[%[c4], %[c5], %[c6], %[c7]] \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[40:41], v[216:217], [%[c4], %[c5], %[c6], %[c7]] \n"
+    "  buffer_load_dwordx4  acc[172:175], %[v_os_b2], s[12:15], 0 offen offset:3072 \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[42:43], v[218:219], [%[c4], %[c5], %[c6], %[c7]] "
+    "\n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[44:45], v[220:221], [%[c4], %[c5], %[c6], "
+    "%[c7]] \n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[46:47], v[222:223], "
+    "[%[c4], %[c5], %[c6], %[c7]] \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[48:49], v[144:145], [%[c8], %[c9], %[c10], %[c11]] \n"
+    "  buffer_load_dwordx4  acc[176:179], %[v_os_b3], s[12:15], 0 offen \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[50:51], v[146:147], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[52:53], v[148:149], [%[c8], %[c9], "
+    "%[c10], %[c11]] \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[54:55], v[150:151], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[56:57], v[152:153], [%[c8], %[c9], %[c10], %[c11]] \n"
+    "  buffer_load_dwordx4  acc[180:183], %[v_os_b3], s[12:15], 0 offen offset:1024 \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[58:59], v[154:155], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[60:61], v[156:157], [%[c8], %[c9], "
+    "%[c10], %[c11]] \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[62:63], v[158:159], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[48:49], v[208:209], [%[c12], %[c13], "
+    "%[c14], %[c15]] \n"
+    "  buffer_load_dwordx4  acc[184:187], %[v_os_b3], s[12:15], 0 offen offset:2048 \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[50:51], v[210:211], [%[c12], %[c13], %[c14], %[c15]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[52:53], v[212:213], [%[c12], %[c13], "
+    "%[c14], %[c15]] \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[54:55], v[214:215], [%[c12], %[c13], %[c14], %[c15]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[56:57], v[216:217], [%[c12], %[c13], "
+    "%[c14], %[c15]] \n"
+    "  buffer_load_dwordx4  acc[188:191], %[v_os_b3], s[12:15], 0 offen offset:3072 \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[58:59], v[218:219], [%[c12], %[c13], %[c14], %[c15]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[60:61], v[220:221], [%[c12], %[c13], "
+    "%[c14], %[c15]] \n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[62:63], "
+    "v[222:223], [%[c12], %[c13], %[c14], %[c15]] \n"
+    "  s_waitcnt     vmcnt(32)                               \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[64:65], v[160:161], [%[c0], %[c1], %[c2], %[c3]] \n"
+    "  buffer_load_dwordx4  acc[192:195], %[v_os_b4], s[12:15], 0 offen \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[66:67], v[162:163], [%[c0], %[c1], %[c2], %[c3]] "
+    "\n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[68:69], v[164:165], [%[c0], %[c1], %[c2], "
+    "%[c3]] \n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[70:71], v[166:167], "
+    "[%[c0], %[c1], %[c2], %[c3]] \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[72:73], v[168:169], [%[c0], %[c1], %[c2], %[c3]] \n"
+    "  buffer_load_dwordx4  acc[196:199], %[v_os_b4], s[12:15], 0 offen offset:1024 \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[74:75], v[170:171], [%[c0], %[c1], %[c2], %[c3]] "
+    "\n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[76:77], v[172:173], [%[c0], %[c1], %[c2], "
+    "%[c3]] \n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[78:79], v[174:175], "
+    "[%[c0], %[c1], %[c2], %[c3]] \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[64:65], v[224:225], [%[c4], %[c5], %[c6], %[c7]] \n"
+    "  buffer_load_dwordx4  acc[200:203], %[v_os_b4], s[12:15], 0 offen offset:2048 \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[66:67], v[226:227], [%[c4], %[c5], %[c6], %[c7]] "
+    "\n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[68:69], v[228:229], [%[c4], %[c5], %[c6], "
+    "%[c7]] \n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[70:71], v[230:231], "
+    "[%[c4], %[c5], %[c6], %[c7]] \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[72:73], v[232:233], [%[c4], %[c5], %[c6], %[c7]] \n"
+    "  buffer_load_dwordx4  acc[204:207], %[v_os_b4], s[12:15], 0 offen offset:3072 \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[74:75], v[234:235], [%[c4], %[c5], %[c6], %[c7]] "
+    "\n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[76:77], v[236:237], [%[c4], %[c5], %[c6], "
+    "%[c7]] \n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[78:79], v[238:239], "
+    "[%[c4], %[c5], %[c6], %[c7]] \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[80:81], v[160:161], [%[c8], %[c9], %[c10], %[c11]] \n"
+    "  buffer_load_dwordx4  acc[208:211], %[v_os_b5], s[12:15], 0 offen \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[82:83], v[162:163], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[84:85], v[164:165], [%[c8], %[c9], "
+    "%[c10], %[c11]] \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[86:87], v[166:167], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[88:89], v[168:169], [%[c8], %[c9], %[c10], %[c11]] \n"
+    "  buffer_load_dwordx4  acc[212:215], %[v_os_b5], s[12:15], 0 offen offset:1024 \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[90:91], v[170:171], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[92:93], v[172:173], [%[c8], %[c9], "
+    "%[c10], %[c11]] \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[94:95], v[174:175], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[80:81], v[224:225], [%[c12], %[c13], "
+    "%[c14], %[c15]] \n"
+    "  buffer_load_dwordx4  acc[216:219], %[v_os_b5], s[12:15], 0 offen offset:2048 \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[82:83], v[226:227], [%[c12], %[c13], %[c14], %[c15]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[84:85], v[228:229], [%[c12], %[c13], "
+    "%[c14], %[c15]] \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[86:87], v[230:231], [%[c12], %[c13], %[c14], %[c15]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[88:89], v[232:233], [%[c12], %[c13], "
+    "%[c14], %[c15]] \n"
+    "  buffer_load_dwordx4  acc[220:223], %[v_os_b5], s[12:15], 0 offen offset:3072 \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[90:91], v[234:235], [%[c12], %[c13], %[c14], %[c15]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[92:93], v[236:237], [%[c12], %[c13], "
+    "%[c14], %[c15]] \n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[94:95], "
+    "v[238:239], [%[c12], %[c13], %[c14], %[c15]] \n"
+    "  s_waitcnt     vmcnt(32)                               \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[96:97], v[176:177], [%[c0], %[c1], %[c2], %[c3]] \n"
+    "  buffer_load_dwordx4  acc[224:227], %[v_os_b6], s[12:15], 0 offen \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[98:99], v[178:179], [%[c0], %[c1], %[c2], %[c3]] "
+    "\n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[100:101], v[180:181], [%[c0], %[c1], "
+    "%[c2], %[c3]] \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[102:103], v[182:183], [%[c0], %[c1], %[c2], %[c3]] "
+    "\n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[104:105], v[184:185], [%[c0], %[c1], %[c2], %[c3]] \n"
+    "  buffer_load_dwordx4  acc[228:231], %[v_os_b6], s[12:15], 0 offen offset:1024 \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[106:107], v[186:187], [%[c0], %[c1], %[c2], %[c3]] "
+    "\n" _UK_MFMA_ "  [%[c0], %[c1], %[c2], %[c3]], acc[108:109], v[188:189], [%[c0], %[c1], "
+    "%[c2], %[c3]] \n" _UK_MFMA_
+    "  [%[c0], %[c1], %[c2], %[c3]], acc[110:111], v[190:191], [%[c0], %[c1], %[c2], %[c3]] "
+    "\n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[96:97], v[240:241], [%[c4], %[c5], %[c6], %[c7]] \n"
+    "  buffer_load_dwordx4  acc[232:235], %[v_os_b6], s[12:15], 0 offen offset:2048 \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[98:99], v[242:243], [%[c4], %[c5], %[c6], %[c7]] "
+    "\n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[100:101], v[244:245], [%[c4], %[c5], "
+    "%[c6], %[c7]] \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[102:103], v[246:247], [%[c4], %[c5], %[c6], %[c7]] "
+    "\n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[104:105], v[248:249], [%[c4], %[c5], %[c6], %[c7]] \n"
+    "  buffer_load_dwordx4  acc[236:239], %[v_os_b6], s[12:15], 0 offen offset:3072 \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[106:107], v[250:251], [%[c4], %[c5], %[c6], %[c7]] "
+    "\n" _UK_MFMA_ "  [%[c4], %[c5], %[c6], %[c7]], acc[108:109], v[252:253], [%[c4], %[c5], "
+    "%[c6], %[c7]] \n" _UK_MFMA_
+    "  [%[c4], %[c5], %[c6], %[c7]], acc[110:111], v[254:255], [%[c4], %[c5], %[c6], %[c7]] "
+    "\n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[112:113], v[176:177], [%[c8], %[c9], %[c10], %[c11]] \n"
+    "  buffer_load_dwordx4  acc[240:243], %[v_os_b7], s[12:15], 0 offen \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[114:115], v[178:179], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[116:117], v[180:181], [%[c8], %[c9], "
+    "%[c10], %[c11]] \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[118:119], v[182:183], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[120:121], v[184:185], [%[c8], %[c9], %[c10], %[c11]] \n"
+    "  buffer_load_dwordx4  acc[244:247], %[v_os_b7], s[12:15], 0 offen offset:1024 \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[122:123], v[186:187], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_ "  [%[c8], %[c9], %[c10], %[c11]], acc[124:125], v[188:189], [%[c8], %[c9], "
+    "%[c10], %[c11]] \n" _UK_MFMA_
+    "  [%[c8], %[c9], %[c10], %[c11]], acc[126:127], v[190:191], [%[c8], %[c9], %[c10], %[c11]] "
+    "\n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[112:113], v[240:241], [%[c12], %[c13], "
+    "%[c14], %[c15]] \n"
+    "  buffer_load_dwordx4  acc[248:251], %[v_os_b7], s[12:15], 0 offen offset:2048 \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[114:115], v[242:243], [%[c12], %[c13], %[c14], "
+    "%[c15]] \n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[116:117], v[244:245], [%[c12], "
+    "%[c13], %[c14], %[c15]] \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[118:119], v[246:247], [%[c12], %[c13], %[c14], "
+    "%[c15]] \n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[120:121], v[248:249], [%[c12], "
+    "%[c13], %[c14], %[c15]] \n"
+    "  buffer_load_dwordx4  acc[252:255], %[v_os_b7], s[12:15], 0 offen offset:3072 \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[122:123], v[250:251], [%[c12], %[c13], %[c14], "
+    "%[c15]] \n" _UK_MFMA_ "  [%[c12], %[c13], %[c14], %[c15]], acc[124:125], v[252:253], [%[c12], "
+    "%[c13], %[c14], %[c15]] \n" _UK_MFMA_
+    "  [%[c12], %[c13], %[c14], %[c15]], acc[126:127], v[254:255], [%[c12], %[c13], %[c14], "
+    "%[c15]]\n"
+    "  v_mul_f32     %[c0], %[scale_0], %[c0]                            \n"
+    "  v_mul_f32     %[c1], %[scale_0], %[c1]                            \n"
+    "  v_mul_f32     %[c2], %[scale_0], %[c2]                            \n"
+    "  v_mul_f32     %[c3], %[scale_0], %[c3]                            \n"
+    "  v_mul_f32     %[c4], %[scale_1], %[c4]                            \n"
+    "  v_mul_f32     %[c5], %[scale_1], %[c5]                            \n"
+    "  v_mul_f32     %[c6], %[scale_1], %[c6]                            \n"
+    "  v_mul_f32     %[c7], %[scale_1], %[c7]                            \n"
+    "  v_mul_f32     %[c8], %[scale_0], %[c8]                            \n"
+    "  v_mul_f32     %[c9], %[scale_0], %[c9]                            \n"
+    "  v_mul_f32     %[c10], %[scale_0], %[c10]                            \n"
+    "  v_mul_f32     %[c11], %[scale_0], %[c11]                            \n"
+    "  v_mul_f32     %[c12], %[scale_1], %[c12]                            \n"
+    "  v_mul_f32     %[c13], %[scale_1], %[c13]                            \n"
+    "  v_mul_f32     %[c14], %[scale_1], %[c14]                            \n"
+    "  v_mul_f32     %[c15], %[scale_1], %[c15]                            \n" _UK_PK_CVT_(
+        "%[c0]", "%[c1]", "%[c0]") _UK_PK_CVT_("%[c2]", "%[c3]", "%[c1]")
+        _UK_PK_CVT_("%[c4]", "%[c5]", "%[c2]") _UK_PK_CVT_("%[c6]", "%[c7]", "%[c3]") _UK_PK_CVT_(
+            "%[c8]", "%[c9]", "%[c4]") _UK_PK_CVT_("%[c10]", "%[c11]", "%[c5]")
+            _UK_PK_CVT_("%[c12]", "%[c13]", "%[c6]") _UK_PK_CVT_(
+                "%[c14]",
+                "%[c15]",
+                "%[c7]") "  ;------------------------------  \n"
+                         "  ds_write_b64  %[v_sfl_sst], [%[c0],%[c1]] offset:0    + %[shfl_base]   "
+                         "            \n"
+                         "  ds_write_b64  %[v_sfl_sst], [%[c2],%[c3]] offset:4352 + %[shfl_base]   "
+                         "            \n"
+                         "  ds_write_b64  %[v_sfl_sst], [%[c4],%[c5]] offset:2176 + %[shfl_base]   "
+                         "            \n"
+                         "  ds_write_b64  %[v_sfl_sst], [%[c6],%[c7]] offset:6528 + %[shfl_base]   "
+                         "            \n"
+                         "  s_waitcnt     lgkmcnt(0)                              \n"
+                         "  s_barrier                                             \n"
+                         "  ds_read_b32   %[c0], %[v_sfl_sld] offset:0    + %[shfl_base]           "
+                         "         \n"
+                         "  ds_read_b32   %[c1], %[v_sfl_sld] offset:32   + %[shfl_base]           "
+                         "         \n"
+                         "  ds_read_b32   %[c2], %[v_sfl_sld] offset:64   + %[shfl_base]           "
+                         "         \n"
+                         "  ds_read_b32   %[c3], %[v_sfl_sld] offset:96   + %[shfl_base]           "
+                         "         \n"
+                         "  ds_read_b32   %[c4], %[v_sfl_sld] offset:4352 + %[shfl_base]           "
+                         "         \n"
+                         "  ds_read_b32   %[c5], %[v_sfl_sld] offset:4384 + %[shfl_base]           "
+                         "         \n"
+                         "  ds_read_b32   %[c6], %[v_sfl_sld] offset:4416 + %[shfl_base]           "
+                         "         \n"
+                         "  ds_read_b32   %[c7], %[v_sfl_sld] offset:4448 + %[shfl_base]           "
+                         "         \n"
+                         "  s_waitcnt     lgkmcnt(0)                              \n"
+                         "  s_mov_b64     exec, %[s_execflag_0]                    "
+                         "\n" _UK_ATOMIC_ADD_ "   %[v_os_o0], %[c0], s[8:9]  \n"
+                         "  s_mov_b64     exec, %[s_execflag_1]                    "
+                         "\n" _UK_ATOMIC_ADD_ "   %[v_os_o1], %[c1], s[8:9]  \n"
+                         "  s_mov_b64     exec, %[s_execflag_2]                    "
+                         "\n" _UK_ATOMIC_ADD_ "   %[v_os_o2], %[c2], s[8:9]  \n"
+                         "  s_mov_b64     exec, %[s_execflag_3]                    "
+                         "\n" _UK_ATOMIC_ADD_ "   %[v_os_o3], %[c3], s[8:9]  \n"
+                         "  s_mov_b64     exec, %[s_execflag_4]                    "
+                         "\n" _UK_ATOMIC_ADD_ "   %[v_os_o4], %[c4], s[8:9]  \n"
+                         "  s_mov_b64     exec, %[s_execflag_5]                    "
+                         "\n" _UK_ATOMIC_ADD_ "   %[v_os_o5], %[c5], s[8:9]  \n"
+                         "  s_mov_b64     exec, %[s_execflag_6]                    "
+                         "\n" _UK_ATOMIC_ADD_ "   %[v_os_o6], %[c6], s[8:9]  \n"
+                         "  s_mov_b64     exec, %[s_execflag_7]                    "
+                         "\n" _UK_ATOMIC_ADD_ "   %[v_os_o7], %[c7], s[8:9]  \n"
+                         "  s_mov_b64     exec, s[38:39]                           \n"
+                         "  s_sub_i32     %[s_loop_cnt], %[s_loop_cnt], 1     ; k--      \n"
+                         "  s_cmp_gt_i32  %[s_loop_cnt] 0                                \n"
+                         "  s_cbranch_scc0 L_end%=                                       \n"
+                         "  s_cmp_gt_i32  %[s_loop_cnt] 1             ; move b with cond \n"
+                         "  s_cselect_b32 s86, %[s_tile_os_b], 0                          \n"
+                         "  s_add_u32     s12, s86, s12                                  \n"
+                         "  s_addc_u32    s13, 0, s13                                    \n"
+                         "  s_add_u32     s8, %[s_tile_os_o], s8                             \n"
+                         "  s_addc_u32    s9, 0, s9                               \n"
+                         "  s_waitcnt     vmcnt(32)                               \n"
+                         "  s_barrier                                             \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[128:129], v[128:129], 0 \n"
+                         "  buffer_load_dwordx4  acc[0:3], %[v_os_b0], s[12:15], 0 offen "
+                         "\n" _UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[130:131], "
+                         "v[130:131], [%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[132:133], v[132:133], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[134:135], v[134:135], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[136:137], v[136:137], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n"
+                         "  buffer_load_dwordx4  acc[4:7], %[v_os_b0], s[12:15], 0 offen "
+                         "offset:1024 \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[138:139], v[138:139], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[140:141], v[140:141], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[142:143], v[142:143], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[128:129], v[192:193], 0 \n"
+                         "  buffer_load_dwordx4  acc[8:11], %[v_os_b0], s[12:15], 0 offen "
+                         "offset:2048 \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[130:131], v[194:195], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[132:133], v[196:197], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[134:135], v[198:199], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[136:137], v[200:201], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n"
+                         "  buffer_load_dwordx4  acc[12:15], %[v_os_b0], s[12:15], 0 offen "
+                         "offset:3072 \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[138:139], v[202:203], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[140:141], v[204:205], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[142:143], v[206:207], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[144:145], v[128:129], 0 \n"
+                         "  buffer_load_dwordx4  acc[16:19], %[v_os_b1], s[12:15], 0 offen "
+                         "\n" _UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[146:147], "
+                         "v[130:131], [%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[148:149], v[132:133], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[150:151], v[134:135], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[152:153], v[136:137], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n"
+                         "  buffer_load_dwordx4  acc[20:23], %[v_os_b1], s[12:15], 0 offen "
+                         "offset:1024 \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[154:155], v[138:139], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[156:157], v[140:141], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[158:159], v[142:143], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[144:145], v[192:193], 0 \n"
+                         "  buffer_load_dwordx4  acc[24:27], %[v_os_b1], s[12:15], 0 offen "
+                         "offset:2048 \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[146:147], v[194:195], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[148:149], v[196:197], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[150:151], v[198:199], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[152:153], v[200:201], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n"
+                         "  buffer_load_dwordx4  acc[28:31], %[v_os_b1], s[12:15], 0 offen "
+                         "offset:3072 \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[154:155], v[202:203], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[156:157], v[204:205], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[158:159], v[206:207], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n"
+                         "  s_waitcnt     vmcnt(32)                               \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[160:161], v[144:145], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n"
+                         "  buffer_load_dwordx4  acc[32:35], %[v_os_b2], s[12:15], 0 offen "
+                         "\n" _UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[162:163], "
+                         "v[146:147], [%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[164:165], v[148:149], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[166:167], v[150:151], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[168:169], v[152:153], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n"
+                         "  buffer_load_dwordx4  acc[36:39], %[v_os_b2], s[12:15], 0 offen "
+                         "offset:1024 \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[170:171], v[154:155], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[172:173], v[156:157], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[174:175], v[158:159], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[160:161], v[208:209], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n"
+                         "  buffer_load_dwordx4  acc[40:43], %[v_os_b2], s[12:15], 0 offen "
+                         "offset:2048 \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[162:163], v[210:211], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[164:165], v[212:213], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[166:167], v[214:215], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[168:169], v[216:217], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n"
+                         "  buffer_load_dwordx4  acc[44:47], %[v_os_b2], s[12:15], 0 offen "
+                         "offset:3072 \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[170:171], v[218:219], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[172:173], v[220:221], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[174:175], v[222:223], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[176:177], v[144:145], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n"
+                         "  buffer_load_dwordx4  acc[48:51], %[v_os_b3], s[12:15], 0 offen "
+                         "\n" _UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[178:179], "
+                         "v[146:147], [%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[180:181], v[148:149], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[182:183], v[150:151], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[184:185], v[152:153], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n"
+                         "  buffer_load_dwordx4  acc[52:55], %[v_os_b3], s[12:15], 0 offen "
+                         "offset:1024 \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[186:187], v[154:155], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[188:189], v[156:157], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[190:191], v[158:159], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[176:177], v[208:209], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n"
+                         "  buffer_load_dwordx4  acc[56:59], %[v_os_b3], s[12:15], 0 offen "
+                         "offset:2048 \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[178:179], v[210:211], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[180:181], v[212:213], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[182:183], v[214:215], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[184:185], v[216:217], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n"
+                         "  buffer_load_dwordx4  acc[60:63], %[v_os_b3], s[12:15], 0 offen "
+                         "offset:3072 \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[186:187], v[218:219], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[188:189], v[220:221], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[190:191], v[222:223], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n"
+                         "  s_waitcnt     vmcnt(32)                               \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[192:193], v[160:161], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n"
+                         "  buffer_load_dwordx4  acc[64:67], %[v_os_b4], s[12:15], 0 offen "
+                         "\n" _UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[194:195], "
+                         "v[162:163], [%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[196:197], v[164:165], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[198:199], v[166:167], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[200:201], v[168:169], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n"
+                         "  buffer_load_dwordx4  acc[68:71], %[v_os_b4], s[12:15], 0 offen "
+                         "offset:1024 \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[202:203], v[170:171], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[204:205], v[172:173], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[206:207], v[174:175], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[192:193], v[224:225], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n"
+                         "  buffer_load_dwordx4  acc[72:75], %[v_os_b4], s[12:15], 0 offen "
+                         "offset:2048 \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[194:195], v[226:227], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[196:197], v[228:229], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[198:199], v[230:231], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[200:201], v[232:233], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n"
+                         "  buffer_load_dwordx4  acc[76:79], %[v_os_b4], s[12:15], 0 offen "
+                         "offset:3072 \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[202:203], v[234:235], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[204:205], v[236:237], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[206:207], v[238:239], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[208:209], v[160:161], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n"
+                         "  buffer_load_dwordx4  acc[80:83], %[v_os_b5], s[12:15], 0 offen "
+                         "\n" _UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[210:211], "
+                         "v[162:163], [%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[212:213], v[164:165], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[214:215], v[166:167], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[216:217], v[168:169], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n"
+                         "  buffer_load_dwordx4  acc[84:87], %[v_os_b5], s[12:15], 0 offen "
+                         "offset:1024 \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[218:219], v[170:171], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[220:221], v[172:173], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[222:223], v[174:175], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[208:209], v[224:225], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n"
+                         "  buffer_load_dwordx4  acc[88:91], %[v_os_b5], s[12:15], 0 offen "
+                         "offset:2048 \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[210:211], v[226:227], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[212:213], v[228:229], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[214:215], v[230:231], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[216:217], v[232:233], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n"
+                         "  buffer_load_dwordx4  acc[92:95], %[v_os_b5], s[12:15], 0 offen "
+                         "offset:3072 \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[218:219], v[234:235], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[220:221], v[236:237], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[222:223], v[238:239], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n"
+                         "  s_waitcnt     vmcnt(32)                               \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[224:225], v[176:177], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n"
+                         "  buffer_load_dwordx4  acc[96:99], %[v_os_b6], s[12:15], 0 offen "
+                         "\n" _UK_MFMA_ "  [%[c16],%[c17],%[c18],%[c19]], acc[226:227], "
+                         "v[178:179], [%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[228:229], v[180:181], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[230:231], v[182:183], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[232:233], v[184:185], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n"
+                         "  buffer_load_dwordx4  acc[100:103], %[v_os_b6], s[12:15], 0 offen "
+                         "offset:1024 \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[234:235], v[186:187], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[236:237], v[188:189], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c16],%[c17],%[c18],%[c19]], acc[238:239], v[190:191], "
+                         "[%[c16],%[c17],%[c18],%[c19]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[224:225], v[240:241], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n"
+                         "  buffer_load_dwordx4  acc[104:107], %[v_os_b6], s[12:15], 0 offen "
+                         "offset:2048 \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[226:227], v[242:243], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[228:229], v[244:245], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[230:231], v[246:247], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[232:233], v[248:249], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n"
+                         "  buffer_load_dwordx4  acc[108:111], %[v_os_b6], s[12:15], 0 offen "
+                         "offset:3072 \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[234:235], v[250:251], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[236:237], v[252:253], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c20],%[c21],%[c22],%[c23]], acc[238:239], v[254:255], "
+                         "[%[c20],%[c21],%[c22],%[c23]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[240:241], v[176:177], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n"
+                         "  buffer_load_dwordx4  acc[112:115], %[v_os_b7], s[12:15], 0 offen "
+                         "\n" _UK_MFMA_ "  [%[c24],%[c25],%[c26],%[c27]], acc[242:243], "
+                         "v[178:179], [%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[244:245], v[180:181], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[246:247], v[182:183], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[248:249], v[184:185], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n"
+                         "  buffer_load_dwordx4  acc[116:119], %[v_os_b7], s[12:15], 0 offen "
+                         "offset:1024 \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[250:251], v[186:187], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[252:253], v[188:189], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c24],%[c25],%[c26],%[c27]], acc[254:255], v[190:191], "
+                         "[%[c24],%[c25],%[c26],%[c27]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[240:241], v[240:241], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n"
+                         "  buffer_load_dwordx4  acc[120:123], %[v_os_b7], s[12:15], 0 offen "
+                         "offset:2048 \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[242:243], v[242:243], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[244:245], v[244:245], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[246:247], v[246:247], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[248:249], v[248:249], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n"
+                         "  buffer_load_dwordx4  acc[124:127], %[v_os_b7], s[12:15], 0 offen "
+                         "offset:3072 \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[250:251], v[250:251], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[252:253], v[252:253], "
+                         "[%[c28],%[c29],%[c30],%[c31]] \n" _UK_MFMA_
+                         "  [%[c28],%[c29],%[c30],%[c31]], acc[254:255], v[254:255], "
+                         "[%[c28],%[c29],%[c30],%[c31]]\n"
+                         "  v_mul_f32     %[c16], %[scale_0], %[c16]                            \n"
+                         "  v_mul_f32     %[c17], %[scale_0], %[c17]                            \n"
+                         "  v_mul_f32     %[c18], %[scale_0], %[c18]                            \n"
+                         "  v_mul_f32     %[c19], %[scale_0], %[c19]                            \n"
+                         "  v_mul_f32     %[c20], %[scale_1], %[c20]                            \n"
+                         "  v_mul_f32     %[c21], %[scale_1], %[c21]                            \n"
+                         "  v_mul_f32     %[c22], %[scale_1], %[c22]                            \n"
+                         "  v_mul_f32     %[c23], %[scale_1], %[c23]                            \n"
+                         "  v_mul_f32     %[c24], %[scale_0], %[c24]                            \n"
+                         "  v_mul_f32     %[c25], %[scale_0], %[c25]                            \n"
+                         "  v_mul_f32     %[c26], %[scale_0], %[c26]                            \n"
+                         "  v_mul_f32     %[c27], %[scale_0], %[c27]                            \n"
+                         "  v_mul_f32     %[c28], %[scale_1], %[c28]                            \n"
+                         "  v_mul_f32     %[c29], %[scale_1], %[c29]                            \n"
+                         "  v_mul_f32     %[c30], %[scale_1], %[c30]                            \n"
+                         "  v_mul_f32     %[c31], %[scale_1], %[c31]                            \n"
 
-_UK_PK_CVT_("%[c16]",  "%[c17]",  "%[c16]")
-_UK_PK_CVT_("%[c18]",  "%[c19]",  "%[c17]")
-_UK_PK_CVT_("%[c20]",  "%[c21]",  "%[c18]")
-_UK_PK_CVT_("%[c22]",  "%[c23]",  "%[c19]")
-_UK_PK_CVT_("%[c24]",  "%[c25]",  "%[c20]")
-_UK_PK_CVT_("%[c26]",  "%[c27]",  "%[c21]")
-_UK_PK_CVT_("%[c28]",  "%[c29]",  "%[c22]")
-_UK_PK_CVT_("%[c30]",  "%[c31]",  "%[c23]")
+    _UK_PK_CVT_("%[c16]", "%[c17]", "%[c16]") _UK_PK_CVT_("%[c18]", "%[c19]", "%[c17]") _UK_PK_CVT_(
+        "%[c20]", "%[c21]", "%[c18]") _UK_PK_CVT_("%[c22]", "%[c23]", "%[c19]")
+        _UK_PK_CVT_("%[c24]", "%[c25]", "%[c20]") _UK_PK_CVT_(
+            "%[c26]", "%[c27]", "%[c21]") _UK_PK_CVT_("%[c28]",
+                                                      "%[c29]",
+                                                      "%[c22]") _UK_PK_CVT_("%[c30]",
+                                                                            "%[c31]",
+                                                                            "%[c23]")
 
-"  ;------------------------------  \n"
-"  ds_write_b64  %[v_sfl_sst], [%[c16],%[c17]] offset:0    + %[shfl_base]         \n"
-"  ds_write_b64  %[v_sfl_sst], [%[c18],%[c19]] offset:4352 + %[shfl_base]         \n"
-"  ds_write_b64  %[v_sfl_sst], [%[c20],%[c21]] offset:2176 + %[shfl_base]         \n"
-"  ds_write_b64  %[v_sfl_sst], [%[c22],%[c23]] offset:6528 + %[shfl_base]         \n"
-"  s_waitcnt     lgkmcnt(0)                              \n"
-"  s_barrier                                             \n"
-"  ds_read_b32   %[c16], %[v_sfl_sld] offset:0    + %[shfl_base]                  \n"
-"  ds_read_b32   %[c17], %[v_sfl_sld] offset:32   + %[shfl_base]                  \n"
-"  ds_read_b32   %[c18], %[v_sfl_sld] offset:64   + %[shfl_base]                  \n"
-"  ds_read_b32   %[c19], %[v_sfl_sld] offset:96   + %[shfl_base]                  \n"
-"  ds_read_b32   %[c20], %[v_sfl_sld] offset:4352 + %[shfl_base]                  \n"
-"  ds_read_b32   %[c21], %[v_sfl_sld] offset:4384 + %[shfl_base]                  \n"
-"  ds_read_b32   %[c22], %[v_sfl_sld] offset:4416 + %[shfl_base]                  \n"
-"  ds_read_b32   %[c23], %[v_sfl_sld] offset:4448 + %[shfl_base]                  \n"
-"  s_waitcnt     lgkmcnt(0)                              \n"
-"  s_mov_b64     exec, %[s_execflag_0]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o0], %[c16], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_1]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o1], %[c17], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_2]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o2], %[c18], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_3]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o3], %[c19], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_4]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o4], %[c20], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_5]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o5], %[c21], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_6]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o6], %[c22], s[8:9]  \n"
-"  s_mov_b64     exec, %[s_execflag_7]                    \n"
-_UK_ATOMIC_ADD_ "   %[v_os_o7], %[c23], s[8:9]  \n"
-"  s_mov_b64     exec, s[38:39]                           \n"
-"  s_sub_i32     %[s_loop_cnt], %[s_loop_cnt], 1     ; k--      \n"
-"  s_cmp_gt_i32  %[s_loop_cnt] 0                                \n"
-"  s_cbranch_scc0 L_end%=                                       \n"
-"  s_cmp_gt_i32  %[s_loop_cnt] 1             ; move b with cond \n"
-"  s_cselect_b32 s86, %[s_tile_os_b], 0                          \n"
-"  s_add_u32     s12, s86, s12                                  \n"
-"  s_addc_u32    s13, 0, s13                                    \n"
-"  s_add_u32     s8, %[s_tile_os_o], s8                             \n"
-"  s_addc_u32    s9, 0, s9                               \n"
-"  s_branch      L_start%=          \n"
-"L_end%=:                                                \n"
+            "  ;------------------------------  \n"
+            "  ds_write_b64  %[v_sfl_sst], [%[c16],%[c17]] offset:0    + %[shfl_base]         \n"
+            "  ds_write_b64  %[v_sfl_sst], [%[c18],%[c19]] offset:4352 + %[shfl_base]         \n"
+            "  ds_write_b64  %[v_sfl_sst], [%[c20],%[c21]] offset:2176 + %[shfl_base]         \n"
+            "  ds_write_b64  %[v_sfl_sst], [%[c22],%[c23]] offset:6528 + %[shfl_base]         \n"
+            "  s_waitcnt     lgkmcnt(0)                              \n"
+            "  s_barrier                                             \n"
+            "  ds_read_b32   %[c16], %[v_sfl_sld] offset:0    + %[shfl_base]                  \n"
+            "  ds_read_b32   %[c17], %[v_sfl_sld] offset:32   + %[shfl_base]                  \n"
+            "  ds_read_b32   %[c18], %[v_sfl_sld] offset:64   + %[shfl_base]                  \n"
+            "  ds_read_b32   %[c19], %[v_sfl_sld] offset:96   + %[shfl_base]                  \n"
+            "  ds_read_b32   %[c20], %[v_sfl_sld] offset:4352 + %[shfl_base]                  \n"
+            "  ds_read_b32   %[c21], %[v_sfl_sld] offset:4384 + %[shfl_base]                  \n"
+            "  ds_read_b32   %[c22], %[v_sfl_sld] offset:4416 + %[shfl_base]                  \n"
+            "  ds_read_b32   %[c23], %[v_sfl_sld] offset:4448 + %[shfl_base]                  \n"
+            "  s_waitcnt     lgkmcnt(0)                              \n"
+            "  s_mov_b64     exec, %[s_execflag_0]                    \n" _UK_ATOMIC_ADD_
+            "   %[v_os_o0], %[c16], s[8:9]  \n"
+            "  s_mov_b64     exec, %[s_execflag_1]                    \n" _UK_ATOMIC_ADD_
+            "   %[v_os_o1], %[c17], s[8:9]  \n"
+            "  s_mov_b64     exec, %[s_execflag_2]                    \n" _UK_ATOMIC_ADD_
+            "   %[v_os_o2], %[c18], s[8:9]  \n"
+            "  s_mov_b64     exec, %[s_execflag_3]                    \n" _UK_ATOMIC_ADD_
+            "   %[v_os_o3], %[c19], s[8:9]  \n"
+            "  s_mov_b64     exec, %[s_execflag_4]                    \n" _UK_ATOMIC_ADD_
+            "   %[v_os_o4], %[c20], s[8:9]  \n"
+            "  s_mov_b64     exec, %[s_execflag_5]                    \n" _UK_ATOMIC_ADD_
+            "   %[v_os_o5], %[c21], s[8:9]  \n"
+            "  s_mov_b64     exec, %[s_execflag_6]                    \n" _UK_ATOMIC_ADD_
+            "   %[v_os_o6], %[c22], s[8:9]  \n"
+            "  s_mov_b64     exec, %[s_execflag_7]                    \n" _UK_ATOMIC_ADD_
+            "   %[v_os_o7], %[c23], s[8:9]  \n"
+            "  s_mov_b64     exec, s[38:39]                           \n"
+            "  s_sub_i32     %[s_loop_cnt], %[s_loop_cnt], 1     ; k--      \n"
+            "  s_cmp_gt_i32  %[s_loop_cnt] 0                                \n"
+            "  s_cbranch_scc0 L_end%=                                       \n"
+            "  s_cmp_gt_i32  %[s_loop_cnt] 1             ; move b with cond \n"
+            "  s_cselect_b32 s86, %[s_tile_os_b], 0                          \n"
+            "  s_add_u32     s12, s86, s12                                  \n"
+            "  s_addc_u32    s13, 0, s13                                    \n"
+            "  s_add_u32     s8, %[s_tile_os_o], s8                             \n"
+            "  s_addc_u32    s9, 0, s9                               \n"
+            "  s_branch      L_start%=          \n"
+            "L_end%=:                                                \n"
 
 #undef _UK_MFMA_
 #undef _UK_PK_CVT_

include/ck_tile/ops/flatmm/block/uk/flatmm_sn_uk_gfx9_32x128x512_1x4x1_16x16x16_itl.inc

+#ifndef CK_TILE_FLATMM_UK_MFMA
+#define CK_TILE_FLATMM_UK_MFMA CK_TILE_FLATMM_UK_MFMA_BF16
+#endif
+
+#if CK_TILE_FLATMM_UK_MFMA == CK_TILE_FLATMM_UK_MFMA_BF16
+#define _UK_MFMA_ "v_mfma_f32_16x16x16_bf16"
+
+#define _UK_PK_CVT_(x0_, x1_, y_)                       \
+    " v_cmp_u_f32 s[36:37], " x0_ ", " x0_ " \n"        \
+    " v_add3_u32 v50, " x0_ ", %[v_nan_lo], 1 \n"       \
+    " v_cndmask_b32 v54, v50, %[v_nan_hi], s[36:37] \n" \
+    " v_cmp_u_f32 s[36:37], " x1_ ", " x1_ " \n"        \
+    " v_add3_u32 v50, " x1_ ", %[v_nan_lo], 1 \n"       \
+    " v_cndmask_b32 v55, v50, %[v_nan_hi], s[36:37] \n" \
+    " v_perm_b32 " y_ ", v55, v54, s52 \n"
+
+#define _UK_ATOMIC_ADD_ "global_atomic_pk_add_bf16"
+
+#elif CK_TILE_FLATMM_UK_MFMA == CK_TILE_FLATMM_UK_MFMA_FP16
+#define _UK_MFMA_ "v_mfma_f32_16x16x16_f16"
+
+#define _UK_PK_CVT_(x0_, x1_, y_)    \
+    " v_cvt_f16_f32 v54, " x0_ " \n" \
+    " v_cvt_f16_f32 v55, " x1_ " \n" \
+    " v_pack_b32_f16 " y_ ", v54, v55 \n"
+
+#define _UK_ATOMIC_ADD_ "global_atomic_pk_add_f16"
+
+#endif
+
+";-------------------------------------------------------------\n"
+    " s_mov_b32 s52, 0x07060302 ; v_perm\n"
+    " s_mov_b64 s[38:39], exec ; save current exec\n"
+    " s_mov_b32 s8, %[s_res_o0] \n"
+    " s_mov_b32 s9, %[s_res_o1] \n"
+    " s_mov_b32 s12, %[s_res_b0] \n"
+    " s_mov_b32 s13, %[s_res_b1] \n"
+    " s_mov_b32 s14, %[s_res_b2] \n"
+    " s_mov_b32 s15, %[s_res_b3] \n"
+    " s_mov_b32 s59, 0 \n"
+    " ds_read_b64 v[128:129], %[v_sld_y_os] offset:0 + %[sld_a_base] \n"
+    " ds_read_b64 v[130:131], %[v_sld_y_os] offset:128 + %[sld_a_base] \n"
+    " ds_read_b64 v[132:133], %[v_sld_y_os] offset:1024 + %[sld_a_base] \n"
+    " ds_read_b64 v[134:135], %[v_sld_y_os] offset:1152 + %[sld_a_base] \n"
+    " ds_read_b64 v[136:137], %[v_sld_y_os] offset:2048 + %[sld_a_base] \n"
+    " ds_read_b64 v[138:139], %[v_sld_y_os] offset:2176 + %[sld_a_base] \n"
+    " ds_read_b64 v[140:141], %[v_sld_y_os] offset:3072 + %[sld_a_base] \n"
+    " ds_read_b64 v[142:143], %[v_sld_y_os] offset:3200 + %[sld_a_base] \n"
+    " ds_read_b64 v[144:145], %[v_sld_y_os] offset:4096 + %[sld_a_base] \n"
+    " ds_read_b64 v[146:147], %[v_sld_y_os] offset:4224 + %[sld_a_base] \n"
+    " ds_read_b64 v[148:149], %[v_sld_y_os] offset:5120 + %[sld_a_base] \n"
+    " ds_read_b64 v[150:151], %[v_sld_y_os] offset:5248 + %[sld_a_base] \n"
+    " ds_read_b64 v[152:153], %[v_sld_y_os] offset:6144 + %[sld_a_base] \n"
+    " ds_read_b64 v[154:155], %[v_sld_y_os] offset:6272 + %[sld_a_base] \n"
+    " ds_read_b64 v[156:157], %[v_sld_y_os] offset:7168 + %[sld_a_base] \n"
+    " ds_read_b64 v[158:159], %[v_sld_y_os] offset:7296 + %[sld_a_base] \n"
+    " ds_read_b64 v[160:161], %[v_sld_y_os] offset:8192 + %[sld_a_base] \n"
+    " ds_read_b64 v[162:163], %[v_sld_y_os] offset:8320 + %[sld_a_base] \n"
+    " ds_read_b64 v[164:165], %[v_sld_y_os] offset:9216 + %[sld_a_base] \n"
+    " ds_read_b64 v[166:167], %[v_sld_y_os] offset:9344 + %[sld_a_base] \n"
+    " ds_read_b64 v[168:169], %[v_sld_y_os] offset:10240 + %[sld_a_base] \n"
+    " ds_read_b64 v[170:171], %[v_sld_y_os] offset:10368 + %[sld_a_base] \n"
+    " ds_read_b64 v[172:173], %[v_sld_y_os] offset:11264 + %[sld_a_base] \n"
+    " ds_read_b64 v[174:175], %[v_sld_y_os] offset:11392 + %[sld_a_base] \n"
+    " ds_read_b64 v[176:177], %[v_sld_y_os] offset:12288 + %[sld_a_base] \n"
+    " ds_read_b64 v[178:179], %[v_sld_y_os] offset:12416 + %[sld_a_base] \n"
+    " ds_read_b64 v[180:181], %[v_sld_y_os] offset:13312 + %[sld_a_base] \n"
+    " ds_read_b64 v[182:183], %[v_sld_y_os] offset:13440 + %[sld_a_base] \n"
+    " ds_read_b64 v[184:185], %[v_sld_y_os] offset:14336 + %[sld_a_base] \n"
+    " ds_read_b64 v[186:187], %[v_sld_y_os] offset:14464 + %[sld_a_base] \n"
+    " ds_read_b64 v[188:189], %[v_sld_y_os] offset:15360 + %[sld_a_base] \n"
+    " ds_read_b64 v[190:191], %[v_sld_y_os] offset:15488 + %[sld_a_base] \n"
+    " ds_read_b64 v[192:193], %[v_sld_y_os] offset:16384 + %[sld_a_base] \n"
+    " ds_read_b64 v[194:195], %[v_sld_y_os] offset:16512 + %[sld_a_base] \n"
+    " ds_read_b64 v[196:197], %[v_sld_y_os] offset:17408 + %[sld_a_base] \n"
+    " ds_read_b64 v[198:199], %[v_sld_y_os] offset:17536 + %[sld_a_base] \n"
+    " ds_read_b64 v[200:201], %[v_sld_y_os] offset:18432 + %[sld_a_base] \n"
+    " ds_read_b64 v[202:203], %[v_sld_y_os] offset:18560 + %[sld_a_base] \n"
+    " ds_read_b64 v[204:205], %[v_sld_y_os] offset:19456 + %[sld_a_base] \n"
+    " ds_read_b64 v[206:207], %[v_sld_y_os] offset:19584 + %[sld_a_base] \n"
+    " ds_read_b64 v[208:209], %[v_sld_y_os] offset:20480 + %[sld_a_base] \n"
+    " ds_read_b64 v[210:211], %[v_sld_y_os] offset:20608 + %[sld_a_base] \n"
+    " ds_read_b64 v[212:213], %[v_sld_y_os] offset:21504 + %[sld_a_base] \n"
+    " ds_read_b64 v[214:215], %[v_sld_y_os] offset:21632 + %[sld_a_base] \n"
+    " ds_read_b64 v[216:217], %[v_sld_y_os] offset:22528 + %[sld_a_base] \n"
+    " ds_read_b64 v[218:219], %[v_sld_y_os] offset:22656 + %[sld_a_base] \n"
+    " ds_read_b64 v[220:221], %[v_sld_y_os] offset:23552 + %[sld_a_base] \n"
+    " ds_read_b64 v[222:223], %[v_sld_y_os] offset:23680 + %[sld_a_base] \n"
+    " ds_read_b64 v[224:225], %[v_sld_y_os] offset:24576 + %[sld_a_base] \n"
+    " ds_read_b64 v[226:227], %[v_sld_y_os] offset:24704 + %[sld_a_base] \n"
+    " ds_read_b64 v[228:229], %[v_sld_y_os] offset:25600 + %[sld_a_base] \n"
+    " ds_read_b64 v[230:231], %[v_sld_y_os] offset:25728 + %[sld_a_base] \n"
+    " ds_read_b64 v[232:233], %[v_sld_y_os] offset:26624 + %[sld_a_base] \n"
+    " ds_read_b64 v[234:235], %[v_sld_y_os] offset:26752 + %[sld_a_base] \n"
+    " ds_read_b64 v[236:237], %[v_sld_y_os] offset:27648 + %[sld_a_base] \n"
+    " ds_read_b64 v[238:239], %[v_sld_y_os] offset:27776 + %[sld_a_base] \n"
+    " ds_read_b64 v[240:241], %[v_sld_y_os] offset:28672 + %[sld_a_base] \n"
+    " ds_read_b64 v[242:243], %[v_sld_y_os] offset:28800 + %[sld_a_base] \n"
+    " ds_read_b64 v[244:245], %[v_sld_y_os] offset:29696 + %[sld_a_base] \n"
+    " ds_read_b64 v[246:247], %[v_sld_y_os] offset:29824 + %[sld_a_base] \n"
+    " ds_read_b64 v[248:249], %[v_sld_y_os] offset:30720 + %[sld_a_base] \n"
+    " ds_read_b64 v[250:251], %[v_sld_y_os] offset:30848 + %[sld_a_base] \n"
+    " ds_read_b64 v[252:253], %[v_sld_y_os] offset:31744 + %[sld_a_base] \n"
+    " ds_read_b64 v[254:255], %[v_sld_y_os] offset:31872 + %[sld_a_base] \n"
+    " s_waitcnt 0 \n"
+    " buffer_load_dwordx4 acc[0:3], %[v_os_b0], s[12:15], 0 offen \n"
+    " buffer_load_dwordx4 acc[4:7], %[v_os_b0], s[12:15], 0 offen offset:1024 \n"
+    " buffer_load_dwordx4 acc[8:11], %[v_os_b0], s[12:15], 0 offen offset:2048 \n"
+    " buffer_load_dwordx4 acc[12:15], %[v_os_b0], s[12:15], 0 offen offset:3072 \n"
+    " buffer_load_dwordx4 acc[16:19], %[v_os_b1], s[12:15], 0 offen \n"
+    " buffer_load_dwordx4 acc[20:23], %[v_os_b1], s[12:15], 0 offen offset:1024 \n"
+    " buffer_load_dwordx4 acc[24:27], %[v_os_b1], s[12:15], 0 offen offset:2048 \n"
+    " buffer_load_dwordx4 acc[28:31], %[v_os_b1], s[12:15], 0 offen offset:3072 \n"
+    " buffer_load_dwordx4 acc[32:35], %[v_os_b2], s[12:15], 0 offen \n"
+    " buffer_load_dwordx4 acc[36:39], %[v_os_b2], s[12:15], 0 offen offset:1024 \n"
+    " buffer_load_dwordx4 acc[40:43], %[v_os_b2], s[12:15], 0 offen offset:2048 \n"
+    " buffer_load_dwordx4 acc[44:47], %[v_os_b2], s[12:15], 0 offen offset:3072 \n"
+    " buffer_load_dwordx4 acc[48:51], %[v_os_b3], s[12:15], 0 offen \n"
+    " buffer_load_dwordx4 acc[52:55], %[v_os_b3], s[12:15], 0 offen offset:1024 \n"
+    " buffer_load_dwordx4 acc[56:59], %[v_os_b3], s[12:15], 0 offen offset:2048 \n"
+    " buffer_load_dwordx4 acc[60:63], %[v_os_b3], s[12:15], 0 offen offset:3072 \n"
+    " buffer_load_dwordx4 acc[64:67], %[v_os_b4], s[12:15], 0 offen \n"
+    " buffer_load_dwordx4 acc[68:71], %[v_os_b4], s[12:15], 0 offen offset:1024 \n"
+    " buffer_load_dwordx4 acc[72:75], %[v_os_b4], s[12:15], 0 offen offset:2048 \n"
+    " buffer_load_dwordx4 acc[76:79], %[v_os_b4], s[12:15], 0 offen offset:3072 \n"
+    " buffer_load_dwordx4 acc[80:83], %[v_os_b5], s[12:15], 0 offen \n"
+    " buffer_load_dwordx4 acc[84:87], %[v_os_b5], s[12:15], 0 offen offset:1024 \n"
+    " buffer_load_dwordx4 acc[88:91], %[v_os_b5], s[12:15], 0 offen offset:2048 \n"
+    " buffer_load_dwordx4 acc[92:95], %[v_os_b5], s[12:15], 0 offen offset:3072 \n"
+    " buffer_load_dwordx4 acc[96:99], %[v_os_b6], s[12:15], 0 offen \n"
+    " buffer_load_dwordx4 acc[100:103], %[v_os_b6], s[12:15], 0 offen offset:1024 \n"
+    " buffer_load_dwordx4 acc[104:107], %[v_os_b6], s[12:15], 0 offen offset:2048 \n"
+    " buffer_load_dwordx4 acc[108:111], %[v_os_b6], s[12:15], 0 offen offset:3072 \n"
+    " buffer_load_dwordx4 acc[112:115], %[v_os_b7], s[12:15], 0 offen \n"
+    " buffer_load_dwordx4 acc[116:119], %[v_os_b7], s[12:15], 0 offen offset:1024 \n"
+    " buffer_load_dwordx4 acc[120:123], %[v_os_b7], s[12:15], 0 offen offset:2048 \n"
+    " buffer_load_dwordx4 acc[124:127], %[v_os_b7], s[12:15], 0 offen offset:3072 \n"
+    " s_add_u32 s12, %[s_tile_os_b], s12 \n"
+    " s_addc_u32 s13, 0, s13 \n"
+    " v_mov_b32 v64, 0 \n"
+    " v_mov_b32 v80, 0 \n"
+    " v_mov_b32 v65, 0 \n"
+    " v_mov_b32 v81, 0 \n"
+    " v_mov_b32 v66, 0 \n"
+    " v_mov_b32 v82, 0 \n"
+    " v_mov_b32 v67, 0 \n"
+    " v_mov_b32 v83, 0 \n"
+    " v_mov_b32 v68, 0 \n"
+    " v_mov_b32 v84, 0 \n"
+    " v_mov_b32 v69, 0 \n"
+    " v_mov_b32 v85, 0 \n"
+    " v_mov_b32 v70, 0 \n"
+    " v_mov_b32 v86, 0 \n"
+    " v_mov_b32 v71, 0 \n"
+    " v_mov_b32 v87, 0 \n"
+    " ds_write_b64 %[v_sfl_sst], [%[c0],%[c1]] offset:16640  \n"
+    " ds_write_b64 %[v_sfl_sst], [%[c2],%[c3]] offset:20992  \n"
+    " ds_write_b64 %[v_sfl_sst], [%[c4],%[c5]] offset:18816  \n"
+    " ds_write_b64 %[v_sfl_sst], [%[c6],%[c7]] offset:23168  \n"
+    " s_mov_b32 s80, 0 \n"
+    " s_waitcnt vmcnt(24) \n"
+    "label_0AA6: \n"
+    " s_waitcnt vmcnt(30) & lgkmcnt(0)  \n"
+    " s_barrier \n" _UK_MFMA_ " [%[c0], %[c1], %[c2], %[c3]], acc[0:1], v[128:129], 0  \n"
+    " ds_read_b32 v10, %[v_sfl_sld] offset:16640  \n"
+    " ds_read_b32 v11, %[v_sfl_sld] offset:16672  \n"
+    " ds_write_b64 %[v_sfl_sst], [%[c16],%[c17]] offset:25344  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[2:3], v[130:131], v[64:67]  \n"
+    " buffer_load_dwordx4 acc[128:131],  %[v_os_b0], s[12:15], 0 offen  \n"
+    " ds_write_b64 %[v_sfl_sst], [%[c18],%[c19]] offset:29696  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[4:5], v[132:133], v[64:67]  \n"
+    " ds_read_b32 v12, %[v_sfl_sld] offset:16704  \n"
+    " ds_read_b32 v13, %[v_sfl_sld] offset:16736  \n"
+    " ds_write_b64 %[v_sfl_sst], [%[c20],%[c21]] offset:27520  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[6:7], v[134:135], v[64:67]  \n"
+    " ds_write_b64 %[v_sfl_sst], [%[c22],%[c23]] offset:31872  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[8:9], v[136:137], v[64:67]  \n"
+    " ds_read_b32 v14, %[v_sfl_sld] offset:20992  \n"
+    " ds_read_b32 v15, %[v_sfl_sld] offset:21024  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[10:11], v[138:139], v[64:67]  \n"
+    " buffer_load_dwordx4 acc[132:135],  %[v_os_b0], s[12:15], 0 offen offset:1024  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[12:13], v[140:141], v[64:67]  \n"
+    " ds_read_b32 v16, %[v_sfl_sld] offset:21056  \n"
+    " ds_read_b32 v17, %[v_sfl_sld] offset:21088  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[14:15], v[142:143], v[64:67]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[0:1], v[192:193], 0  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[2:3], v[194:195], v[68:71]  \n"
+    " buffer_load_dwordx4 acc[136:139],  %[v_os_b0], s[12:15], 0 offen offset:2048  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[4:5], v[196:197], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[6:7], v[198:199], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[8:9], v[200:201], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[10:11], v[202:203], v[68:71]  \n"
+    " buffer_load_dwordx4 acc[140:143],  %[v_os_b0], s[12:15], 0 offen offset:3072  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[12:13], v[204:205], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[14:15], v[206:207], v[68:71]  \n"
+    " s_waitcnt lgkmcnt(0) \n"
+    " s_mov_b64 exec, %[s_execflag_0] \n" _UK_ATOMIC_ADD_ " %[v_os_o0], v10, s[8:9] \n"
+    "  s_mov_b64     exec, s[38:39]                           \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[16:17], v[128:129], 0  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[18:19], v[130:131], v[72:75]  \n"
+    " buffer_load_dwordx4 acc[144:147],  %[v_os_b1], s[12:15], 0 offen  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[20:21], v[132:133], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[22:23], v[134:135], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[24:25], v[136:137], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[26:27], v[138:139], v[72:75]  \n"
+    " buffer_load_dwordx4 acc[148:151],  %[v_os_b1], s[12:15], 0 offen offset:1024  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[28:29], v[140:141], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[30:31], v[142:143], v[72:75]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[16:17], v[192:193], 0  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[18:19], v[194:195], v[76:79]  \n"
+    " buffer_load_dwordx4 acc[152:155],  %[v_os_b1], s[12:15], 0 offen offset:2048  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[20:21], v[196:197], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[22:23], v[198:199], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[24:25], v[200:201], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[26:27], v[202:203], v[76:79]  \n"
+    " buffer_load_dwordx4 acc[156:159],  %[v_os_b1], s[12:15], 0 offen offset:3072  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[28:29], v[204:205], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[30:31], v[206:207], v[76:79]  \n"
+    " s_mov_b64 exec, %[s_execflag_1] \n" _UK_ATOMIC_ADD_ " %[v_os_o1], v11, s[8:9] \n"
+    "  s_mov_b64     exec, s[38:39]  \n"
+    " s_waitcnt vmcnt(30)   \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[32:33], v[144:145], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[34:35], v[146:147], v[64:67]  \n"
+    " buffer_load_dwordx4 acc[160:163],  %[v_os_b2], s[12:15], 0 offen  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[36:37], v[148:149], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[38:39], v[150:151], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[40:41], v[152:153], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[42:43], v[154:155], v[64:67]  \n"
+    " buffer_load_dwordx4 acc[164:167],  %[v_os_b2], s[12:15], 0 offen offset:1024  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[44:45], v[156:157], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[46:47], v[158:159], v[64:67]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[32:33], v[208:209], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[34:35], v[210:211], v[68:71]  \n"
+    " buffer_load_dwordx4 acc[168:171],  %[v_os_b2], s[12:15], 0 offen offset:2048  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[36:37], v[212:213], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[38:39], v[214:215], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[40:41], v[216:217], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[42:43], v[218:219], v[68:71]  \n"
+    " buffer_load_dwordx4 acc[172:175],  %[v_os_b2], s[12:15], 0 offen offset:3072  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[44:45], v[220:221], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[46:47], v[222:223], v[68:71]  \n"
+    " s_mov_b64 exec, %[s_execflag_2] \n" _UK_ATOMIC_ADD_ " %[v_os_o2], v12, s[8:9] \n"
+    "  s_mov_b64     exec, s[38:39]                           \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[48:49], v[144:145], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[50:51], v[146:147], v[72:75]  \n"
+    " buffer_load_dwordx4 acc[176:179],  %[v_os_b3], s[12:15], 0 offen  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[52:53], v[148:149], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[54:55], v[150:151], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[56:57], v[152:153], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[58:59], v[154:155], v[72:75]  \n"
+    " buffer_load_dwordx4 acc[180:183],  %[v_os_b3], s[12:15], 0 offen offset:1024  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[60:61], v[156:157], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[62:63], v[158:159], v[72:75]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[48:49], v[208:209], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[50:51], v[210:211], v[76:79]  \n"
+    " buffer_load_dwordx4 acc[184:187],  %[v_os_b3], s[12:15], 0 offen offset:2048  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[52:53], v[212:213], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[54:55], v[214:215], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[56:57], v[216:217], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[58:59], v[218:219], v[76:79]  \n"
+    " buffer_load_dwordx4 acc[188:191],  %[v_os_b3], s[12:15], 0 offen offset:3072  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[60:61], v[220:221], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[62:63], v[222:223], v[76:79]  \n"
+    " s_mov_b64 exec, %[s_execflag_3] \n" _UK_ATOMIC_ADD_ " %[v_os_o3], v13, s[8:9] \n"
+    "  s_mov_b64     exec, s[38:39]    \n"
+    " s_waitcnt vmcnt(30)   \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[64:65], v[160:161], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[66:67], v[162:163], v[64:67]  \n"
+    " buffer_load_dwordx4 acc[192:195],  %[v_os_b4], s[12:15], 0 offen  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[68:69], v[164:165], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[70:71], v[166:167], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[72:73], v[168:169], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[74:75], v[170:171], v[64:67]  \n"
+    " buffer_load_dwordx4 acc[196:199],  %[v_os_b4], s[12:15], 0 offen offset:1024  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[76:77], v[172:173], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[78:79], v[174:175], v[64:67]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[64:65], v[224:225], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[66:67], v[226:227], v[68:71]  \n"
+    " buffer_load_dwordx4 acc[200:203],  %[v_os_b4], s[12:15], 0 offen offset:2048  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[68:69], v[228:229], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[70:71], v[230:231], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[72:73], v[232:233], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[74:75], v[234:235], v[68:71]  \n"
+    " buffer_load_dwordx4 acc[204:207],  %[v_os_b4], s[12:15], 0 offen offset:3072  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[76:77], v[236:237], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[78:79], v[238:239], v[68:71]  \n"
+    " s_mov_b64 exec, %[s_execflag_4] \n" _UK_ATOMIC_ADD_ " %[v_os_o4], v14, s[8:9] \n"
+    "  s_mov_b64     exec, s[38:39]                           \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[80:81], v[160:161], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[82:83], v[162:163], v[72:75]  \n"
+    " buffer_load_dwordx4 acc[208:211],  %[v_os_b5], s[12:15], 0 offen  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[84:85], v[164:165], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[86:87], v[166:167], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[88:89], v[168:169], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[90:91], v[170:171], v[72:75]  \n"
+    " buffer_load_dwordx4 acc[212:215],  %[v_os_b5], s[12:15], 0 offen offset:1024  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[92:93], v[172:173], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[94:95], v[174:175], v[72:75]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[80:81], v[224:225], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[82:83], v[226:227], v[76:79]  \n"
+    " buffer_load_dwordx4 acc[216:219],  %[v_os_b5], s[12:15], 0 offen offset:2048  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[84:85], v[228:229], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[86:87], v[230:231], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[88:89], v[232:233], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[90:91], v[234:235], v[76:79]  \n"
+    " buffer_load_dwordx4 acc[220:223],  %[v_os_b5], s[12:15], 0 offen offset:3072  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[92:93], v[236:237], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[94:95], v[238:239], v[76:79]  \n"
+    " s_mov_b64 exec, %[s_execflag_5] \n" _UK_ATOMIC_ADD_ " %[v_os_o5], v15, s[8:9] \n"
+    "  s_mov_b64     exec, s[38:39]                           \n"
+    " s_waitcnt vmcnt(30)   \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[96:97], v[176:177], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[98:99], v[178:179], v[64:67]  \n"
+    " buffer_load_dwordx4 acc[224:227],  %[v_os_b6], s[12:15], 0 offen  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[100:101], v[180:181], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[102:103], v[182:183], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[104:105], v[184:185], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[106:107], v[186:187], v[64:67]  \n"
+    " buffer_load_dwordx4 acc[228:231],  %[v_os_b6], s[12:15], 0 offen offset:1024  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[108:109], v[188:189], v[64:67]  \n" _UK_MFMA_
+    " [%[c0], %[c1], %[c2], %[c3]], acc[110:111], v[190:191], v[64:67]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[96:97], v[240:241], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[98:99], v[242:243], v[68:71]  \n"
+    " buffer_load_dwordx4 acc[232:235],  %[v_os_b6], s[12:15], 0 offen offset:2048  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[100:101], v[244:245], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[102:103], v[246:247], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[104:105], v[248:249], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[106:107], v[250:251], v[68:71]  \n"
+    " buffer_load_dwordx4 acc[236:239],  %[v_os_b6], s[12:15], 0 offen offset:3072  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[108:109], v[252:253], v[68:71]  \n" _UK_MFMA_
+    " [%[c4], %[c5], %[c6], %[c7]], acc[110:111], v[254:255], v[68:71]  \n"
+    " s_mov_b64 exec, %[s_execflag_6] \n" _UK_ATOMIC_ADD_ " %[v_os_o6], v16, s[8:9] \n"
+    "  s_mov_b64     exec, s[38:39]                           \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[112:113], v[176:177], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[114:115], v[178:179], v[72:75]  \n"
+    " buffer_load_dwordx4 acc[240:243],  %[v_os_b7], s[12:15], 0 offen  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[116:117], v[180:181], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[118:119], v[182:183], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[120:121], v[184:185], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[122:123], v[186:187], v[72:75]  \n"
+    " buffer_load_dwordx4 acc[244:247],  %[v_os_b7], s[12:15], 0 offen offset:1024  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[124:125], v[188:189], v[72:75]  \n" _UK_MFMA_
+    " [%[c8], %[c9], %[c10], %[c11]], acc[126:127], v[190:191], v[72:75]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[112:113], v[240:241], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[114:115], v[242:243], v[76:79]  \n"
+    " buffer_load_dwordx4 acc[248:251],  %[v_os_b7], s[12:15], 0 offen offset:2048  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[116:117], v[244:245], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[118:119], v[246:247], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[120:121], v[248:249], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[122:123], v[250:251], v[76:79]  \n"
+    " buffer_load_dwordx4 acc[252:255],  %[v_os_b7], s[12:15], 0 offen offset:3072  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[124:125], v[252:253], v[76:79]  \n" _UK_MFMA_
+    " [%[c12], %[c13], %[c14], %[c15]], acc[126:127], v[254:255], v[76:79]  \n"
+    " s_mov_b64 exec, %[s_execflag_7] \n" _UK_ATOMIC_ADD_ " %[v_os_o7], v17, s[8:9] \n"
+    "  s_mov_b64     exec, s[38:39]                           \n"
+    " s_add_u32 s60, 0x00000100, s80  \n"
+    " s_cmp_lt_u32 s60, %[s_loop_cnt]  \n"
+    " s_cselect_b32 s56, %[s_tile_os_b], 0  \n"
+    " s_add_u32 s12, s56, s12  \n"
+    " s_addc_u32 s13, 0, s13  \n"
+    " s_cmp_ge_u32 s80, 0x00000100  \n"
+    " s_cselect_b32 s59, %[s_tile_os_o], s59  \n"
+    " s_add_u32 s8, s59, s8  \n"
+    " s_addc_u32 s9, 0, s9  \n"
+    " v_mul_f32 %[c0], %[scale_0], %[c0] \n"
+    " v_mul_f32 %[c1], %[scale_0], %[c1] \n"
+    " v_mul_f32 %[c2], %[scale_0], %[c2] \n"
+    " v_mul_f32 %[c3], %[scale_0], %[c3] \n"
+    " v_mul_f32 %[c4], %[scale_1], %[c4] \n"
+    " v_mul_f32 %[c5], %[scale_1], %[c5] \n"
+    " v_mul_f32 %[c6], %[scale_1], %[c6] \n"
+    " v_mul_f32 %[c7], %[scale_1], %[c7] \n"
+    " v_mul_f32 %[c8], %[scale_0], %[c8] \n"
+    " v_mul_f32 %[c9], %[scale_0], %[c9] \n"
+    " v_mul_f32 %[c10], %[scale_0], %[c10] \n"
+    " v_mul_f32 %[c11], %[scale_0], %[c11] \n"
+    " v_mul_f32 %[c12], %[scale_1], %[c12] \n"
+    " v_mul_f32 %[c13], %[scale_1], %[c13] \n"
+    " v_mul_f32 %[c14], %[scale_1], %[c14] \n"
+    " v_mul_f32 %[c15], %[scale_1], %[c15] \n" _UK_PK_CVT_("%[c0]", "%[c1]", "%[c0]") _UK_PK_CVT_(
+        "%[c2]",
+        "%[c3]",
+        "%[c1]") _UK_PK_CVT_("%[c4]",
+                             "%[c5]",
+                             "%[c2]") _UK_PK_CVT_("%[c6]",
+                                                  "%[c7]",
+                                                  "%[c3]") _UK_PK_CVT_("%[c8]",
+                                                                       "%[c9]",
+                                                                       "%[c4]") _UK_PK_CVT_("%["
+                                                                                            "c10]",
+                                                                                            "%["
+                                                                                            "c11]",
+                                                                                            "%[c5]")
+        _UK_PK_CVT_("%[c12]", "%[c13]", "%[c6]") _UK_PK_CVT_(
+            "%[c14]",
+            "%[c15]",
+            "%[c7]") " s_addk_i32 s80, 0x0080  \n"
+                     " s_cmp_lt_i32 s80, %[s_loop_cnt]  \n"
+                     " s_cbranch_scc0 label_0EC1  \n"
+                     " s_waitcnt vmcnt(30) & lgkmcnt(0)  \n"
+                     " s_barrier  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[128:129], v[128:129], 0  \n"
+                     " ds_read_b32 v10, %[v_sfl_sld] offset:25344  \n"
+                     " ds_read_b32 v11, %[v_sfl_sld] offset:25376  \n"
+                     " ds_write_b64 v3, v[64:65] offset:16640  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[130:131], v[130:131], v[80:83]  \n"
+                     " buffer_load_dwordx4 acc[0:3],  %[v_os_b0], s[12:15], 0 offen  \n"
+                     " ds_write_b64 v3, v[66:67] offset:20992  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[132:133], v[132:133], v[80:83]  \n"
+                     " ds_read_b32 v12, %[v_sfl_sld] offset:25408  \n"
+                     " ds_read_b32 v13, %[v_sfl_sld] offset:25440  \n"
+                     " ds_write_b64 v3, v[68:69] offset:18816  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[134:135], v[134:135], v[80:83]  \n"
+                     " ds_write_b64 v3, v[70:71] offset:23168  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[136:137], v[136:137], v[80:83]  \n"
+                     " ds_read_b32 v14, %[v_sfl_sld] offset:29696  \n"
+                     " ds_read_b32 v15, %[v_sfl_sld] offset:29728  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[138:139], v[138:139], v[80:83]  \n"
+                     " buffer_load_dwordx4 acc[4:7],  %[v_os_b0], s[12:15], 0 offen offset:1024  "
+                     "\n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[140:141], v[140:141], v[80:83]  \n"
+                     " ds_read_b32 v16, %[v_sfl_sld] offset:29760  \n"
+                     " ds_read_b32 v17, %[v_sfl_sld] offset:29792  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[142:143], v[142:143], v[80:83]  "
+                     "\n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[128:129], v[192:193], 0  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[130:131], v[194:195], v[84:87]  \n"
+                     " buffer_load_dwordx4 acc[8:11],  %[v_os_b0], s[12:15], 0 offen offset:2048  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[132:133], v[196:197], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[134:135], v[198:199], v[84:87]  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[136:137], v[200:201], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[138:139], v[202:203], v[84:87]  \n"
+                     " buffer_load_dwordx4 acc[12:15],  %[v_os_b0], s[12:15], 0 offen offset:3072  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[140:141], v[204:205], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[142:143], v[206:207], v[84:87]  \n"
+                     " s_waitcnt lgkmcnt(0) \n"
+                     " s_mov_b64 exec, %[s_execflag_0] \n" _UK_ATOMIC_ADD_
+                     " %[v_os_o0], v10, s[8:9] \n"
+                     "  s_mov_b64     exec, s[38:39]                           \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[144:145], v[128:129], 0  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[146:147], v[130:131], v[88:91]  \n"
+                     " buffer_load_dwordx4 acc[16:19],  %[v_os_b1], s[12:15], 0 offen  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[148:149], v[132:133], v[88:91]  "
+                     "\n" _UK_MFMA_ " [%[c24], %[c25], %[c26], %[c27]], acc[150:151], v[134:135], "
+                     "v[88:91]  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[152:153], v[136:137], v[88:91]  "
+                     "\n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[154:155], v[138:139], v[88:91]  \n"
+                     " buffer_load_dwordx4 acc[20:23],  %[v_os_b1], s[12:15], 0 offen offset:1024  "
+                     "\n" _UK_MFMA_ " [%[c24], %[c25], %[c26], %[c27]], acc[156:157], v[140:141], "
+                     "v[88:91]  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[158:159], v[142:143], v[88:91]  "
+                     "\n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[144:145], v[192:193], 0  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[146:147], v[194:195], v[92:95]  \n"
+                     " buffer_load_dwordx4 acc[24:27],  %[v_os_b1], s[12:15], 0 offen offset:2048  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[148:149], v[196:197], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[150:151], v[198:199], v[92:95]  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[152:153], v[200:201], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[154:155], v[202:203], v[92:95]  \n"
+                     " buffer_load_dwordx4 acc[28:31],  %[v_os_b1], s[12:15], 0 offen offset:3072  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[156:157], v[204:205], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[158:159], v[206:207], v[92:95]  \n"
+                     " s_mov_b64 exec, %[s_execflag_1] \n" _UK_ATOMIC_ADD_
+                     " %[v_os_o1], v11, s[8:9] \n"
+                     "  s_mov_b64     exec, s[38:39]                           \n"
+                     " s_waitcnt vmcnt(30) \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[160:161], v[144:145], v[80:83]  "
+                     "\n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[162:163], v[146:147], v[80:83]  \n"
+                     " buffer_load_dwordx4 acc[32:35],  %[v_os_b2], s[12:15], 0 offen  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[164:165], v[148:149], v[80:83]  "
+                     "\n" _UK_MFMA_ " [%[c16], %[c17], %[c18], %[c19]], acc[166:167], v[150:151], "
+                     "v[80:83]  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[168:169], v[152:153], v[80:83]  "
+                     "\n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[170:171], v[154:155], v[80:83]  \n"
+                     " buffer_load_dwordx4 acc[36:39],  %[v_os_b2], s[12:15], 0 offen offset:1024  "
+                     "\n" _UK_MFMA_ " [%[c16], %[c17], %[c18], %[c19]], acc[172:173], v[156:157], "
+                     "v[80:83]  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[174:175], v[158:159], v[80:83]  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[160:161], v[208:209], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[162:163], v[210:211], v[84:87]  \n"
+                     " buffer_load_dwordx4 acc[40:43],  %[v_os_b2], s[12:15], 0 offen offset:2048  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[164:165], v[212:213], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[166:167], v[214:215], v[84:87]  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[168:169], v[216:217], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[170:171], v[218:219], v[84:87]  \n"
+                     " buffer_load_dwordx4 acc[44:47],  %[v_os_b2], s[12:15], 0 offen offset:3072  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[172:173], v[220:221], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[174:175], v[222:223], v[84:87]  \n"
+                     " s_mov_b64 exec, %[s_execflag_2] \n" _UK_ATOMIC_ADD_
+                     " %[v_os_o2], v12, s[8:9] \n"
+                     "  s_mov_b64     exec, s[38:39]                           \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[176:177], v[144:145], v[88:91]  "
+                     "\n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[178:179], v[146:147], v[88:91]  \n"
+                     " buffer_load_dwordx4 acc[48:51],  %[v_os_b3], s[12:15], 0 offen  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[180:181], v[148:149], v[88:91]  "
+                     "\n" _UK_MFMA_ " [%[c24], %[c25], %[c26], %[c27]], acc[182:183], v[150:151], "
+                     "v[88:91]  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[184:185], v[152:153], v[88:91]  "
+                     "\n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[186:187], v[154:155], v[88:91]  \n"
+                     " buffer_load_dwordx4 acc[52:55],  %[v_os_b3], s[12:15], 0 offen offset:1024  "
+                     "\n" _UK_MFMA_ " [%[c24], %[c25], %[c26], %[c27]], acc[188:189], v[156:157], "
+                     "v[88:91]  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[190:191], v[158:159], v[88:91]  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[176:177], v[208:209], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[178:179], v[210:211], v[92:95]  \n"
+                     " buffer_load_dwordx4 acc[56:59],  %[v_os_b3], s[12:15], 0 offen offset:2048  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[180:181], v[212:213], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[182:183], v[214:215], v[92:95]  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[184:185], v[216:217], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[186:187], v[218:219], v[92:95]  \n"
+                     " buffer_load_dwordx4 acc[60:63],  %[v_os_b3], s[12:15], 0 offen offset:3072  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[188:189], v[220:221], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[190:191], v[222:223], v[92:95]  \n"
+                     " s_mov_b64 exec, %[s_execflag_3] \n" _UK_ATOMIC_ADD_
+                     " %[v_os_o3], v13, s[8:9] \n"
+                     "  s_mov_b64     exec, s[38:39]                           \n"
+                     " s_waitcnt vmcnt(30) \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[192:193], v[160:161], v[80:83]  "
+                     "\n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[194:195], v[162:163], v[80:83]  \n"
+                     " buffer_load_dwordx4 acc[64:67],  %[v_os_b4], s[12:15], 0 offen  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[196:197], v[164:165], v[80:83]  "
+                     "\n" _UK_MFMA_ " [%[c16], %[c17], %[c18], %[c19]], acc[198:199], v[166:167], "
+                     "v[80:83]  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[200:201], v[168:169], v[80:83]  "
+                     "\n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[202:203], v[170:171], v[80:83]  \n"
+                     " buffer_load_dwordx4 acc[68:71],  %[v_os_b4], s[12:15], 0 offen offset:1024  "
+                     "\n" _UK_MFMA_ " [%[c16], %[c17], %[c18], %[c19]], acc[204:205], v[172:173], "
+                     "v[80:83]  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[206:207], v[174:175], v[80:83]  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[192:193], v[224:225], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[194:195], v[226:227], v[84:87]  \n"
+                     " buffer_load_dwordx4 acc[72:75],  %[v_os_b4], s[12:15], 0 offen offset:2048  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[196:197], v[228:229], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[198:199], v[230:231], v[84:87]  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[200:201], v[232:233], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[202:203], v[234:235], v[84:87]  \n"
+                     " buffer_load_dwordx4 acc[76:79],  %[v_os_b4], s[12:15], 0 offen offset:3072  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[204:205], v[236:237], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[206:207], v[238:239], v[84:87]  \n"
+                     " s_mov_b64 exec, %[s_execflag_4] \n" _UK_ATOMIC_ADD_
+                     " %[v_os_o4], v14, s[8:9] \n"
+                     "  s_mov_b64     exec, s[38:39]                           \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[208:209], v[160:161], v[88:91]  "
+                     "\n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[210:211], v[162:163], v[88:91]  \n"
+                     " buffer_load_dwordx4 acc[80:83],  %[v_os_b5], s[12:15], 0 offen  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[212:213], v[164:165], v[88:91]  "
+                     "\n" _UK_MFMA_ " [%[c24], %[c25], %[c26], %[c27]], acc[214:215], v[166:167], "
+                     "v[88:91]  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[216:217], v[168:169], v[88:91]  "
+                     "\n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[218:219], v[170:171], v[88:91]  \n"
+                     " buffer_load_dwordx4 acc[84:87],  %[v_os_b5], s[12:15], 0 offen offset:1024  "
+                     "\n" _UK_MFMA_ " [%[c24], %[c25], %[c26], %[c27]], acc[220:221], v[172:173], "
+                     "v[88:91]  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[222:223], v[174:175], v[88:91]  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[208:209], v[224:225], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[210:211], v[226:227], v[92:95]  \n"
+                     " buffer_load_dwordx4 acc[88:91],  %[v_os_b5], s[12:15], 0 offen offset:2048  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[212:213], v[228:229], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[214:215], v[230:231], v[92:95]  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[216:217], v[232:233], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[218:219], v[234:235], v[92:95]  \n"
+                     " buffer_load_dwordx4 acc[92:95],  %[v_os_b5], s[12:15], 0 offen offset:3072  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[220:221], v[236:237], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[222:223], v[238:239], v[92:95]  \n"
+                     " s_mov_b64 exec, %[s_execflag_5] \n" _UK_ATOMIC_ADD_
+                     " %[v_os_o5], v15, s[8:9] \n"
+                     "  s_mov_b64     exec, s[38:39]                           \n"
+                     " s_waitcnt vmcnt(30)  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[224:225], v[176:177], v[80:83]  "
+                     "\n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[226:227], v[178:179], v[80:83]  \n"
+                     " buffer_load_dwordx4 acc[96:99],  %[v_os_b6], s[12:15], 0 offen  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[228:229], v[180:181], v[80:83]  "
+                     "\n" _UK_MFMA_ " [%[c16], %[c17], %[c18], %[c19]], acc[230:231], v[182:183], "
+                     "v[80:83]  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[232:233], v[184:185], v[80:83]  "
+                     "\n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[234:235], v[186:187], v[80:83]  \n"
+                     " buffer_load_dwordx4 acc[100:103],  %[v_os_b6], s[12:15], 0 offen "
+                     "offset:1024  \n" _UK_MFMA_ " [%[c16], %[c17], %[c18], %[c19]], acc[236:237], "
+                     "v[188:189], v[80:83]  \n" _UK_MFMA_
+                     " [%[c16], %[c17], %[c18], %[c19]], acc[238:239], v[190:191], v[80:83]  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[224:225], v[240:241], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[226:227], v[242:243], v[84:87]  \n"
+                     " buffer_load_dwordx4 acc[104:107],  %[v_os_b6], s[12:15], 0 offen "
+                     "offset:2048  \n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[228:229], "
+                     "v[244:245], v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[230:231], v[246:247], v[84:87]  "
+                     "\n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[232:233], v[248:249], "
+                     "v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[234:235], v[250:251], v[84:87]  \n"
+                     " buffer_load_dwordx4 acc[108:111],  %[v_os_b6], s[12:15], 0 offen "
+                     "offset:3072  \n" _UK_MFMA_ " [%[c20], %[c21], %[c22], %[c23]], acc[236:237], "
+                     "v[252:253], v[84:87]  \n" _UK_MFMA_
+                     " [%[c20], %[c21], %[c22], %[c23]], acc[238:239], v[254:255], v[84:87]  \n"
+                     " s_mov_b64 exec, %[s_execflag_6] \n" _UK_ATOMIC_ADD_
+                     " %[v_os_o6], v16, s[8:9] \n"
+                     "  s_mov_b64     exec, s[38:39]                           \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[240:241], v[176:177], v[88:91]  "
+                     "\n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[242:243], v[178:179], v[88:91]  \n"
+                     " buffer_load_dwordx4 acc[112:115],  %[v_os_b7], s[12:15], 0 offen  "
+                     "\n" _UK_MFMA_ " [%[c24], %[c25], %[c26], %[c27]], acc[244:245], v[180:181], "
+                     "v[88:91]  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[246:247], v[182:183], v[88:91]  "
+                     "\n" _UK_MFMA_ " [%[c24], %[c25], %[c26], %[c27]], acc[248:249], v[184:185], "
+                     "v[88:91]  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[250:251], v[186:187], v[88:91]  \n"
+                     " buffer_load_dwordx4 acc[116:119],  %[v_os_b7], s[12:15], 0 offen "
+                     "offset:1024  \n" _UK_MFMA_ " [%[c24], %[c25], %[c26], %[c27]], acc[252:253], "
+                     "v[188:189], v[88:91]  \n" _UK_MFMA_
+                     " [%[c24], %[c25], %[c26], %[c27]], acc[254:255], v[190:191], v[88:91]  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[240:241], v[240:241], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[242:243], v[242:243], v[92:95]  \n"
+                     " buffer_load_dwordx4 acc[120:123],  %[v_os_b7], s[12:15], 0 offen "
+                     "offset:2048  \n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[244:245], "
+                     "v[244:245], v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[246:247], v[246:247], v[92:95]  "
+                     "\n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[248:249], v[248:249], "
+                     "v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[250:251], v[250:251], v[92:95]  \n"
+                     " buffer_load_dwordx4 acc[124:127],  %[v_os_b7], s[12:15], 0 offen "
+                     "offset:3072  \n" _UK_MFMA_ " [%[c28], %[c29], %[c30], %[c31]], acc[252:253], "
+                     "v[252:253], v[92:95]  \n" _UK_MFMA_
+                     " [%[c28], %[c29], %[c30], %[c31]], acc[254:255], v[254:255], v[92:95]  \n"
+                     " s_mov_b64 exec, %[s_execflag_7] \n" _UK_ATOMIC_ADD_
+                     " %[v_os_o7], v17, s[8:9] \n"
+                     "  s_mov_b64     exec, s[38:39]                           \n"
+                     " s_add_u32 s60, 0x00000100, s80  \n"
+                     " s_cmp_lt_u32 s60, %[s_loop_cnt]  \n"
+                     " s_cselect_b32 s56, s56, 0  \n"
+                     " s_add_u32 s12, s56, s12  \n"
+                     " s_addc_u32 s13, 0, s13  \n"
+                     " s_cmp_ge_u32 s80, 0x00000100  \n"
+                     " s_cselect_b32 s59, 0x00000100, s59  \n"
+                     " s_add_u32 s8, s59, s8  \n"
+                     " s_addc_u32 s9, 0, s9  \n"
+                     " v_mul_f32 %[c16], %[scale_0], %[c16] \n"
+                     " v_mul_f32 %[c17], %[scale_0], %[c17] \n"
+                     " v_mul_f32 %[c18], %[scale_0], %[c18] \n"
+                     " v_mul_f32 %[c19], %[scale_0], %[c19] \n"
+                     " v_mul_f32 %[c20], %[scale_1], %[c20] \n"
+                     " v_mul_f32 %[c21], %[scale_1], %[c21] \n"
+                     " v_mul_f32 %[c22], %[scale_1], %[c22] \n"
+                     " v_mul_f32 %[c23], %[scale_1], %[c23] \n"
+                     " v_mul_f32 %[c24], %[scale_0], %[c24] \n"
+                     " v_mul_f32 %[c25], %[scale_0], %[c25] \n"
+                     " v_mul_f32 %[c26], %[scale_0], %[c26] \n"
+                     " v_mul_f32 %[c27], %[scale_0], %[c27] \n"
+                     " v_mul_f32 %[c28], %[scale_1], %[c28] \n"
+                     " v_mul_f32 %[c29], %[scale_1], %[c29] \n"
+                     " v_mul_f32 %[c30], %[scale_1], %[c30] \n"
+                     " v_mul_f32 %[c31], %[scale_1], %[c31] \n" _UK_PK_CVT_(
+                         "%[c16]", "%[c17]", "%[c16]") _UK_PK_CVT_("%[c18]", "%[c19]", "%[c17]")
+                         _UK_PK_CVT_("%[c20]", "%[c21]", "%[c18]") _UK_PK_CVT_(
+                             "%[c22]", "%[c23]", "%[c19]") _UK_PK_CVT_("%[c24]", "%[c25]", "%[c20]")
+                             _UK_PK_CVT_("%[c26]", "%[c27]", "%[c21]")
+                                 _UK_PK_CVT_("%[c28]", "%[c29]", "%[c22]") _UK_PK_CVT_(
+                                     "%[c30]",
+                                     "%[c31]",
+                                     "%[c23]") " s_addk_i32 s80, 0x0080  \n"
+                                               " s_cmp_lt_i32 s80, %[s_loop_cnt]  \n"
+                                               " s_cbranch_scc0 label_0EC1  \n"
+                                               " s_branch label_0AA6  \n"
+                                               " label_0EC1: \n"
+                                               " s_waitcnt lgkmcnt(0)  \n"
+                                               " s_barrier  \n"
+                                               " ds_read_b32 v10, %[v_sfl_sld] offset:16640  \n"
+                                               " ds_read_b32 v11, %[v_sfl_sld] offset:16672  \n"
+                                               " ds_read_b32 v12, %[v_sfl_sld] offset:16704  \n"
+                                               " ds_read_b32 v13, %[v_sfl_sld] offset:16736  \n"
+                                               " ds_read_b32 v14, %[v_sfl_sld] offset:20992  \n"
+                                               " ds_read_b32 v15, %[v_sfl_sld] offset:21024  \n"
+                                               " ds_read_b32 v16, %[v_sfl_sld] offset:21056  \n"
+                                               " ds_read_b32 v17, %[v_sfl_sld] offset:21088  \n"
+                                               " s_waitcnt lgkmcnt(0)  \n"
+                                               " s_mov_b64 exec, %[s_execflag_0] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o0], v10, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_1] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o1], v11, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_2] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o2], v12, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_3] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o3], v13, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_4] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o4], v14, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_5] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o5], v15, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_6] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o6], v16, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_7] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o7], v17, s[8:9] \n"
+                                               "  s_mov_b64     exec, s[38:39]                     "
+                                               "      \n"
+                                               " s_add_u32 s8, s59, s8  \n"
+                                               " s_addc_u32 s9, 0, s9  \n"
+                                               " ds_write_b64 %[v_sfl_sst], [%[c16],%[c17]] "
+                                               "offset:25344  \n"
+                                               " ds_write_b64 %[v_sfl_sst], [%[c18],%[c19]] "
+                                               "offset:29696  \n"
+                                               " ds_write_b64 %[v_sfl_sst], [%[c20],%[c21]] "
+                                               "offset:27520  \n"
+                                               " ds_write_b64 %[v_sfl_sst], [%[c22],%[c23]] "
+                                               "offset:31872  \n"
+                                               " s_waitcnt lgkmcnt(0)  \n"
+                                               " s_barrier  \n"
+                                               " ds_read_b32 v10, %[v_sfl_sld] offset:25344  \n"
+                                               " ds_read_b32 v11, %[v_sfl_sld] offset:25376  \n"
+                                               " ds_read_b32 v12, %[v_sfl_sld] offset:25408  \n"
+                                               " ds_read_b32 v13, %[v_sfl_sld] offset:25440  \n"
+                                               " ds_read_b32 v14, %[v_sfl_sld] offset:29696  \n"
+                                               " ds_read_b32 v15, %[v_sfl_sld] offset:29728  \n"
+                                               " ds_read_b32 v16, %[v_sfl_sld] offset:29760  \n"
+                                               " ds_read_b32 v17, %[v_sfl_sld] offset:29792  \n"
+                                               " s_waitcnt lgkmcnt(0)  \n"
+                                               " s_mov_b64 exec, %[s_execflag_0] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o0], v10, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_1] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o1], v11, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_2] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o2], v12, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_3] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o3], v13, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_4] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o4], v14, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_5] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o5], v15, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_6] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o6], v16, s[8:9] \n"
+                                               " s_mov_b64 exec, %[s_execflag_7] \n" _UK_ATOMIC_ADD_
+                                               " %[v_os_o7], v17, s[8:9] \n"
+                                               "  s_mov_b64     exec, s[38:39]  \n"
+
+#undef _UK_MFMA_
+#undef _UK_PK_CVT_
+#undef _UK_ATOMIC_ADD_

include/ck_tile/ops/flatmm/block/uk/flatmm_uk_gfx9_32x512x128_1x1x1_16x16x16.inc

@@ -9,508 +9,509 @@
 #endif
 
 "s_mov_b32 s16,    %[s_res_a0] \n"
-"s_mov_b32 s17,    %[s_res_a1] \n"
-"s_mov_b32 s18,    %[s_res_a2] \n"
-"s_mov_b32 s19,    %[s_res_a3] \n"
-"s_mov_b32 s20,    %[s_res_b0] \n"
-"s_mov_b32 s21,    %[s_res_b1] \n"
-"s_mov_b32 s22,    %[s_res_b2] \n"
-"s_mov_b32 s23,    %[s_res_b3] \n"
-// "s_nop  4\n"
-"; -- prefetch A0\n"
-"s_add_u32     m0, 0, %[s_m0_init]                        \n"
-"buffer_load_dword   %[v_os_a0], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                \n"
-"buffer_load_dword   %[v_os_a1], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                \n"
-"buffer_load_dword   %[v_os_a2], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                \n"
-"buffer_load_dword   %[v_os_a3], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                \n"
-"buffer_load_dword   %[v_os_a4], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                \n"
-"buffer_load_dword   %[v_os_a5], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                \n"
-"buffer_load_dword   %[v_os_a6], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                \n"
-"buffer_load_dword   %[v_os_a7], s[16:19], 0 offen lds    \n"
-"s_add_u32 m0, %[smem_sz], %[s_m0_init]                       \n"
-"s_cmp_gt_i32  %[s_loop_cnt] 1             ; move a with cond \n"
-"s_cselect_b32 s86, %[s_tile_os_a], 0      ; move a with cond  \n"
-"s_add_u32     s16, s86, s16               ; move a with cond \n"
-"s_addc_u32    s17, 0, s17                 ; move a with cond \n"
-"; -- prefetch A1\n"
-"buffer_load_dword   %[v_os_a0], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-"buffer_load_dword   %[v_os_a1], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-"buffer_load_dword   %[v_os_a2], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-"buffer_load_dword   %[v_os_a3], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-"buffer_load_dword   %[v_os_a4], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-"buffer_load_dword   %[v_os_a5], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-"buffer_load_dword   %[v_os_a6], s[16:19], 0 offen lds    \n"
-"s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-"buffer_load_dword   %[v_os_a7], s[16:19], 0 offen lds    \n"
-"s_add_u32 m0, 0, %[s_m0_init]                                \n"
-"s_cmp_gt_i32  %[s_loop_cnt] 2             ; move a with cond \n"
-"s_cselect_b32 s86, %[s_tile_os_a], 0      ; move a with cond  \n"
-"s_add_u32     s16, s86, s16               ; move a with cond \n"
-"s_addc_u32    s17, 0, s17                 ; move a with cond \n"
-"; -- prefetch B0\n"
-"buffer_load_dwordx4  acc[0:3], %[v_os_b0], s[20:23], 0 offen \n"
-"buffer_load_dwordx4  acc[4:7], %[v_os_b0], s[20:23], 0 offen offset:1024  \n"
-"buffer_load_dwordx4  acc[8:11], %[v_os_b0], s[20:23], 0 offen offset:2048  \n"
-"buffer_load_dwordx4  acc[12:15], %[v_os_b0], s[20:23], 0 offen offset:3072  \n"
-"buffer_load_dwordx4  acc[16:19], %[v_os_b1], s[20:23], 0 offen  \n"
-"buffer_load_dwordx4  acc[20:23], %[v_os_b1], s[20:23], 0 offen offset:1024  \n"
-"buffer_load_dwordx4  acc[24:27], %[v_os_b1], s[20:23], 0 offen offset:2048  \n"
-"buffer_load_dwordx4  acc[28:31], %[v_os_b1], s[20:23], 0 offen offset:3072  \n"
-"buffer_load_dwordx4  acc[32:35], %[v_os_b2], s[20:23], 0 offen  \n"
-"buffer_load_dwordx4  acc[36:39], %[v_os_b2], s[20:23], 0 offen offset:1024  \n"
-"buffer_load_dwordx4  acc[40:43], %[v_os_b2], s[20:23], 0 offen offset:2048  \n"
-"buffer_load_dwordx4  acc[44:47], %[v_os_b2], s[20:23], 0 offen offset:3072  \n"
-"buffer_load_dwordx4  acc[48:51], %[v_os_b3], s[20:23], 0 offen  \n"
-"buffer_load_dwordx4  acc[52:55], %[v_os_b3], s[20:23], 0 offen offset:1024    \n"
-"buffer_load_dwordx4  acc[56:59], %[v_os_b3], s[20:23], 0 offen offset:2048    \n"
-"buffer_load_dwordx4  acc[60:63], %[v_os_b3], s[20:23], 0 offen offset:3072    \n"
-"buffer_load_dwordx4  acc[64:67], %[v_os_b4], s[20:23], 0 offen                \n"
-"buffer_load_dwordx4  acc[68:71], %[v_os_b4], s[20:23], 0 offen offset:1024    \n"
-"buffer_load_dwordx4  acc[72:75], %[v_os_b4], s[20:23], 0 offen offset:2048    \n"
-"buffer_load_dwordx4  acc[76:79], %[v_os_b4], s[20:23], 0 offen offset:3072    \n"
-"buffer_load_dwordx4  acc[80:83], %[v_os_b5], s[20:23], 0 offen                \n"
-"buffer_load_dwordx4  acc[84:87], %[v_os_b5], s[20:23], 0 offen offset:1024    \n"
-"buffer_load_dwordx4  acc[88:91], %[v_os_b5], s[20:23], 0 offen offset:2048    \n"
-"buffer_load_dwordx4  acc[92:95], %[v_os_b5], s[20:23], 0 offen offset:3072    \n"
-"buffer_load_dwordx4  acc[96:99], %[v_os_b6], s[20:23], 0 offen                \n"
-"buffer_load_dwordx4  acc[100:103], %[v_os_b6], s[20:23], 0 offen offset:1024  \n"
-"buffer_load_dwordx4  acc[104:107], %[v_os_b6], s[20:23], 0 offen offset:2048  \n"
-"buffer_load_dwordx4  acc[108:111], %[v_os_b6], s[20:23], 0 offen offset:3072  \n"
-"buffer_load_dwordx4  acc[112:115], %[v_os_b7], s[20:23], 0 offen              \n"
-"buffer_load_dwordx4  acc[116:119], %[v_os_b7], s[20:23], 0 offen offset:1024  \n"
-"buffer_load_dwordx4  acc[120:123], %[v_os_b7], s[20:23], 0 offen offset:2048  \n"
-"buffer_load_dwordx4  acc[124:127], %[v_os_b7], s[20:23], 0 offen offset:3072  \n"
-"s_cmp_gt_i32  %[s_loop_cnt] 1             ; move b with cond \n"
-"s_cselect_b32 s86, %[s_tile_os_b], 0      ; move b with cond \n"
-"s_add_u32     s20, s86, s20               ; move b with cond \n"
-"s_addc_u32    s21, 0, s21                 ; move b with cond \n"
-"s_waitcnt     vmcnt(40)                        \n"
-"s_barrier                                      \n"
-"ds_read_b128  v[64:67], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_0]\n"    // 1024: N stride, 64 K stride
-"ds_read_b128  v[68:71], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_1]\n"
-"ds_read_b128  v[72:75], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_2]\n"
-"ds_read_b128  v[76:79], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_3]\n"
-"ds_read_b128  v[80:83], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_4]\n"
-"ds_read_b128  v[84:87], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_5]\n"
-"ds_read_b128  v[88:91], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_6]\n"
-"ds_read_b128  v[92:95], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_7]\n"
-"L_start%=:                                                         \n"
-"  s_waitcnt     vmcnt(24) & lgkmcnt(0)                             \n"
-"  s_barrier                                                        \n"
-_UK_MFMA_ "  %[v_acc_0], acc[0:1], v[64:65], %[v_acc_0] \n"
-_UK_MFMA_ "  %[v_acc_0], acc[2:3], v[66:67], %[v_acc_0] \n"
-"  buffer_load_dwordx4  acc[128:131], %[v_os_b0], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_0], acc[4:5], v[68:69], %[v_acc_0] \n"
-_UK_MFMA_ "  %[v_acc_0], acc[6:7], v[70:71], %[v_acc_0] \n"
-"  buffer_load_dword   %[v_os_a0], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_0], acc[8:9], v[72:73], %[v_acc_0] \n"
-_UK_MFMA_ "  %[v_acc_0], acc[10:11], v[74:75], %[v_acc_0] \n"
-"  buffer_load_dwordx4  acc[132:135], %[v_os_b0], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_0], acc[12:13], v[76:77], %[v_acc_0] \n"
-_UK_MFMA_ "  %[v_acc_0], acc[14:15], v[78:79], %[v_acc_0] \n"
-"  buffer_load_dword   %[v_os_a1], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_1], acc[0:1], v[80:81], %[v_acc_1] \n"
-_UK_MFMA_ "  %[v_acc_1], acc[2:3], v[82:83], %[v_acc_1] \n"
-"  buffer_load_dwordx4  acc[136:139], %[v_os_b0], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_1], acc[4:5], v[84:85], %[v_acc_1] \n"
-_UK_MFMA_ "  %[v_acc_1], acc[6:7], v[86:87], %[v_acc_1] \n"
-"  buffer_load_dword   %[v_os_a2], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_1], acc[8:9], v[88:89], %[v_acc_1] \n"
-_UK_MFMA_ "  %[v_acc_1], acc[10:11], v[90:91], %[v_acc_1] \n"
-"  buffer_load_dwordx4  acc[140:143], %[v_os_b0], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_1], acc[12:13], v[92:93], %[v_acc_1] \n"
-_UK_MFMA_ "  %[v_acc_1], acc[14:15], v[94:95], %[v_acc_1] \n"
-"  buffer_load_dword   %[v_os_a3], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_2], acc[16:17], v[64:65], %[v_acc_2] \n"
-_UK_MFMA_ "  %[v_acc_2], acc[18:19], v[66:67], %[v_acc_2] \n"
-"  buffer_load_dwordx4  acc[144:147], %[v_os_b1], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_2], acc[20:21], v[68:69], %[v_acc_2] \n"
-_UK_MFMA_ "  %[v_acc_2], acc[22:23], v[70:71], %[v_acc_2] \n"
-"  buffer_load_dword   %[v_os_a4], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_2], acc[24:25], v[72:73], %[v_acc_2] \n"
-_UK_MFMA_ "  %[v_acc_2], acc[26:27], v[74:75], %[v_acc_2] \n"
-"  buffer_load_dwordx4  acc[148:151], %[v_os_b1], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_2], acc[28:29], v[76:77], %[v_acc_2] \n"
-_UK_MFMA_ "  %[v_acc_2], acc[30:31], v[78:79], %[v_acc_2] \n"
-"  buffer_load_dword   %[v_os_a5], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_3], acc[16:17], v[80:81], %[v_acc_3] \n"
-_UK_MFMA_ "  %[v_acc_3], acc[18:19], v[82:83], %[v_acc_3] \n"
-"  buffer_load_dwordx4  acc[152:155], %[v_os_b1], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_3], acc[20:21], v[84:85], %[v_acc_3] \n"
-_UK_MFMA_ "  %[v_acc_3], acc[22:23], v[86:87], %[v_acc_3] \n"
-"  buffer_load_dword   %[v_os_a6], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_3], acc[24:25], v[88:89], %[v_acc_3] \n"
-_UK_MFMA_ "  %[v_acc_3], acc[26:27], v[90:91], %[v_acc_3] \n"
-"  buffer_load_dwordx4  acc[156:159], %[v_os_b1], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_3], acc[28:29], v[92:93], %[v_acc_3] \n"
-_UK_MFMA_ "  %[v_acc_3], acc[30:31], v[94:95], %[v_acc_3] \n"
-"  buffer_load_dword   %[v_os_a7], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[smem_sz], %[s_m0_init]                  \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-_UK_MFMA_ "  %[v_acc_4], acc[32:33], v[64:65], %[v_acc_4] \n"
-_UK_MFMA_ "  %[v_acc_4], acc[34:35], v[66:67], %[v_acc_4] \n"
-"  buffer_load_dwordx4  acc[160:163], %[v_os_b2], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_4], acc[36:37], v[68:69], %[v_acc_4] \n"
-_UK_MFMA_ "  %[v_acc_4], acc[38:39], v[70:71], %[v_acc_4] \n"
-"  ds_read_b128  v[96:99], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_0]                \n"
-_UK_MFMA_ "  %[v_acc_4], acc[40:41], v[72:73], %[v_acc_4] \n"
-_UK_MFMA_ "  %[v_acc_4], acc[42:43], v[74:75], %[v_acc_4] \n"
-"  buffer_load_dwordx4  acc[164:167], %[v_os_b2], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_4], acc[44:45], v[76:77], %[v_acc_4] \n"
-_UK_MFMA_ "  %[v_acc_4], acc[46:47], v[78:79], %[v_acc_4] \n"
-"  ds_read_b128  v[100:103], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_1]              \n"
-_UK_MFMA_ "  %[v_acc_5], acc[32:33], v[80:81], %[v_acc_5] \n"
-_UK_MFMA_ "  %[v_acc_5], acc[34:35], v[82:83], %[v_acc_5] \n"
-"  buffer_load_dwordx4  acc[168:171], %[v_os_b2], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_5], acc[36:37], v[84:85], %[v_acc_5] \n"
-_UK_MFMA_ "  %[v_acc_5], acc[38:39], v[86:87], %[v_acc_5] \n"
-"  ds_read_b128  v[104:107], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_2]              \n"
-_UK_MFMA_ "  %[v_acc_5], acc[40:41], v[88:89], %[v_acc_5] \n"
-_UK_MFMA_ "  %[v_acc_5], acc[42:43], v[90:91], %[v_acc_5] \n"
-"  buffer_load_dwordx4  acc[172:175], %[v_os_b2], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_5], acc[44:45], v[92:93], %[v_acc_5] \n"
-_UK_MFMA_ "  %[v_acc_5], acc[46:47], v[94:95], %[v_acc_5] \n"
-"  ds_read_b128  v[108:111], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_3]              \n"
-_UK_MFMA_ "  %[v_acc_6], acc[48:49], v[64:65], %[v_acc_6] \n"
-_UK_MFMA_ "  %[v_acc_6], acc[50:51], v[66:67], %[v_acc_6] \n"
-"  buffer_load_dwordx4  acc[176:179], %[v_os_b3], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_6], acc[52:53], v[68:69], %[v_acc_6] \n"
-_UK_MFMA_ "  %[v_acc_6], acc[54:55], v[70:71], %[v_acc_6] \n"
-"  ds_read_b128  v[112:115], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_4]              \n"
-_UK_MFMA_ "  %[v_acc_6], acc[56:57], v[72:73], %[v_acc_6] \n"
-_UK_MFMA_ "  %[v_acc_6], acc[58:59], v[74:75], %[v_acc_6] \n"
-"  buffer_load_dwordx4  acc[180:183], %[v_os_b3], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_6], acc[60:61], v[76:77], %[v_acc_6] \n"
-_UK_MFMA_ "  %[v_acc_6], acc[62:63], v[78:79], %[v_acc_6] \n"
-"  ds_read_b128  v[116:119], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_5]              \n"
-_UK_MFMA_ "  %[v_acc_7], acc[48:49], v[80:81], %[v_acc_7] \n"
-_UK_MFMA_ "  %[v_acc_7], acc[50:51], v[82:83], %[v_acc_7] \n"
-"  buffer_load_dwordx4  acc[184:187], %[v_os_b3], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_7], acc[52:53], v[84:85], %[v_acc_7] \n"
-_UK_MFMA_ "  %[v_acc_7], acc[54:55], v[86:87], %[v_acc_7] \n"
-"  ds_read_b128  v[120:123], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_6]              \n"
-_UK_MFMA_ "  %[v_acc_7], acc[56:57], v[88:89], %[v_acc_7] \n"
-_UK_MFMA_ "  %[v_acc_7], acc[58:59], v[90:91], %[v_acc_7] \n"
-"  buffer_load_dwordx4  acc[188:191], %[v_os_b3], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_7], acc[60:61], v[92:93], %[v_acc_7] \n"
-_UK_MFMA_ "  %[v_acc_7], acc[62:63], v[94:95], %[v_acc_7] \n"
-"  ds_read_b128  v[124:127], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_7]              \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-_UK_MFMA_ "  %[v_acc_8], acc[64:65], v[64:65], %[v_acc_8] \n"
-_UK_MFMA_ "  %[v_acc_8], acc[66:67], v[66:67], %[v_acc_8] \n"
-"  buffer_load_dwordx4  acc[192:195], %[v_os_b4], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_8], acc[68:69], v[68:69], %[v_acc_8] \n"
-_UK_MFMA_ "  %[v_acc_8], acc[70:71], v[70:71], %[v_acc_8] \n"
-_UK_MFMA_ "  %[v_acc_8], acc[72:73], v[72:73], %[v_acc_8] \n"
-_UK_MFMA_ "  %[v_acc_8], acc[74:75], v[74:75], %[v_acc_8] \n"
-"  buffer_load_dwordx4  acc[196:199], %[v_os_b4], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_8], acc[76:77], v[76:77], %[v_acc_8] \n"
-_UK_MFMA_ "  %[v_acc_8], acc[78:79], v[78:79], %[v_acc_8] \n"
-_UK_MFMA_ "  %[v_acc_9], acc[64:65], v[80:81], %[v_acc_9] \n"
-_UK_MFMA_ "  %[v_acc_9], acc[66:67], v[82:83], %[v_acc_9] \n"
-"  buffer_load_dwordx4  acc[200:203], %[v_os_b4], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_9], acc[68:69], v[84:85], %[v_acc_9] \n"
-_UK_MFMA_ "  %[v_acc_9], acc[70:71], v[86:87], %[v_acc_9] \n"
-_UK_MFMA_ "  %[v_acc_9], acc[72:73], v[88:89], %[v_acc_9] \n"
-_UK_MFMA_ "  %[v_acc_9], acc[74:75], v[90:91], %[v_acc_9] \n"
-"  buffer_load_dwordx4  acc[204:207], %[v_os_b4], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_9], acc[76:77], v[92:93], %[v_acc_9] \n"
-_UK_MFMA_ "  %[v_acc_9], acc[78:79], v[94:95], %[v_acc_9] \n"
-_UK_MFMA_ "  %[v_acc_10], acc[80:81], v[64:65], %[v_acc_10] \n"
-_UK_MFMA_ "  %[v_acc_10], acc[82:83], v[66:67], %[v_acc_10] \n"
-"  buffer_load_dwordx4  acc[208:211], %[v_os_b5], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_10], acc[84:85], v[68:69], %[v_acc_10] \n"
-_UK_MFMA_ "  %[v_acc_10], acc[86:87], v[70:71], %[v_acc_10] \n"
-_UK_MFMA_ "  %[v_acc_10], acc[88:89], v[72:73], %[v_acc_10] \n"
-_UK_MFMA_ "  %[v_acc_10], acc[90:91], v[74:75], %[v_acc_10] \n"
-"  buffer_load_dwordx4  acc[212:215], %[v_os_b5], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_10], acc[92:93], v[76:77], %[v_acc_10] \n"
-_UK_MFMA_ "  %[v_acc_10], acc[94:95], v[78:79], %[v_acc_10] \n"
-_UK_MFMA_ "  %[v_acc_11], acc[80:81], v[80:81], %[v_acc_11] \n"
-_UK_MFMA_ "  %[v_acc_11], acc[82:83], v[82:83], %[v_acc_11] \n"
-"  buffer_load_dwordx4  acc[216:219], %[v_os_b5], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_11], acc[84:85], v[84:85], %[v_acc_11] \n"
-_UK_MFMA_ "  %[v_acc_11], acc[86:87], v[86:87], %[v_acc_11] \n"
-_UK_MFMA_ "  %[v_acc_11], acc[88:89], v[88:89], %[v_acc_11] \n"
-_UK_MFMA_ "  %[v_acc_11], acc[90:91], v[90:91], %[v_acc_11] \n"
-"  buffer_load_dwordx4  acc[220:223], %[v_os_b5], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_11], acc[92:93], v[92:93], %[v_acc_11] \n"
-_UK_MFMA_ "  %[v_acc_11], acc[94:95], v[94:95], %[v_acc_11] \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-_UK_MFMA_ "  %[v_acc_12], acc[96:97], v[64:65], %[v_acc_12] \n"
-_UK_MFMA_ "  %[v_acc_12], acc[98:99], v[66:67], %[v_acc_12] \n"
-"  buffer_load_dwordx4  acc[224:227], %[v_os_b6], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_12], acc[100:101], v[68:69], %[v_acc_12] \n"
-_UK_MFMA_ "  %[v_acc_12], acc[102:103], v[70:71], %[v_acc_12] \n"
-_UK_MFMA_ "  %[v_acc_12], acc[104:105], v[72:73], %[v_acc_12] \n"
-_UK_MFMA_ "  %[v_acc_12], acc[106:107], v[74:75], %[v_acc_12] \n"
-"  buffer_load_dwordx4  acc[228:231], %[v_os_b6], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_12], acc[108:109], v[76:77], %[v_acc_12] \n"
-_UK_MFMA_ "  %[v_acc_12], acc[110:111], v[78:79], %[v_acc_12] \n"
-_UK_MFMA_ "  %[v_acc_13], acc[96:97], v[80:81], %[v_acc_13] \n"
-_UK_MFMA_ "  %[v_acc_13], acc[98:99], v[82:83], %[v_acc_13] \n"
-"  buffer_load_dwordx4  acc[232:235], %[v_os_b6], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_13], acc[100:101], v[84:85], %[v_acc_13] \n"
-_UK_MFMA_ "  %[v_acc_13], acc[102:103], v[86:87], %[v_acc_13] \n"
-_UK_MFMA_ "  %[v_acc_13], acc[104:105], v[88:89], %[v_acc_13] \n"
-_UK_MFMA_ "  %[v_acc_13], acc[106:107], v[90:91], %[v_acc_13] \n"
-"  buffer_load_dwordx4  acc[236:239], %[v_os_b6], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_13], acc[108:109], v[92:93], %[v_acc_13] \n"
-_UK_MFMA_ "  %[v_acc_13], acc[110:111], v[94:95], %[v_acc_13] \n"
-_UK_MFMA_ "  %[v_acc_14], acc[112:113], v[64:65], %[v_acc_14] \n"
-_UK_MFMA_ "  %[v_acc_14], acc[114:115], v[66:67], %[v_acc_14] \n"
-"  buffer_load_dwordx4  acc[240:243], %[v_os_b7], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_14], acc[116:117], v[68:69], %[v_acc_14] \n"
-_UK_MFMA_ "  %[v_acc_14], acc[118:119], v[70:71], %[v_acc_14] \n"
-_UK_MFMA_ "  %[v_acc_14], acc[120:121], v[72:73], %[v_acc_14] \n"
-_UK_MFMA_ "  %[v_acc_14], acc[122:123], v[74:75], %[v_acc_14] \n"
-"  buffer_load_dwordx4  acc[244:247], %[v_os_b7], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_14], acc[124:125], v[76:77], %[v_acc_14] \n"
-_UK_MFMA_ "  %[v_acc_14], acc[126:127], v[78:79], %[v_acc_14] \n"
-_UK_MFMA_ "  %[v_acc_15], acc[112:113], v[80:81], %[v_acc_15] \n"
-_UK_MFMA_ "  %[v_acc_15], acc[114:115], v[82:83], %[v_acc_15] \n"
-"  buffer_load_dwordx4  acc[248:251], %[v_os_b7], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_15], acc[116:117], v[84:85], %[v_acc_15] \n"
-_UK_MFMA_ "  %[v_acc_15], acc[118:119], v[86:87], %[v_acc_15] \n"
-_UK_MFMA_ "  %[v_acc_15], acc[120:121], v[88:89], %[v_acc_15] \n"
-_UK_MFMA_ "  %[v_acc_15], acc[122:123], v[90:91], %[v_acc_15] \n"
-"  buffer_load_dwordx4  acc[252:255], %[v_os_b7], s[20:23], 0 offen offset:3072\n"
-_UK_MFMA_ "  %[v_acc_15], acc[124:125], v[92:93], %[v_acc_15] \n"
-_UK_MFMA_ "  %[v_acc_15], acc[126:127], v[94:95], %[v_acc_15] \n"
-"  s_sub_i32     %[s_loop_cnt], %[s_loop_cnt], 1                \n"
-"  s_cmp_gt_i32  %[s_loop_cnt] 0                                \n"
-"  s_cbranch_scc0 L_end%=                                       \n"
-"  s_cmp_gt_i32  %[s_loop_cnt] 2             ; move a with cond \n"
-"  s_cselect_b32 s86, %[s_tile_os_a], 0                          \n"
-"  s_add_u32     s16, s86, s16                                  \n"
-"  s_addc_u32    s17, 0, s17                                    \n"
-"  s_cmp_gt_i32  %[s_loop_cnt] 1             ; move b with cond \n"
-"  s_cselect_b32 s86, %[s_tile_os_b], 0                          \n"
-"  s_add_u32     s20, s86, s20                                  \n"
-"  s_addc_u32    s21, 0, s21                                    \n"
-"  ;------------------------------------------                  \n"
-"  s_waitcnt     vmcnt(24) & lgkmcnt(0)                  \n"
-"  s_barrier                                             \n"
-_UK_MFMA_ "  %[v_acc_0], acc[128:129], v[96:97], %[v_acc_0] \n"
-_UK_MFMA_ "  %[v_acc_0], acc[130:131], v[98:99], %[v_acc_0] \n"
-"  buffer_load_dwordx4  acc[0:3], %[v_os_b0], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_0], acc[132:133], v[100:101], %[v_acc_0] \n"
-_UK_MFMA_ "  %[v_acc_0], acc[134:135], v[102:103], %[v_acc_0] \n"
-"  buffer_load_dword   %[v_os_a0], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_0], acc[136:137], v[104:105], %[v_acc_0] \n"
-_UK_MFMA_ "  %[v_acc_0], acc[138:139], v[106:107], %[v_acc_0] \n"
-"  buffer_load_dwordx4  acc[4:7], %[v_os_b0], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_0], acc[140:141], v[108:109], %[v_acc_0] \n"
-_UK_MFMA_ "  %[v_acc_0], acc[142:143], v[110:111], %[v_acc_0] \n"
-"  buffer_load_dword   %[v_os_a1], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_1], acc[128:129], v[112:113], %[v_acc_1] \n"
-_UK_MFMA_ "  %[v_acc_1], acc[130:131], v[114:115], %[v_acc_1] \n"
-"  buffer_load_dwordx4  acc[8:11], %[v_os_b0], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_1], acc[132:133], v[116:117], %[v_acc_1] \n"
-_UK_MFMA_ "  %[v_acc_1], acc[134:135], v[118:119], %[v_acc_1] \n"
-"  buffer_load_dword   %[v_os_a2], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_1], acc[136:137], v[120:121], %[v_acc_1] \n"
-_UK_MFMA_ "  %[v_acc_1], acc[138:139], v[122:123], %[v_acc_1] \n"
-"  buffer_load_dwordx4  acc[12:15], %[v_os_b0], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_1], acc[140:141], v[124:125], %[v_acc_1] \n"
-_UK_MFMA_ "  %[v_acc_1], acc[142:143], v[126:127], %[v_acc_1] \n"
-"  buffer_load_dword   %[v_os_a3], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_2], acc[144:145], v[96:97], %[v_acc_2] \n"
-_UK_MFMA_ "  %[v_acc_2], acc[146:147], v[98:99], %[v_acc_2] \n"
-"  buffer_load_dwordx4  acc[16:19], %[v_os_b1], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_2], acc[148:149], v[100:101], %[v_acc_2] \n"
-_UK_MFMA_ "  %[v_acc_2], acc[150:151], v[102:103], %[v_acc_2] \n"
-"  buffer_load_dword   %[v_os_a4], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_2], acc[152:153], v[104:105], %[v_acc_2] \n"
-_UK_MFMA_ "  %[v_acc_2], acc[154:155], v[106:107], %[v_acc_2] \n"
-"  buffer_load_dwordx4  acc[20:23], %[v_os_b1], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_2], acc[156:157], v[108:109], %[v_acc_2] \n"
-_UK_MFMA_ "  %[v_acc_2], acc[158:159], v[110:111], %[v_acc_2] \n"
-"  buffer_load_dword   %[v_os_a5], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_3], acc[144:145], v[112:113], %[v_acc_3] \n"
-_UK_MFMA_ "  %[v_acc_3], acc[146:147], v[114:115], %[v_acc_3] \n"
-"  buffer_load_dwordx4  acc[24:27], %[v_os_b1], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_3], acc[148:149], v[116:117], %[v_acc_3] \n"
-_UK_MFMA_ "  %[v_acc_3], acc[150:151], v[118:119], %[v_acc_3] \n"
-"  buffer_load_dword   %[v_os_a6], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, %[s_size_per_issue], m0                  \n"
-_UK_MFMA_ "  %[v_acc_3], acc[152:153], v[120:121], %[v_acc_3] \n"
-_UK_MFMA_ "  %[v_acc_3], acc[154:155], v[122:123], %[v_acc_3] \n"
-"  buffer_load_dwordx4  acc[28:31], %[v_os_b1], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_3], acc[156:157], v[124:125], %[v_acc_3] \n"
-_UK_MFMA_ "  %[v_acc_3], acc[158:159], v[126:127], %[v_acc_3] \n"
-"  buffer_load_dword   %[v_os_a7], s[16:19], 0 offen lds     \n"
-"  s_add_u32     m0, 0, %[s_m0_init]                  \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-_UK_MFMA_ "  %[v_acc_4], acc[160:161], v[96:97], %[v_acc_4] \n"
-_UK_MFMA_ "  %[v_acc_4], acc[162:163], v[98:99], %[v_acc_4] \n"
-"  buffer_load_dwordx4  acc[32:35], %[v_os_b2], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_4], acc[164:165], v[100:101], %[v_acc_4] \n"
-_UK_MFMA_ "  %[v_acc_4], acc[166:167], v[102:103], %[v_acc_4] \n"
-"  ds_read_b128  v[64:67], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_0]  \n"
-_UK_MFMA_ "  %[v_acc_4], acc[168:169], v[104:105], %[v_acc_4] \n"
-_UK_MFMA_ "  %[v_acc_4], acc[170:171], v[106:107], %[v_acc_4] \n"
-"  buffer_load_dwordx4  acc[36:39], %[v_os_b2], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_4], acc[172:173], v[108:109], %[v_acc_4] \n"
-_UK_MFMA_ "  %[v_acc_4], acc[174:175], v[110:111], %[v_acc_4] \n"
-"  ds_read_b128  v[68:71], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_1]  \n"
-_UK_MFMA_ "  %[v_acc_5], acc[160:161], v[112:113], %[v_acc_5] \n"
-_UK_MFMA_ "  %[v_acc_5], acc[162:163], v[114:115], %[v_acc_5] \n"
-"  buffer_load_dwordx4  acc[40:43], %[v_os_b2], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_5], acc[164:165], v[116:117], %[v_acc_5] \n"
-_UK_MFMA_ "  %[v_acc_5], acc[166:167], v[118:119], %[v_acc_5] \n"
-"  ds_read_b128  v[72:75], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_2]                 \n"
-_UK_MFMA_ "  %[v_acc_5], acc[168:169], v[120:121], %[v_acc_5] \n"
-_UK_MFMA_ "  %[v_acc_5], acc[170:171], v[122:123], %[v_acc_5] \n"
-"  buffer_load_dwordx4  acc[44:47], %[v_os_b2], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_5], acc[172:173], v[124:125], %[v_acc_5] \n"
-_UK_MFMA_ "  %[v_acc_5], acc[174:175], v[126:127], %[v_acc_5] \n"
-"  ds_read_b128  v[76:79], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_3]                \n"
-_UK_MFMA_ "  %[v_acc_6], acc[176:177], v[96:97], %[v_acc_6] \n"
-_UK_MFMA_ "  %[v_acc_6], acc[178:179], v[98:99], %[v_acc_6] \n"
-"  buffer_load_dwordx4  acc[48:51], %[v_os_b3], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_6], acc[180:181], v[100:101], %[v_acc_6] \n"
-_UK_MFMA_ "  %[v_acc_6], acc[182:183], v[102:103], %[v_acc_6] \n"
-"  ds_read_b128  v[80:83], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_4]               \n"
-_UK_MFMA_ "  %[v_acc_6], acc[184:185], v[104:105], %[v_acc_6] \n"
-_UK_MFMA_ "  %[v_acc_6], acc[186:187], v[106:107], %[v_acc_6] \n"
-"  buffer_load_dwordx4  acc[52:55], %[v_os_b3], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_6], acc[188:189], v[108:109], %[v_acc_6] \n"
-_UK_MFMA_ "  %[v_acc_6], acc[190:191], v[110:111], %[v_acc_6] \n"
-"  ds_read_b128  v[84:87], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_5]            \n"
-_UK_MFMA_ "  %[v_acc_7], acc[176:177], v[112:113], %[v_acc_7] \n"
-_UK_MFMA_ "  %[v_acc_7], acc[178:179], v[114:115], %[v_acc_7] \n"
-"  buffer_load_dwordx4  acc[56:59], %[v_os_b3], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_7], acc[180:181], v[116:117], %[v_acc_7] \n"
-_UK_MFMA_ "  %[v_acc_7], acc[182:183], v[118:119], %[v_acc_7] \n"
-"  ds_read_b128  v[88:91], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_6]                \n"
-_UK_MFMA_ "  %[v_acc_7], acc[184:185], v[120:121], %[v_acc_7] \n"
-_UK_MFMA_ "  %[v_acc_7], acc[186:187], v[122:123], %[v_acc_7] \n"
-"  buffer_load_dwordx4  acc[60:63], %[v_os_b3], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_7], acc[188:189], v[124:125], %[v_acc_7] \n"
-_UK_MFMA_ "  %[v_acc_7], acc[190:191], v[126:127], %[v_acc_7] \n"
-"  ds_read_b128  v[92:95], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_7]           \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-_UK_MFMA_ "  %[v_acc_8], acc[192:193], v[96:97], %[v_acc_8] \n"
-_UK_MFMA_ "  %[v_acc_8], acc[194:195], v[98:99], %[v_acc_8] \n"
-"  buffer_load_dwordx4  acc[64:67], %[v_os_b4], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_8], acc[196:197], v[100:101], %[v_acc_8] \n"
-_UK_MFMA_ "  %[v_acc_8], acc[198:199], v[102:103], %[v_acc_8] \n"
-_UK_MFMA_ "  %[v_acc_8], acc[200:201], v[104:105], %[v_acc_8] \n"
-_UK_MFMA_ "  %[v_acc_8], acc[202:203], v[106:107], %[v_acc_8] \n"
-"  buffer_load_dwordx4  acc[68:71], %[v_os_b4], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_8], acc[204:205], v[108:109], %[v_acc_8] \n"
-_UK_MFMA_ "  %[v_acc_8], acc[206:207], v[110:111], %[v_acc_8] \n"
-_UK_MFMA_ "  %[v_acc_9], acc[192:193], v[112:113], %[v_acc_9] \n"
-_UK_MFMA_ "  %[v_acc_9], acc[194:195], v[114:115], %[v_acc_9] \n"
-"  buffer_load_dwordx4  acc[72:75], %[v_os_b4], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_9], acc[196:197], v[116:117], %[v_acc_9] \n"
-_UK_MFMA_ "  %[v_acc_9], acc[198:199], v[118:119], %[v_acc_9] \n"
-_UK_MFMA_ "  %[v_acc_9], acc[200:201], v[120:121], %[v_acc_9] \n"
-_UK_MFMA_ "  %[v_acc_9], acc[202:203], v[122:123], %[v_acc_9] \n"
-"  buffer_load_dwordx4  acc[76:79], %[v_os_b4], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_9], acc[204:205], v[124:125], %[v_acc_9] \n"
-_UK_MFMA_ "  %[v_acc_9], acc[206:207], v[126:127], %[v_acc_9] \n"
-_UK_MFMA_ "  %[v_acc_10], acc[208:209], v[96:97], %[v_acc_10] \n"
-_UK_MFMA_ "  %[v_acc_10], acc[210:211], v[98:99], %[v_acc_10] \n"
-"  buffer_load_dwordx4  acc[80:83], %[v_os_b5], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_10], acc[212:213], v[100:101], %[v_acc_10] \n"
-_UK_MFMA_ "  %[v_acc_10], acc[214:215], v[102:103], %[v_acc_10] \n"
-_UK_MFMA_ "  %[v_acc_10], acc[216:217], v[104:105], %[v_acc_10] \n"
-_UK_MFMA_ "  %[v_acc_10], acc[218:219], v[106:107], %[v_acc_10] \n"
-"  buffer_load_dwordx4  acc[84:87], %[v_os_b5], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_10], acc[220:221], v[108:109], %[v_acc_10] \n"
-_UK_MFMA_ "  %[v_acc_10], acc[222:223], v[110:111], %[v_acc_10] \n"
-_UK_MFMA_ "  %[v_acc_11], acc[208:209], v[112:113], %[v_acc_11] \n"
-_UK_MFMA_ "  %[v_acc_11], acc[210:211], v[114:115], %[v_acc_11] \n"
-"  buffer_load_dwordx4  acc[88:91], %[v_os_b5], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_11], acc[212:213], v[116:117], %[v_acc_11] \n"
-_UK_MFMA_ "  %[v_acc_11], acc[214:215], v[118:119], %[v_acc_11] \n"
-_UK_MFMA_ "  %[v_acc_11], acc[216:217], v[120:121], %[v_acc_11] \n"
-_UK_MFMA_ "  %[v_acc_11], acc[218:219], v[122:123], %[v_acc_11] \n"
-"  buffer_load_dwordx4  acc[92:95], %[v_os_b5], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_11], acc[220:221], v[124:125], %[v_acc_11] \n"
-_UK_MFMA_ "  %[v_acc_11], acc[222:223], v[126:127], %[v_acc_11] \n"
-"  s_waitcnt     vmcnt(32)                               \n"
-_UK_MFMA_ "  %[v_acc_12], acc[224:225], v[96:97], %[v_acc_12] \n"
-_UK_MFMA_ "  %[v_acc_12], acc[226:227], v[98:99], %[v_acc_12] \n"
-"  buffer_load_dwordx4  acc[96:99], %[v_os_b6], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_12], acc[228:229], v[100:101], %[v_acc_12] \n"
-_UK_MFMA_ "  %[v_acc_12], acc[230:231], v[102:103], %[v_acc_12] \n"
-_UK_MFMA_ "  %[v_acc_12], acc[232:233], v[104:105], %[v_acc_12] \n"
-_UK_MFMA_ "  %[v_acc_12], acc[234:235], v[106:107], %[v_acc_12] \n"
-"  buffer_load_dwordx4  acc[100:103], %[v_os_b6], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_12], acc[236:237], v[108:109], %[v_acc_12] \n"
-_UK_MFMA_ "  %[v_acc_12], acc[238:239], v[110:111], %[v_acc_12] \n"
-_UK_MFMA_ "  %[v_acc_13], acc[224:225], v[112:113], %[v_acc_13] \n"
-_UK_MFMA_ "  %[v_acc_13], acc[226:227], v[114:115], %[v_acc_13] \n"
-"  buffer_load_dwordx4  acc[104:107], %[v_os_b6], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_13], acc[228:229], v[116:117], %[v_acc_13] \n"
-_UK_MFMA_ "  %[v_acc_13], acc[230:231], v[118:119], %[v_acc_13] \n"
-_UK_MFMA_ "  %[v_acc_13], acc[232:233], v[120:121], %[v_acc_13] \n"
-_UK_MFMA_ "  %[v_acc_13], acc[234:235], v[122:123], %[v_acc_13] \n"
-"  buffer_load_dwordx4  acc[108:111], %[v_os_b6], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_13], acc[236:237], v[124:125], %[v_acc_13] \n"
-_UK_MFMA_ "  %[v_acc_13], acc[238:239], v[126:127], %[v_acc_13] \n"
-_UK_MFMA_ "  %[v_acc_14], acc[240:241], v[96:97], %[v_acc_14] \n"
-_UK_MFMA_ "  %[v_acc_14], acc[242:243], v[98:99], %[v_acc_14] \n"
-"  buffer_load_dwordx4  acc[112:115], %[v_os_b7], s[20:23], 0 offen \n"
-_UK_MFMA_ "  %[v_acc_14], acc[244:245], v[100:101], %[v_acc_14] \n"
-_UK_MFMA_ "  %[v_acc_14], acc[246:247], v[102:103], %[v_acc_14] \n"
-_UK_MFMA_ "  %[v_acc_14], acc[248:249], v[104:105], %[v_acc_14] \n"
-_UK_MFMA_ "  %[v_acc_14], acc[250:251], v[106:107], %[v_acc_14] \n"
-"  buffer_load_dwordx4  acc[116:119], %[v_os_b7], s[20:23], 0 offen offset:1024 \n"
-_UK_MFMA_ "  %[v_acc_14], acc[252:253], v[108:109], %[v_acc_14] \n"
-_UK_MFMA_ "  %[v_acc_14], acc[254:255], v[110:111], %[v_acc_14] \n"
-_UK_MFMA_ "  %[v_acc_15], acc[240:241], v[112:113], %[v_acc_15] \n"
-_UK_MFMA_ "  %[v_acc_15], acc[242:243], v[114:115], %[v_acc_15] \n"
-"  buffer_load_dwordx4  acc[120:123], %[v_os_b7], s[20:23], 0 offen offset:2048 \n"
-_UK_MFMA_ "  %[v_acc_15], acc[244:245], v[116:117], %[v_acc_15] \n"
-_UK_MFMA_ "  %[v_acc_15], acc[246:247], v[118:119], %[v_acc_15] \n"
-_UK_MFMA_ "  %[v_acc_15], acc[248:249], v[120:121], %[v_acc_15] \n"
-_UK_MFMA_ "  %[v_acc_15], acc[250:251], v[122:123], %[v_acc_15] \n"
-"  buffer_load_dwordx4  acc[124:127], %[v_os_b7], s[20:23], 0 offen offset:3072 \n"
-_UK_MFMA_ "  %[v_acc_15], acc[252:253], v[124:125], %[v_acc_15] \n"
-_UK_MFMA_ "  %[v_acc_15], acc[254:255], v[126:127], %[v_acc_15] \n"
-"  s_sub_i32     %[s_loop_cnt], %[s_loop_cnt], 1                \n"
-"  s_cmp_gt_i32  %[s_loop_cnt] 0                                \n"
-"  s_cbranch_scc0 L_end%=                                       \n"
-"  s_cmp_gt_i32  %[s_loop_cnt] 2             ; move a with cond \n"
-"  s_cselect_b32 s86, %[s_tile_os_a], 0                          \n"
-"  s_add_u32     s16, s86, s16                                  \n"
-"  s_addc_u32    s17, 0, s17                                    \n"
-"  s_cmp_gt_i32  %[s_loop_cnt] 1             ; move b with cond \n"
-"  s_cselect_b32 s86, %[s_tile_os_b], 0                          \n"
-"  s_add_u32     s20, s86, s20                                  \n"
-"  s_addc_u32    s21, 0, s21                                    \n"
-"  s_branch     L_start%=                                       \n"
-"L_end%=:                                                       \n"
-"  s_nop 2                                                      \n"
+    "s_mov_b32 s17,    %[s_res_a1] \n"
+    "s_mov_b32 s18,    %[s_res_a2] \n"
+    "s_mov_b32 s19,    %[s_res_a3] \n"
+    "s_mov_b32 s20,    %[s_res_b0] \n"
+    "s_mov_b32 s21,    %[s_res_b1] \n"
+    "s_mov_b32 s22,    %[s_res_b2] \n"
+    "s_mov_b32 s23,    %[s_res_b3] \n"
+    // "s_nop  4\n"
+    "; -- prefetch A0\n"
+    "s_add_u32     m0, 0, %[s_m0_init]                        \n"
+    "buffer_load_dword   %[v_os_a0], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                \n"
+    "buffer_load_dword   %[v_os_a1], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                \n"
+    "buffer_load_dword   %[v_os_a2], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                \n"
+    "buffer_load_dword   %[v_os_a3], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                \n"
+    "buffer_load_dword   %[v_os_a4], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                \n"
+    "buffer_load_dword   %[v_os_a5], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                \n"
+    "buffer_load_dword   %[v_os_a6], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                \n"
+    "buffer_load_dword   %[v_os_a7], s[16:19], 0 offen lds    \n"
+    "s_add_u32 m0, %[smem_sz], %[s_m0_init]                       \n"
+    "s_cmp_gt_i32  %[s_loop_cnt] 1             ; move a with cond \n"
+    "s_cselect_b32 s86, %[s_tile_os_a], 0      ; move a with cond  \n"
+    "s_add_u32     s16, s86, s16               ; move a with cond \n"
+    "s_addc_u32    s17, 0, s17                 ; move a with cond \n"
+    "; -- prefetch A1\n"
+    "buffer_load_dword   %[v_os_a0], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                  \n"
+    "buffer_load_dword   %[v_os_a1], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                  \n"
+    "buffer_load_dword   %[v_os_a2], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                  \n"
+    "buffer_load_dword   %[v_os_a3], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                  \n"
+    "buffer_load_dword   %[v_os_a4], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                  \n"
+    "buffer_load_dword   %[v_os_a5], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                  \n"
+    "buffer_load_dword   %[v_os_a6], s[16:19], 0 offen lds    \n"
+    "s_add_u32     m0, %[s_size_per_issue], m0                  \n"
+    "buffer_load_dword   %[v_os_a7], s[16:19], 0 offen lds    \n"
+    "s_add_u32 m0, 0, %[s_m0_init]                                \n"
+    "s_cmp_gt_i32  %[s_loop_cnt] 2             ; move a with cond \n"
+    "s_cselect_b32 s86, %[s_tile_os_a], 0      ; move a with cond  \n"
+    "s_add_u32     s16, s86, s16               ; move a with cond \n"
+    "s_addc_u32    s17, 0, s17                 ; move a with cond \n"
+    "; -- prefetch B0\n"
+    "buffer_load_dwordx4  acc[0:3], %[v_os_b0], s[20:23], 0 offen \n"
+    "buffer_load_dwordx4  acc[4:7], %[v_os_b0], s[20:23], 0 offen offset:1024  \n"
+    "buffer_load_dwordx4  acc[8:11], %[v_os_b0], s[20:23], 0 offen offset:2048  \n"
+    "buffer_load_dwordx4  acc[12:15], %[v_os_b0], s[20:23], 0 offen offset:3072  \n"
+    "buffer_load_dwordx4  acc[16:19], %[v_os_b1], s[20:23], 0 offen  \n"
+    "buffer_load_dwordx4  acc[20:23], %[v_os_b1], s[20:23], 0 offen offset:1024  \n"
+    "buffer_load_dwordx4  acc[24:27], %[v_os_b1], s[20:23], 0 offen offset:2048  \n"
+    "buffer_load_dwordx4  acc[28:31], %[v_os_b1], s[20:23], 0 offen offset:3072  \n"
+    "buffer_load_dwordx4  acc[32:35], %[v_os_b2], s[20:23], 0 offen  \n"
+    "buffer_load_dwordx4  acc[36:39], %[v_os_b2], s[20:23], 0 offen offset:1024  \n"
+    "buffer_load_dwordx4  acc[40:43], %[v_os_b2], s[20:23], 0 offen offset:2048  \n"
+    "buffer_load_dwordx4  acc[44:47], %[v_os_b2], s[20:23], 0 offen offset:3072  \n"
+    "buffer_load_dwordx4  acc[48:51], %[v_os_b3], s[20:23], 0 offen  \n"
+    "buffer_load_dwordx4  acc[52:55], %[v_os_b3], s[20:23], 0 offen offset:1024    \n"
+    "buffer_load_dwordx4  acc[56:59], %[v_os_b3], s[20:23], 0 offen offset:2048    \n"
+    "buffer_load_dwordx4  acc[60:63], %[v_os_b3], s[20:23], 0 offen offset:3072    \n"
+    "buffer_load_dwordx4  acc[64:67], %[v_os_b4], s[20:23], 0 offen                \n"
+    "buffer_load_dwordx4  acc[68:71], %[v_os_b4], s[20:23], 0 offen offset:1024    \n"
+    "buffer_load_dwordx4  acc[72:75], %[v_os_b4], s[20:23], 0 offen offset:2048    \n"
+    "buffer_load_dwordx4  acc[76:79], %[v_os_b4], s[20:23], 0 offen offset:3072    \n"
+    "buffer_load_dwordx4  acc[80:83], %[v_os_b5], s[20:23], 0 offen                \n"
+    "buffer_load_dwordx4  acc[84:87], %[v_os_b5], s[20:23], 0 offen offset:1024    \n"
+    "buffer_load_dwordx4  acc[88:91], %[v_os_b5], s[20:23], 0 offen offset:2048    \n"
+    "buffer_load_dwordx4  acc[92:95], %[v_os_b5], s[20:23], 0 offen offset:3072    \n"
+    "buffer_load_dwordx4  acc[96:99], %[v_os_b6], s[20:23], 0 offen                \n"
+    "buffer_load_dwordx4  acc[100:103], %[v_os_b6], s[20:23], 0 offen offset:1024  \n"
+    "buffer_load_dwordx4  acc[104:107], %[v_os_b6], s[20:23], 0 offen offset:2048  \n"
+    "buffer_load_dwordx4  acc[108:111], %[v_os_b6], s[20:23], 0 offen offset:3072  \n"
+    "buffer_load_dwordx4  acc[112:115], %[v_os_b7], s[20:23], 0 offen              \n"
+    "buffer_load_dwordx4  acc[116:119], %[v_os_b7], s[20:23], 0 offen offset:1024  \n"
+    "buffer_load_dwordx4  acc[120:123], %[v_os_b7], s[20:23], 0 offen offset:2048  \n"
+    "buffer_load_dwordx4  acc[124:127], %[v_os_b7], s[20:23], 0 offen offset:3072  \n"
+    "s_cmp_gt_i32  %[s_loop_cnt] 1             ; move b with cond \n"
+    "s_cselect_b32 s86, %[s_tile_os_b], 0      ; move b with cond \n"
+    "s_add_u32     s20, s86, s20               ; move b with cond \n"
+    "s_addc_u32    s21, 0, s21                 ; move b with cond \n"
+    "s_waitcnt     vmcnt(40)                        \n"
+    "s_barrier                                      \n"
+    "ds_read_b128  v[64:67], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_0]\n" // 1024: N stride, 64
+                                                                               // K stride
+    "ds_read_b128  v[68:71], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_1]\n"
+    "ds_read_b128  v[72:75], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_2]\n"
+    "ds_read_b128  v[76:79], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_3]\n"
+    "ds_read_b128  v[80:83], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_4]\n"
+    "ds_read_b128  v[84:87], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_5]\n"
+    "ds_read_b128  v[88:91], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_6]\n"
+    "ds_read_b128  v[92:95], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_7]\n"
+    "L_start%=:                                                         \n"
+    "  s_waitcnt     vmcnt(24) & lgkmcnt(0)                             \n"
+    "  s_barrier                                                        \n" _UK_MFMA_
+    "  %[v_acc_0], acc[0:1], v[64:65], %[v_acc_0] \n" _UK_MFMA_
+    "  %[v_acc_0], acc[2:3], v[66:67], %[v_acc_0] \n"
+    "  buffer_load_dwordx4  acc[128:131], %[v_os_b0], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_0], acc[4:5], v[68:69], %[v_acc_0] \n" _UK_MFMA_
+    "  %[v_acc_0], acc[6:7], v[70:71], %[v_acc_0] \n"
+    "  buffer_load_dword   %[v_os_a0], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_0], acc[8:9], v[72:73], %[v_acc_0] \n" _UK_MFMA_
+    "  %[v_acc_0], acc[10:11], v[74:75], %[v_acc_0] \n"
+    "  buffer_load_dwordx4  acc[132:135], %[v_os_b0], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_0], acc[12:13], v[76:77], %[v_acc_0] \n" _UK_MFMA_
+    "  %[v_acc_0], acc[14:15], v[78:79], %[v_acc_0] \n"
+    "  buffer_load_dword   %[v_os_a1], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_1], acc[0:1], v[80:81], %[v_acc_1] \n" _UK_MFMA_
+    "  %[v_acc_1], acc[2:3], v[82:83], %[v_acc_1] \n"
+    "  buffer_load_dwordx4  acc[136:139], %[v_os_b0], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_1], acc[4:5], v[84:85], %[v_acc_1] \n" _UK_MFMA_
+    "  %[v_acc_1], acc[6:7], v[86:87], %[v_acc_1] \n"
+    "  buffer_load_dword   %[v_os_a2], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_1], acc[8:9], v[88:89], %[v_acc_1] \n" _UK_MFMA_
+    "  %[v_acc_1], acc[10:11], v[90:91], %[v_acc_1] \n"
+    "  buffer_load_dwordx4  acc[140:143], %[v_os_b0], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_1], acc[12:13], v[92:93], %[v_acc_1] \n" _UK_MFMA_
+    "  %[v_acc_1], acc[14:15], v[94:95], %[v_acc_1] \n"
+    "  buffer_load_dword   %[v_os_a3], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_2], acc[16:17], v[64:65], %[v_acc_2] \n" _UK_MFMA_
+    "  %[v_acc_2], acc[18:19], v[66:67], %[v_acc_2] \n"
+    "  buffer_load_dwordx4  acc[144:147], %[v_os_b1], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_2], acc[20:21], v[68:69], %[v_acc_2] \n" _UK_MFMA_
+    "  %[v_acc_2], acc[22:23], v[70:71], %[v_acc_2] \n"
+    "  buffer_load_dword   %[v_os_a4], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_2], acc[24:25], v[72:73], %[v_acc_2] \n" _UK_MFMA_
+    "  %[v_acc_2], acc[26:27], v[74:75], %[v_acc_2] \n"
+    "  buffer_load_dwordx4  acc[148:151], %[v_os_b1], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_2], acc[28:29], v[76:77], %[v_acc_2] \n" _UK_MFMA_
+    "  %[v_acc_2], acc[30:31], v[78:79], %[v_acc_2] \n"
+    "  buffer_load_dword   %[v_os_a5], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_3], acc[16:17], v[80:81], %[v_acc_3] \n" _UK_MFMA_
+    "  %[v_acc_3], acc[18:19], v[82:83], %[v_acc_3] \n"
+    "  buffer_load_dwordx4  acc[152:155], %[v_os_b1], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_3], acc[20:21], v[84:85], %[v_acc_3] \n" _UK_MFMA_
+    "  %[v_acc_3], acc[22:23], v[86:87], %[v_acc_3] \n"
+    "  buffer_load_dword   %[v_os_a6], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_3], acc[24:25], v[88:89], %[v_acc_3] \n" _UK_MFMA_
+    "  %[v_acc_3], acc[26:27], v[90:91], %[v_acc_3] \n"
+    "  buffer_load_dwordx4  acc[156:159], %[v_os_b1], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_3], acc[28:29], v[92:93], %[v_acc_3] \n" _UK_MFMA_
+    "  %[v_acc_3], acc[30:31], v[94:95], %[v_acc_3] \n"
+    "  buffer_load_dword   %[v_os_a7], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[smem_sz], %[s_m0_init]                  \n"
+    "  s_waitcnt     vmcnt(32)                               \n" _UK_MFMA_
+    "  %[v_acc_4], acc[32:33], v[64:65], %[v_acc_4] \n" _UK_MFMA_
+    "  %[v_acc_4], acc[34:35], v[66:67], %[v_acc_4] \n"
+    "  buffer_load_dwordx4  acc[160:163], %[v_os_b2], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_4], acc[36:37], v[68:69], %[v_acc_4] \n" _UK_MFMA_
+    "  %[v_acc_4], acc[38:39], v[70:71], %[v_acc_4] \n"
+    "  ds_read_b128  v[96:99], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_0]                "
+    "\n" _UK_MFMA_ "  %[v_acc_4], acc[40:41], v[72:73], %[v_acc_4] \n" _UK_MFMA_
+    "  %[v_acc_4], acc[42:43], v[74:75], %[v_acc_4] \n"
+    "  buffer_load_dwordx4  acc[164:167], %[v_os_b2], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_4], acc[44:45], v[76:77], %[v_acc_4] \n" _UK_MFMA_
+    "  %[v_acc_4], acc[46:47], v[78:79], %[v_acc_4] \n"
+    "  ds_read_b128  v[100:103], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_1]              "
+    "\n" _UK_MFMA_ "  %[v_acc_5], acc[32:33], v[80:81], %[v_acc_5] \n" _UK_MFMA_
+    "  %[v_acc_5], acc[34:35], v[82:83], %[v_acc_5] \n"
+    "  buffer_load_dwordx4  acc[168:171], %[v_os_b2], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_5], acc[36:37], v[84:85], %[v_acc_5] \n" _UK_MFMA_
+    "  %[v_acc_5], acc[38:39], v[86:87], %[v_acc_5] \n"
+    "  ds_read_b128  v[104:107], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_2]              "
+    "\n" _UK_MFMA_ "  %[v_acc_5], acc[40:41], v[88:89], %[v_acc_5] \n" _UK_MFMA_
+    "  %[v_acc_5], acc[42:43], v[90:91], %[v_acc_5] \n"
+    "  buffer_load_dwordx4  acc[172:175], %[v_os_b2], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_5], acc[44:45], v[92:93], %[v_acc_5] \n" _UK_MFMA_
+    "  %[v_acc_5], acc[46:47], v[94:95], %[v_acc_5] \n"
+    "  ds_read_b128  v[108:111], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_3]              "
+    "\n" _UK_MFMA_ "  %[v_acc_6], acc[48:49], v[64:65], %[v_acc_6] \n" _UK_MFMA_
+    "  %[v_acc_6], acc[50:51], v[66:67], %[v_acc_6] \n"
+    "  buffer_load_dwordx4  acc[176:179], %[v_os_b3], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_6], acc[52:53], v[68:69], %[v_acc_6] \n" _UK_MFMA_
+    "  %[v_acc_6], acc[54:55], v[70:71], %[v_acc_6] \n"
+    "  ds_read_b128  v[112:115], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_4]              "
+    "\n" _UK_MFMA_ "  %[v_acc_6], acc[56:57], v[72:73], %[v_acc_6] \n" _UK_MFMA_
+    "  %[v_acc_6], acc[58:59], v[74:75], %[v_acc_6] \n"
+    "  buffer_load_dwordx4  acc[180:183], %[v_os_b3], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_6], acc[60:61], v[76:77], %[v_acc_6] \n" _UK_MFMA_
+    "  %[v_acc_6], acc[62:63], v[78:79], %[v_acc_6] \n"
+    "  ds_read_b128  v[116:119], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_5]              "
+    "\n" _UK_MFMA_ "  %[v_acc_7], acc[48:49], v[80:81], %[v_acc_7] \n" _UK_MFMA_
+    "  %[v_acc_7], acc[50:51], v[82:83], %[v_acc_7] \n"
+    "  buffer_load_dwordx4  acc[184:187], %[v_os_b3], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_7], acc[52:53], v[84:85], %[v_acc_7] \n" _UK_MFMA_
+    "  %[v_acc_7], acc[54:55], v[86:87], %[v_acc_7] \n"
+    "  ds_read_b128  v[120:123], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_6]              "
+    "\n" _UK_MFMA_ "  %[v_acc_7], acc[56:57], v[88:89], %[v_acc_7] \n" _UK_MFMA_
+    "  %[v_acc_7], acc[58:59], v[90:91], %[v_acc_7] \n"
+    "  buffer_load_dwordx4  acc[188:191], %[v_os_b3], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_7], acc[60:61], v[92:93], %[v_acc_7] \n" _UK_MFMA_
+    "  %[v_acc_7], acc[62:63], v[94:95], %[v_acc_7] \n"
+    "  ds_read_b128  v[124:127], %[v_os_slda], offset:1*%[smem_sz] + %[sld_os_7]              \n"
+    "  s_waitcnt     vmcnt(32)                               \n" _UK_MFMA_
+    "  %[v_acc_8], acc[64:65], v[64:65], %[v_acc_8] \n" _UK_MFMA_
+    "  %[v_acc_8], acc[66:67], v[66:67], %[v_acc_8] \n"
+    "  buffer_load_dwordx4  acc[192:195], %[v_os_b4], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_8], acc[68:69], v[68:69], %[v_acc_8] \n" _UK_MFMA_
+    "  %[v_acc_8], acc[70:71], v[70:71], %[v_acc_8] \n" _UK_MFMA_
+    "  %[v_acc_8], acc[72:73], v[72:73], %[v_acc_8] \n" _UK_MFMA_
+    "  %[v_acc_8], acc[74:75], v[74:75], %[v_acc_8] \n"
+    "  buffer_load_dwordx4  acc[196:199], %[v_os_b4], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_8], acc[76:77], v[76:77], %[v_acc_8] \n" _UK_MFMA_
+    "  %[v_acc_8], acc[78:79], v[78:79], %[v_acc_8] \n" _UK_MFMA_
+    "  %[v_acc_9], acc[64:65], v[80:81], %[v_acc_9] \n" _UK_MFMA_
+    "  %[v_acc_9], acc[66:67], v[82:83], %[v_acc_9] \n"
+    "  buffer_load_dwordx4  acc[200:203], %[v_os_b4], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_9], acc[68:69], v[84:85], %[v_acc_9] \n" _UK_MFMA_
+    "  %[v_acc_9], acc[70:71], v[86:87], %[v_acc_9] \n" _UK_MFMA_
+    "  %[v_acc_9], acc[72:73], v[88:89], %[v_acc_9] \n" _UK_MFMA_
+    "  %[v_acc_9], acc[74:75], v[90:91], %[v_acc_9] \n"
+    "  buffer_load_dwordx4  acc[204:207], %[v_os_b4], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_9], acc[76:77], v[92:93], %[v_acc_9] \n" _UK_MFMA_
+    "  %[v_acc_9], acc[78:79], v[94:95], %[v_acc_9] \n" _UK_MFMA_
+    "  %[v_acc_10], acc[80:81], v[64:65], %[v_acc_10] \n" _UK_MFMA_
+    "  %[v_acc_10], acc[82:83], v[66:67], %[v_acc_10] \n"
+    "  buffer_load_dwordx4  acc[208:211], %[v_os_b5], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_10], acc[84:85], v[68:69], %[v_acc_10] \n" _UK_MFMA_
+    "  %[v_acc_10], acc[86:87], v[70:71], %[v_acc_10] \n" _UK_MFMA_
+    "  %[v_acc_10], acc[88:89], v[72:73], %[v_acc_10] \n" _UK_MFMA_
+    "  %[v_acc_10], acc[90:91], v[74:75], %[v_acc_10] \n"
+    "  buffer_load_dwordx4  acc[212:215], %[v_os_b5], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_10], acc[92:93], v[76:77], %[v_acc_10] \n" _UK_MFMA_
+    "  %[v_acc_10], acc[94:95], v[78:79], %[v_acc_10] \n" _UK_MFMA_
+    "  %[v_acc_11], acc[80:81], v[80:81], %[v_acc_11] \n" _UK_MFMA_
+    "  %[v_acc_11], acc[82:83], v[82:83], %[v_acc_11] \n"
+    "  buffer_load_dwordx4  acc[216:219], %[v_os_b5], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_11], acc[84:85], v[84:85], %[v_acc_11] \n" _UK_MFMA_
+    "  %[v_acc_11], acc[86:87], v[86:87], %[v_acc_11] \n" _UK_MFMA_
+    "  %[v_acc_11], acc[88:89], v[88:89], %[v_acc_11] \n" _UK_MFMA_
+    "  %[v_acc_11], acc[90:91], v[90:91], %[v_acc_11] \n"
+    "  buffer_load_dwordx4  acc[220:223], %[v_os_b5], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_11], acc[92:93], v[92:93], %[v_acc_11] \n" _UK_MFMA_
+    "  %[v_acc_11], acc[94:95], v[94:95], %[v_acc_11] \n"
+    "  s_waitcnt     vmcnt(32)                               \n" _UK_MFMA_
+    "  %[v_acc_12], acc[96:97], v[64:65], %[v_acc_12] \n" _UK_MFMA_
+    "  %[v_acc_12], acc[98:99], v[66:67], %[v_acc_12] \n"
+    "  buffer_load_dwordx4  acc[224:227], %[v_os_b6], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_12], acc[100:101], v[68:69], %[v_acc_12] \n" _UK_MFMA_
+    "  %[v_acc_12], acc[102:103], v[70:71], %[v_acc_12] \n" _UK_MFMA_
+    "  %[v_acc_12], acc[104:105], v[72:73], %[v_acc_12] \n" _UK_MFMA_
+    "  %[v_acc_12], acc[106:107], v[74:75], %[v_acc_12] \n"
+    "  buffer_load_dwordx4  acc[228:231], %[v_os_b6], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_12], acc[108:109], v[76:77], %[v_acc_12] \n" _UK_MFMA_
+    "  %[v_acc_12], acc[110:111], v[78:79], %[v_acc_12] \n" _UK_MFMA_
+    "  %[v_acc_13], acc[96:97], v[80:81], %[v_acc_13] \n" _UK_MFMA_
+    "  %[v_acc_13], acc[98:99], v[82:83], %[v_acc_13] \n"
+    "  buffer_load_dwordx4  acc[232:235], %[v_os_b6], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_13], acc[100:101], v[84:85], %[v_acc_13] \n" _UK_MFMA_
+    "  %[v_acc_13], acc[102:103], v[86:87], %[v_acc_13] \n" _UK_MFMA_
+    "  %[v_acc_13], acc[104:105], v[88:89], %[v_acc_13] \n" _UK_MFMA_
+    "  %[v_acc_13], acc[106:107], v[90:91], %[v_acc_13] \n"
+    "  buffer_load_dwordx4  acc[236:239], %[v_os_b6], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_13], acc[108:109], v[92:93], %[v_acc_13] \n" _UK_MFMA_
+    "  %[v_acc_13], acc[110:111], v[94:95], %[v_acc_13] \n" _UK_MFMA_
+    "  %[v_acc_14], acc[112:113], v[64:65], %[v_acc_14] \n" _UK_MFMA_
+    "  %[v_acc_14], acc[114:115], v[66:67], %[v_acc_14] \n"
+    "  buffer_load_dwordx4  acc[240:243], %[v_os_b7], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_14], acc[116:117], v[68:69], %[v_acc_14] \n" _UK_MFMA_
+    "  %[v_acc_14], acc[118:119], v[70:71], %[v_acc_14] \n" _UK_MFMA_
+    "  %[v_acc_14], acc[120:121], v[72:73], %[v_acc_14] \n" _UK_MFMA_
+    "  %[v_acc_14], acc[122:123], v[74:75], %[v_acc_14] \n"
+    "  buffer_load_dwordx4  acc[244:247], %[v_os_b7], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_14], acc[124:125], v[76:77], %[v_acc_14] \n" _UK_MFMA_
+    "  %[v_acc_14], acc[126:127], v[78:79], %[v_acc_14] \n" _UK_MFMA_
+    "  %[v_acc_15], acc[112:113], v[80:81], %[v_acc_15] \n" _UK_MFMA_
+    "  %[v_acc_15], acc[114:115], v[82:83], %[v_acc_15] \n"
+    "  buffer_load_dwordx4  acc[248:251], %[v_os_b7], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_15], acc[116:117], v[84:85], %[v_acc_15] \n" _UK_MFMA_
+    "  %[v_acc_15], acc[118:119], v[86:87], %[v_acc_15] \n" _UK_MFMA_
+    "  %[v_acc_15], acc[120:121], v[88:89], %[v_acc_15] \n" _UK_MFMA_
+    "  %[v_acc_15], acc[122:123], v[90:91], %[v_acc_15] \n"
+    "  buffer_load_dwordx4  acc[252:255], %[v_os_b7], s[20:23], 0 offen offset:3072\n" _UK_MFMA_
+    "  %[v_acc_15], acc[124:125], v[92:93], %[v_acc_15] \n" _UK_MFMA_
+    "  %[v_acc_15], acc[126:127], v[94:95], %[v_acc_15] \n"
+    "  s_sub_i32     %[s_loop_cnt], %[s_loop_cnt], 1                \n"
+    "  s_cmp_gt_i32  %[s_loop_cnt] 0                                \n"
+    "  s_cbranch_scc0 L_end%=                                       \n"
+    "  s_cmp_gt_i32  %[s_loop_cnt] 2             ; move a with cond \n"
+    "  s_cselect_b32 s86, %[s_tile_os_a], 0                          \n"
+    "  s_add_u32     s16, s86, s16                                  \n"
+    "  s_addc_u32    s17, 0, s17                                    \n"
+    "  s_cmp_gt_i32  %[s_loop_cnt] 1             ; move b with cond \n"
+    "  s_cselect_b32 s86, %[s_tile_os_b], 0                          \n"
+    "  s_add_u32     s20, s86, s20                                  \n"
+    "  s_addc_u32    s21, 0, s21                                    \n"
+    "  ;------------------------------------------                  \n"
+    "  s_waitcnt     vmcnt(24) & lgkmcnt(0)                  \n"
+    "  s_barrier                                             \n" _UK_MFMA_
+    "  %[v_acc_0], acc[128:129], v[96:97], %[v_acc_0] \n" _UK_MFMA_
+    "  %[v_acc_0], acc[130:131], v[98:99], %[v_acc_0] \n"
+    "  buffer_load_dwordx4  acc[0:3], %[v_os_b0], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_0], acc[132:133], v[100:101], %[v_acc_0] \n" _UK_MFMA_
+    "  %[v_acc_0], acc[134:135], v[102:103], %[v_acc_0] \n"
+    "  buffer_load_dword   %[v_os_a0], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_0], acc[136:137], v[104:105], %[v_acc_0] \n" _UK_MFMA_
+    "  %[v_acc_0], acc[138:139], v[106:107], %[v_acc_0] \n"
+    "  buffer_load_dwordx4  acc[4:7], %[v_os_b0], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_0], acc[140:141], v[108:109], %[v_acc_0] \n" _UK_MFMA_
+    "  %[v_acc_0], acc[142:143], v[110:111], %[v_acc_0] \n"
+    "  buffer_load_dword   %[v_os_a1], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_1], acc[128:129], v[112:113], %[v_acc_1] \n" _UK_MFMA_
+    "  %[v_acc_1], acc[130:131], v[114:115], %[v_acc_1] \n"
+    "  buffer_load_dwordx4  acc[8:11], %[v_os_b0], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_1], acc[132:133], v[116:117], %[v_acc_1] \n" _UK_MFMA_
+    "  %[v_acc_1], acc[134:135], v[118:119], %[v_acc_1] \n"
+    "  buffer_load_dword   %[v_os_a2], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_1], acc[136:137], v[120:121], %[v_acc_1] \n" _UK_MFMA_
+    "  %[v_acc_1], acc[138:139], v[122:123], %[v_acc_1] \n"
+    "  buffer_load_dwordx4  acc[12:15], %[v_os_b0], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_1], acc[140:141], v[124:125], %[v_acc_1] \n" _UK_MFMA_
+    "  %[v_acc_1], acc[142:143], v[126:127], %[v_acc_1] \n"
+    "  buffer_load_dword   %[v_os_a3], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_2], acc[144:145], v[96:97], %[v_acc_2] \n" _UK_MFMA_
+    "  %[v_acc_2], acc[146:147], v[98:99], %[v_acc_2] \n"
+    "  buffer_load_dwordx4  acc[16:19], %[v_os_b1], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_2], acc[148:149], v[100:101], %[v_acc_2] \n" _UK_MFMA_
+    "  %[v_acc_2], acc[150:151], v[102:103], %[v_acc_2] \n"
+    "  buffer_load_dword   %[v_os_a4], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_2], acc[152:153], v[104:105], %[v_acc_2] \n" _UK_MFMA_
+    "  %[v_acc_2], acc[154:155], v[106:107], %[v_acc_2] \n"
+    "  buffer_load_dwordx4  acc[20:23], %[v_os_b1], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_2], acc[156:157], v[108:109], %[v_acc_2] \n" _UK_MFMA_
+    "  %[v_acc_2], acc[158:159], v[110:111], %[v_acc_2] \n"
+    "  buffer_load_dword   %[v_os_a5], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_3], acc[144:145], v[112:113], %[v_acc_3] \n" _UK_MFMA_
+    "  %[v_acc_3], acc[146:147], v[114:115], %[v_acc_3] \n"
+    "  buffer_load_dwordx4  acc[24:27], %[v_os_b1], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_3], acc[148:149], v[116:117], %[v_acc_3] \n" _UK_MFMA_
+    "  %[v_acc_3], acc[150:151], v[118:119], %[v_acc_3] \n"
+    "  buffer_load_dword   %[v_os_a6], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, %[s_size_per_issue], m0                  \n" _UK_MFMA_
+    "  %[v_acc_3], acc[152:153], v[120:121], %[v_acc_3] \n" _UK_MFMA_
+    "  %[v_acc_3], acc[154:155], v[122:123], %[v_acc_3] \n"
+    "  buffer_load_dwordx4  acc[28:31], %[v_os_b1], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_3], acc[156:157], v[124:125], %[v_acc_3] \n" _UK_MFMA_
+    "  %[v_acc_3], acc[158:159], v[126:127], %[v_acc_3] \n"
+    "  buffer_load_dword   %[v_os_a7], s[16:19], 0 offen lds     \n"
+    "  s_add_u32     m0, 0, %[s_m0_init]                  \n"
+    "  s_waitcnt     vmcnt(32)                               \n" _UK_MFMA_
+    "  %[v_acc_4], acc[160:161], v[96:97], %[v_acc_4] \n" _UK_MFMA_
+    "  %[v_acc_4], acc[162:163], v[98:99], %[v_acc_4] \n"
+    "  buffer_load_dwordx4  acc[32:35], %[v_os_b2], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_4], acc[164:165], v[100:101], %[v_acc_4] \n" _UK_MFMA_
+    "  %[v_acc_4], acc[166:167], v[102:103], %[v_acc_4] \n"
+    "  ds_read_b128  v[64:67], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_0]  \n" _UK_MFMA_
+    "  %[v_acc_4], acc[168:169], v[104:105], %[v_acc_4] \n" _UK_MFMA_
+    "  %[v_acc_4], acc[170:171], v[106:107], %[v_acc_4] \n"
+    "  buffer_load_dwordx4  acc[36:39], %[v_os_b2], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_4], acc[172:173], v[108:109], %[v_acc_4] \n" _UK_MFMA_
+    "  %[v_acc_4], acc[174:175], v[110:111], %[v_acc_4] \n"
+    "  ds_read_b128  v[68:71], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_1]  \n" _UK_MFMA_
+    "  %[v_acc_5], acc[160:161], v[112:113], %[v_acc_5] \n" _UK_MFMA_
+    "  %[v_acc_5], acc[162:163], v[114:115], %[v_acc_5] \n"
+    "  buffer_load_dwordx4  acc[40:43], %[v_os_b2], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_5], acc[164:165], v[116:117], %[v_acc_5] \n" _UK_MFMA_
+    "  %[v_acc_5], acc[166:167], v[118:119], %[v_acc_5] \n"
+    "  ds_read_b128  v[72:75], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_2]                 "
+    "\n" _UK_MFMA_ "  %[v_acc_5], acc[168:169], v[120:121], %[v_acc_5] \n" _UK_MFMA_
+    "  %[v_acc_5], acc[170:171], v[122:123], %[v_acc_5] \n"
+    "  buffer_load_dwordx4  acc[44:47], %[v_os_b2], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_5], acc[172:173], v[124:125], %[v_acc_5] \n" _UK_MFMA_
+    "  %[v_acc_5], acc[174:175], v[126:127], %[v_acc_5] \n"
+    "  ds_read_b128  v[76:79], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_3]                "
+    "\n" _UK_MFMA_ "  %[v_acc_6], acc[176:177], v[96:97], %[v_acc_6] \n" _UK_MFMA_
+    "  %[v_acc_6], acc[178:179], v[98:99], %[v_acc_6] \n"
+    "  buffer_load_dwordx4  acc[48:51], %[v_os_b3], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_6], acc[180:181], v[100:101], %[v_acc_6] \n" _UK_MFMA_
+    "  %[v_acc_6], acc[182:183], v[102:103], %[v_acc_6] \n"
+    "  ds_read_b128  v[80:83], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_4]               "
+    "\n" _UK_MFMA_ "  %[v_acc_6], acc[184:185], v[104:105], %[v_acc_6] \n" _UK_MFMA_
+    "  %[v_acc_6], acc[186:187], v[106:107], %[v_acc_6] \n"
+    "  buffer_load_dwordx4  acc[52:55], %[v_os_b3], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_6], acc[188:189], v[108:109], %[v_acc_6] \n" _UK_MFMA_
+    "  %[v_acc_6], acc[190:191], v[110:111], %[v_acc_6] \n"
+    "  ds_read_b128  v[84:87], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_5]            "
+    "\n" _UK_MFMA_ "  %[v_acc_7], acc[176:177], v[112:113], %[v_acc_7] \n" _UK_MFMA_
+    "  %[v_acc_7], acc[178:179], v[114:115], %[v_acc_7] \n"
+    "  buffer_load_dwordx4  acc[56:59], %[v_os_b3], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_7], acc[180:181], v[116:117], %[v_acc_7] \n" _UK_MFMA_
+    "  %[v_acc_7], acc[182:183], v[118:119], %[v_acc_7] \n"
+    "  ds_read_b128  v[88:91], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_6]                "
+    "\n" _UK_MFMA_ "  %[v_acc_7], acc[184:185], v[120:121], %[v_acc_7] \n" _UK_MFMA_
+    "  %[v_acc_7], acc[186:187], v[122:123], %[v_acc_7] \n"
+    "  buffer_load_dwordx4  acc[60:63], %[v_os_b3], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_7], acc[188:189], v[124:125], %[v_acc_7] \n" _UK_MFMA_
+    "  %[v_acc_7], acc[190:191], v[126:127], %[v_acc_7] \n"
+    "  ds_read_b128  v[92:95], %[v_os_slda] offset:0*%[smem_sz] + %[sld_os_7]           \n"
+    "  s_waitcnt     vmcnt(32)                               \n" _UK_MFMA_
+    "  %[v_acc_8], acc[192:193], v[96:97], %[v_acc_8] \n" _UK_MFMA_
+    "  %[v_acc_8], acc[194:195], v[98:99], %[v_acc_8] \n"
+    "  buffer_load_dwordx4  acc[64:67], %[v_os_b4], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_8], acc[196:197], v[100:101], %[v_acc_8] \n" _UK_MFMA_
+    "  %[v_acc_8], acc[198:199], v[102:103], %[v_acc_8] \n" _UK_MFMA_
+    "  %[v_acc_8], acc[200:201], v[104:105], %[v_acc_8] \n" _UK_MFMA_
+    "  %[v_acc_8], acc[202:203], v[106:107], %[v_acc_8] \n"
+    "  buffer_load_dwordx4  acc[68:71], %[v_os_b4], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_8], acc[204:205], v[108:109], %[v_acc_8] \n" _UK_MFMA_
+    "  %[v_acc_8], acc[206:207], v[110:111], %[v_acc_8] \n" _UK_MFMA_
+    "  %[v_acc_9], acc[192:193], v[112:113], %[v_acc_9] \n" _UK_MFMA_
+    "  %[v_acc_9], acc[194:195], v[114:115], %[v_acc_9] \n"
+    "  buffer_load_dwordx4  acc[72:75], %[v_os_b4], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_9], acc[196:197], v[116:117], %[v_acc_9] \n" _UK_MFMA_
+    "  %[v_acc_9], acc[198:199], v[118:119], %[v_acc_9] \n" _UK_MFMA_
+    "  %[v_acc_9], acc[200:201], v[120:121], %[v_acc_9] \n" _UK_MFMA_
+    "  %[v_acc_9], acc[202:203], v[122:123], %[v_acc_9] \n"
+    "  buffer_load_dwordx4  acc[76:79], %[v_os_b4], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_9], acc[204:205], v[124:125], %[v_acc_9] \n" _UK_MFMA_
+    "  %[v_acc_9], acc[206:207], v[126:127], %[v_acc_9] \n" _UK_MFMA_
+    "  %[v_acc_10], acc[208:209], v[96:97], %[v_acc_10] \n" _UK_MFMA_
+    "  %[v_acc_10], acc[210:211], v[98:99], %[v_acc_10] \n"
+    "  buffer_load_dwordx4  acc[80:83], %[v_os_b5], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_10], acc[212:213], v[100:101], %[v_acc_10] \n" _UK_MFMA_
+    "  %[v_acc_10], acc[214:215], v[102:103], %[v_acc_10] \n" _UK_MFMA_
+    "  %[v_acc_10], acc[216:217], v[104:105], %[v_acc_10] \n" _UK_MFMA_
+    "  %[v_acc_10], acc[218:219], v[106:107], %[v_acc_10] \n"
+    "  buffer_load_dwordx4  acc[84:87], %[v_os_b5], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_10], acc[220:221], v[108:109], %[v_acc_10] \n" _UK_MFMA_
+    "  %[v_acc_10], acc[222:223], v[110:111], %[v_acc_10] \n" _UK_MFMA_
+    "  %[v_acc_11], acc[208:209], v[112:113], %[v_acc_11] \n" _UK_MFMA_
+    "  %[v_acc_11], acc[210:211], v[114:115], %[v_acc_11] \n"
+    "  buffer_load_dwordx4  acc[88:91], %[v_os_b5], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_11], acc[212:213], v[116:117], %[v_acc_11] \n" _UK_MFMA_
+    "  %[v_acc_11], acc[214:215], v[118:119], %[v_acc_11] \n" _UK_MFMA_
+    "  %[v_acc_11], acc[216:217], v[120:121], %[v_acc_11] \n" _UK_MFMA_
+    "  %[v_acc_11], acc[218:219], v[122:123], %[v_acc_11] \n"
+    "  buffer_load_dwordx4  acc[92:95], %[v_os_b5], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_11], acc[220:221], v[124:125], %[v_acc_11] \n" _UK_MFMA_
+    "  %[v_acc_11], acc[222:223], v[126:127], %[v_acc_11] \n"
+    "  s_waitcnt     vmcnt(32)                               \n" _UK_MFMA_
+    "  %[v_acc_12], acc[224:225], v[96:97], %[v_acc_12] \n" _UK_MFMA_
+    "  %[v_acc_12], acc[226:227], v[98:99], %[v_acc_12] \n"
+    "  buffer_load_dwordx4  acc[96:99], %[v_os_b6], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_12], acc[228:229], v[100:101], %[v_acc_12] \n" _UK_MFMA_
+    "  %[v_acc_12], acc[230:231], v[102:103], %[v_acc_12] \n" _UK_MFMA_
+    "  %[v_acc_12], acc[232:233], v[104:105], %[v_acc_12] \n" _UK_MFMA_
+    "  %[v_acc_12], acc[234:235], v[106:107], %[v_acc_12] \n"
+    "  buffer_load_dwordx4  acc[100:103], %[v_os_b6], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_12], acc[236:237], v[108:109], %[v_acc_12] \n" _UK_MFMA_
+    "  %[v_acc_12], acc[238:239], v[110:111], %[v_acc_12] \n" _UK_MFMA_
+    "  %[v_acc_13], acc[224:225], v[112:113], %[v_acc_13] \n" _UK_MFMA_
+    "  %[v_acc_13], acc[226:227], v[114:115], %[v_acc_13] \n"
+    "  buffer_load_dwordx4  acc[104:107], %[v_os_b6], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_13], acc[228:229], v[116:117], %[v_acc_13] \n" _UK_MFMA_
+    "  %[v_acc_13], acc[230:231], v[118:119], %[v_acc_13] \n" _UK_MFMA_
+    "  %[v_acc_13], acc[232:233], v[120:121], %[v_acc_13] \n" _UK_MFMA_
+    "  %[v_acc_13], acc[234:235], v[122:123], %[v_acc_13] \n"
+    "  buffer_load_dwordx4  acc[108:111], %[v_os_b6], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_13], acc[236:237], v[124:125], %[v_acc_13] \n" _UK_MFMA_
+    "  %[v_acc_13], acc[238:239], v[126:127], %[v_acc_13] \n" _UK_MFMA_
+    "  %[v_acc_14], acc[240:241], v[96:97], %[v_acc_14] \n" _UK_MFMA_
+    "  %[v_acc_14], acc[242:243], v[98:99], %[v_acc_14] \n"
+    "  buffer_load_dwordx4  acc[112:115], %[v_os_b7], s[20:23], 0 offen \n" _UK_MFMA_
+    "  %[v_acc_14], acc[244:245], v[100:101], %[v_acc_14] \n" _UK_MFMA_
+    "  %[v_acc_14], acc[246:247], v[102:103], %[v_acc_14] \n" _UK_MFMA_
+    "  %[v_acc_14], acc[248:249], v[104:105], %[v_acc_14] \n" _UK_MFMA_
+    "  %[v_acc_14], acc[250:251], v[106:107], %[v_acc_14] \n"
+    "  buffer_load_dwordx4  acc[116:119], %[v_os_b7], s[20:23], 0 offen offset:1024 \n" _UK_MFMA_
+    "  %[v_acc_14], acc[252:253], v[108:109], %[v_acc_14] \n" _UK_MFMA_
+    "  %[v_acc_14], acc[254:255], v[110:111], %[v_acc_14] \n" _UK_MFMA_
+    "  %[v_acc_15], acc[240:241], v[112:113], %[v_acc_15] \n" _UK_MFMA_
+    "  %[v_acc_15], acc[242:243], v[114:115], %[v_acc_15] \n"
+    "  buffer_load_dwordx4  acc[120:123], %[v_os_b7], s[20:23], 0 offen offset:2048 \n" _UK_MFMA_
+    "  %[v_acc_15], acc[244:245], v[116:117], %[v_acc_15] \n" _UK_MFMA_
+    "  %[v_acc_15], acc[246:247], v[118:119], %[v_acc_15] \n" _UK_MFMA_
+    "  %[v_acc_15], acc[248:249], v[120:121], %[v_acc_15] \n" _UK_MFMA_
+    "  %[v_acc_15], acc[250:251], v[122:123], %[v_acc_15] \n"
+    "  buffer_load_dwordx4  acc[124:127], %[v_os_b7], s[20:23], 0 offen offset:3072 \n" _UK_MFMA_
+    "  %[v_acc_15], acc[252:253], v[124:125], %[v_acc_15] \n" _UK_MFMA_
+    "  %[v_acc_15], acc[254:255], v[126:127], %[v_acc_15] \n"
+    "  s_sub_i32     %[s_loop_cnt], %[s_loop_cnt], 1                \n"
+    "  s_cmp_gt_i32  %[s_loop_cnt] 0                                \n"
+    "  s_cbranch_scc0 L_end%=                                       \n"
+    "  s_cmp_gt_i32  %[s_loop_cnt] 2             ; move a with cond \n"
+    "  s_cselect_b32 s86, %[s_tile_os_a], 0                          \n"
+    "  s_add_u32     s16, s86, s16                                  \n"
+    "  s_addc_u32    s17, 0, s17                                    \n"
+    "  s_cmp_gt_i32  %[s_loop_cnt] 1             ; move b with cond \n"
+    "  s_cselect_b32 s86, %[s_tile_os_b], 0                          \n"
+    "  s_add_u32     s20, s86, s20                                  \n"
+    "  s_addc_u32    s21, 0, s21                                    \n"
+    "  s_branch     L_start%=                                       \n"
+    "L_end%=:                                                       \n"
+    "  s_nop 2                                                      \n"
 
 #undef _UK_MFMA_

include/ck_tile/ops/fused_moe/pipeline/fused_moegemm_pipeline_flatmm_policy.hpp

@@ -810,21 +810,46 @@ struct FusedMoeGemmPipelineFlatmmPolicy
     CK_TILE_HOST_DEVICE static constexpr auto GetUK_1()
     {
         using S_ = typename Problem::BlockShape;
+        using T_ = typename Problem::Traits;
         if constexpr(std::is_same_v<typename Problem::YDataType, ck_tile::bf16_t> &&
                      std::is_same_v<typename Problem::DDataType, ck_tile::bf16_t> &&
                      std::is_same_v<typename Problem::TopkWeightDataType, float> &&
                      S_::Block_M1 == 32 && S_::Block_N1 == 128 && S_::Block_K1 == 512 &&
-                     S_::Warp_M0 == 16 && S_::Warp_N0 == 16 && S_::Warp_K0 == 32)
+                     S_::Warp_M0 == 16 && S_::Warp_N0 == 16 && S_::Warp_K0 == 32 &&
+                     T_::PipeInterleave == false)
         {
             return FlatmmSn_32x128x512_1x4x1_16x16x32_BF16{};
+            // return FlatmmSn_32x128x512_1x4x1_16x16x32_BF16_itl{};
         }
         else if constexpr(std::is_same_v<typename Problem::YDataType, ck_tile::fp16_t> &&
                           std::is_same_v<typename Problem::DDataType, ck_tile::fp16_t> &&
                           std::is_same_v<typename Problem::TopkWeightDataType, float> &&
                           S_::Block_M1 == 32 && S_::Block_N1 == 128 && S_::Block_K1 == 512 &&
-                          S_::Warp_M0 == 16 && S_::Warp_N0 == 16 && S_::Warp_K0 == 32)
+                          S_::Warp_M0 == 16 && S_::Warp_N0 == 16 && S_::Warp_K0 == 32 &&
+                          T_::PipeInterleave == false)
         {
             return FlatmmSn_32x128x512_1x4x1_16x16x32_FP16{};
+            // return FlatmmSn_32x128x512_1x4x1_16x16x32_FP16_itl{};
+        }
+        else if constexpr(std::is_same_v<typename Problem::YDataType, ck_tile::bf16_t> &&
+                          std::is_same_v<typename Problem::DDataType, ck_tile::bf16_t> &&
+                          std::is_same_v<typename Problem::TopkWeightDataType, float> &&
+                          S_::Block_M1 == 32 && S_::Block_N1 == 128 && S_::Block_K1 == 512 &&
+                          S_::Warp_M0 == 16 && S_::Warp_N0 == 16 && S_::Warp_K0 == 32 &&
+                          T_::PipeInterleave == true)
+        {
+            // return FlatmmSn_32x128x512_1x4x1_16x16x32_FP16{};
+            return FlatmmSn_32x128x512_1x4x1_16x16x32_BF16_itl{};
+        }
+        else if constexpr(std::is_same_v<typename Problem::YDataType, ck_tile::fp16_t> &&
+                          std::is_same_v<typename Problem::DDataType, ck_tile::fp16_t> &&
+                          std::is_same_v<typename Problem::TopkWeightDataType, float> &&
+                          S_::Block_M1 == 32 && S_::Block_N1 == 128 && S_::Block_K1 == 512 &&
+                          S_::Warp_M0 == 16 && S_::Warp_N0 == 16 && S_::Warp_K0 == 32 &&
+                          T_::PipeInterleave == true)
+        {
+            // return FlatmmSn_32x128x512_1x4x1_16x16x32_FP16{};
+            return FlatmmSn_32x128x512_1x4x1_16x16x32_FP16_itl{};
         }
     }
 };

include/ck_tile/ops/fused_moe/pipeline/fused_moegemm_traits.hpp

@@ -22,7 +22,8 @@ template <bool IsGateOnly_,
           FusedMoeGemmWeightPermuteEnum PermuteEnum_ =
               FusedMoeGemmWeightPermuteEnum::b_nr_kr_waveflatten,
           bool PadHiddenSize_       = false,
-          bool PadIntermediateSize_ = false>
+          bool PadIntermediateSize_ = false,
+          bool PipeInterleave_      = true>
 struct FusedMoeGemmTraits
 {
     // Gate+Up or Gate only
@@ -32,6 +33,7 @@ struct FusedMoeGemmTraits
     static constexpr FusedMoeGemmWeightPermuteEnum PermuteEnum = PermuteEnum_;
     static constexpr bool PadHiddenSize                        = PadHiddenSize_;
     static constexpr bool PadIntermediateSize                  = PadIntermediateSize_;
+    static constexpr bool PipeInterleave                       = PipeInterleave_;
 };
 
 // Note: this need to be a bit mask

include/ck_tile/ops/gemm.hpp

@@ -23,10 +23,10 @@
 #include "ck_tile/ops/gemm/block/block_gemm_asmem_bsmem_creg_v1_default_policy.hpp"
 #include "ck_tile/ops/gemm/block/block_gemm_problem.hpp"
 #include "ck_tile/ops/gemm/block/block_universal_gemm_as_bs_cr.hpp"
+#include "ck_tile/ops/gemm/kernel/batched_gemm_kernel.hpp"
 #include "ck_tile/ops/gemm/kernel/gemm_kernel.hpp"
 #include "ck_tile/ops/gemm/kernel/gemm_tile_partitioner.hpp"
 #include "ck_tile/ops/gemm/kernel/grouped_gemm_kernel.hpp"
-#include "ck_tile/ops/gemm/kernel/batched_gemm_kernel.hpp"
 #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_base.hpp"
 #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v3.hpp"
 #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_mem.hpp"