add block_sync_lds in deterministic mode

include/ck/tensor_operation/gpu/grid/gridwise_batched_multihead_attention_backward_xdl_cshuffle_pt1.hpp

@@ -1789,6 +1789,11 @@ struct GridwiseBatchedMultiheadAttentionBackward_Xdl_CShuffle_V1
         auto y_dot_ygrad_thread_buf = make_static_buffer<AddressSpaceEnum::Vgpr, FloatGemmAcc>(
             y_dot_ygrad_thread_desc_mblock_mrepeat_mwave_mperxdl.GetElementSpaceSize());
 
+        if constexpr(Deterministic)
+        {
+            block_sync_lds();
+        }
+
         // load ygrad
         gemm_tile_ygrad_blockwise_copy.Run(ygrad_grid_desc_o0_m_o1,
                                            ygrad_grid_buf,

include/ck/tensor_operation/gpu/grid/gridwise_batched_multihead_attention_backward_xdl_cshuffle_pt2.hpp

@@ -1698,6 +1698,11 @@ struct GridwiseBatchedMultiheadAttentionBackward_Xdl_CShuffle_V2
         auto y_dot_ygrad_thread_buf = make_static_buffer<AddressSpaceEnum::Vgpr, FloatGemmAcc>(
             y_dot_ygrad_thread_desc_mblock_mrepeat_mwave_mperxdl.GetElementSpaceSize());
 
+        if constexpr(Deterministic)
+        {
+            block_sync_lds();
+        }
+
         //
         // calculate Y dot dY
         //

include/ck/tensor_operation/gpu/grid/gridwise_batched_multihead_attention_forward_xdl_cshuffle.hpp

@@ -913,6 +913,11 @@ struct GridwiseBatchedMultiheadAttentionForward_Xdl_CShuffle
                                    0),
                   tensor_operation::element_wise::PassThrough{}};
 
+        if constexpr(Deterministic)
+        {
+            block_sync_lds();
+        }
+        
         do
         {
             auto n_block_data_idx_on_grid =