Merge branch 'main' into inference

8f7c0d11 · mshoeybi · a5bfc296 · 5478d67e · 8f7c0d11
Commit 8f7c0d11 authored Oct 12, 2021 by mshoeybi
Hide whitespace changes
Inline Side-by-side

Showing with 2 additions and 0 deletions

megatron/fused_kernels/layer_norm_cuda_kernel.cu megatron/fused_kernels/layer_norm_cuda_kernel.cu +2 -0

No files found.
--- a/megatron/fused_kernels/layer_norm_cuda_kernel.cu
+++ b/megatron/fused_kernels/layer_norm_cuda_kernel.cu
@@ -645,6 +645,8 @@ void cuComputeGradInput(
        k_grad_input[l] = static_cast<T>(f_grad_input);
      }
    }
+    // prevent race where buf is written again before reads are done
+    __syncthreads();
  }
 }