megatron-lm0.3.2 based on dtk-22.10

1016e98a · zhuww · 6c3f6c7b · 1016e98a · 1016e98a · 1016e98a
Commit 1016e98a authored Feb 18, 2023 by zhuww
20 changed files
--- a/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h
+++ b/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h
--- a/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu
+++ b/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu
--- a/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp
+++ b/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp
--- a/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip
+++ b/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip
--- a/megatron/fused_kernels/tests/__init__.py
+++ b/megatron/fused_kernels/tests/__init__.py
--- a/megatron/fused_kernels/tests/test_fused_kernels.py
+++ b/megatron/fused_kernels/tests/test_fused_kernels.py
--- a/megatron/fused_kernels/type_shim.h
+++ b/megatron/fused_kernels/type_shim.h
--- a/megatron/global_vars.py
+++ b/megatron/global_vars.py
--- a/megatron/indexer.py
+++ b/megatron/indexer.py
--- a/megatron/initialize.py
+++ b/megatron/initialize.py
--- a/megatron/memory.py
+++ b/megatron/memory.py
--- a/megatron/microbatches.py
+++ b/megatron/microbatches.py
--- a/megatron/model/__init__.py
+++ b/megatron/model/__init__.py
--- a/megatron/model/bert_model.py
+++ b/megatron/model/bert_model.py
--- a/megatron/model/biencoder_model.py
+++ b/megatron/model/biencoder_model.py
--- a/megatron/model/classification.py
+++ b/megatron/model/classification.py
--- a/megatron/model/distributed.py
+++ b/megatron/model/distributed.py
--- a/megatron/model/enums.py
+++ b/megatron/model/enums.py
--- a/megatron/model/fused_bias_gelu.py
+++ b/megatron/model/fused_bias_gelu.py
--- a/megatron/model/fused_layer_norm.py
+++ b/megatron/model/fused_layer_norm.py