megatron-lm0.3.2 based on dtk-22.10

1016e98a · zhuww · 6c3f6c7b · 1016e98a · 1016e98a · 1016e98a
Commit 1016e98a authored Feb 18, 2023 by zhuww
20 changed files
--- a/megatron/mpu/data.py
+++ b/megatron/mpu/data.py
--- a/megatron/mpu/initialize.py
+++ b/megatron/mpu/initialize.py
--- a/megatron/mpu/layers.py
+++ b/megatron/mpu/layers.py
--- a/megatron/mpu/mappings.py
+++ b/megatron/mpu/mappings.py
--- a/megatron/mpu/random.py
+++ b/megatron/mpu/random.py
--- a/megatron/mpu/tests/__init__.py
+++ b/megatron/mpu/tests/__init__.py
--- a/megatron/mpu/tests/commons.py
+++ b/megatron/mpu/tests/commons.py
--- a/megatron/mpu/tests/test_cross_entropy.py
+++ b/megatron/mpu/tests/test_cross_entropy.py
--- a/megatron/mpu/tests/test_data.py
+++ b/megatron/mpu/tests/test_data.py
--- a/megatron/mpu/tests/test_initialize.py
+++ b/megatron/mpu/tests/test_initialize.py
--- a/megatron/mpu/tests/test_layers.py
+++ b/megatron/mpu/tests/test_layers.py
--- a/megatron/mpu/tests/test_random.py
+++ b/megatron/mpu/tests/test_random.py
--- a/megatron/mpu/utils.py
+++ b/megatron/mpu/utils.py
--- a/megatron/optimizer/__init__.py
+++ b/megatron/optimizer/__init__.py
--- a/megatron/optimizer/clip_grads.py
+++ b/megatron/optimizer/clip_grads.py
--- a/megatron/optimizer/grad_scaler.py
+++ b/megatron/optimizer/grad_scaler.py
--- a/megatron/optimizer/optimizer.py
+++ b/megatron/optimizer/optimizer.py
--- a/megatron/optimizer_param_scheduler.py
+++ b/megatron/optimizer_param_scheduler.py
--- a/megatron/p2p_communication.py
+++ b/megatron/p2p_communication.py
--- a/megatron/schedules.py
+++ b/megatron/schedules.py