Merge branch 'deepspeed-branch' into 'main'

Deepspeed branch See merge request dcutoolkit/deeplearing/dlexamples_new!22

Merge branch 'deepspeed-branch' into 'main'
Deepspeed branch See merge request dcutoolkit/deeplearing/dlexamples_new!22
5394b117 · sunxx1 · 491af051 · 316d3f90 · 5394b117 · 5394b117
Commit 5394b117 authored Jul 14, 2022 by sunxx1
20 changed files
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/model/language_model.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/model/language_model.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/model/multiple_choice.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/model/multiple_choice.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/model/realm_model.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/model/realm_model.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/model/transformer.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/model/transformer.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/model/utils.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/model/utils.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/module.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/module.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/__init__.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/__init__.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/cross_entropy.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/cross_entropy.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/data.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/data.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/grads.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/grads.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/initialize.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/initialize.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/layers.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/layers.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/mappings.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/mappings.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/random.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/random.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/tests/__init__.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/tests/__init__.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/tests/commons.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/tests/commons.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/tests/test_cross_entropy.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/tests/test_cross_entropy.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/tests/test_data.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/tests/test_data.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/tests/test_initialize.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/tests/test_initialize.py
--- a/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/tests/test_layers.py
+++ b/Deepspeed/Megatron-LM-v1.1.5-3D_parallelism/megatron/mpu/tests/test_layers.py