Merge branch 'citest' into 'main'

Testing infrastructure for Megatron core

See merge request ADLR/megatron-lm!514

tests/functional_tests/test_scripts/gpt3/sbatch_gpt3_distributed_resume_checkpoint_test.sh

+#!/bin/bash
+
+# Parameters
+#SBATCH --account=adlr
+#SBATCH --job-name=adlr-ci:megatron-job
+#SBATCH --nodes=1
+#SBATCH --partition=luna
+
+DATA_PATH=/workspace/data/gpt3_data/my-gpt3_00_text_document
+CHECKPOINT_PATH=/workspace/checkpoints
+TENSORBOARD_DIR=/workspace/logs
+
+srun --output $BASE_DIR/results/slurm-%j.out --error $BASE_DIR/results/slurm-%j.out --container-image gitlab-master.nvidia.com/dl/dgx/pytorch:21.12-py3-devel --container-mounts $BASE_DIR/logs:/workspace/logs,$BASE_DIR/checkpoints:/workspace/checkpoints,$BUILD_DIR:/workspace/megatron-lm,$DATA_DIR:/workspace/data --no-container-mount-home bash -c "
+  ls 
+  cd /workspace/megatron-lm
+  ./tests/functional_tests/test_scripts/gpt3/pretrain_gpt3_distributed_resume_checkpoint_test.sh $DATA_PATH $CHECKPOINT_PATH $TENSORBOARD_DIR $TP_SIZE $PP_SIZE $NUM_NODES"
\ No newline at end of file

tests/functional_tests/test_scripts/gpt3/sbatch_gpt3_distributed_test.sh

+#!/bin/bash
+
+# Parameters
+#SBATCH --account=adlr
+#SBATCH --job-name=adlr-ci:megatron-job
+#SBATCH --nodes=1
+#SBATCH --partition=luna
+
+DATA_PATH=/workspace/data/gpt3_data/my-gpt3_00_text_document
+CHECKPOINT_PATH=/workspace/checkpoints
+TENSORBOARD_DIR=/workspace/logs
+
+srun --output $BASE_DIR/results/slurm-%j.out --error $BASE_DIR/results/slurm-%j.out --container-image gitlab-master.nvidia.com/dl/dgx/pytorch:21.12-py3-devel --container-mounts $BASE_DIR/logs:/workspace/logs,$BASE_DIR/checkpoints:/workspace/checkpoints,$BUILD_DIR:/workspace/megatron-lm,$DATA_DIR:/workspace/data --no-container-mount-home bash -c "
+  ls 
+  cd /workspace/megatron-lm
+  ./tests/functional_tests/test_scripts/gpt3/pretrain_gpt3_distributed_test.sh $DATA_PATH $CHECKPOINT_PATH $TENSORBOARD_DIR $TP_SIZE $PP_SIZE $NUM_NODES $MAX_STEPS"
\ No newline at end of file

tests/tensor_parallel/test_cross_entropy.py → tests/unit_tests/tensor_parallel/test_cross_entropy.py

 from megatron.core.tensor_parallel.cross_entropy import vocab_parallel_cross_entropy
 import torch
-from tests.test_utilities import Utils
+from tests.unit_tests.test_utilities import Utils
 import numpy as np
 
 def test_vocab_parallel_cross_entropy():

tests/tensor_parallel/test_data.py → tests/unit_tests/tensor_parallel/test_data.py

 from megatron.core.tensor_parallel.data import broadcast_data
 import torch
-from tests.test_utilities import Utils
+from tests.unit_tests.test_utilities import Utils
 
 def test_broadcast_data():
     Utils.initialize_model_parallel(2,4)

tests/tensor_parallel/test_mappings.py → tests/unit_tests/tensor_parallel/test_mappings.py

 from megatron.core.tensor_parallel import mappings
-from tests.test_utilities import Utils
+from tests.unit_tests.test_utilities import Utils
 import torch
 
 def test_CopyToModelParallelRegion():

tests/tensor_parallel/test_random.py → tests/unit_tests/tensor_parallel/test_random.py

@@ -2,7 +2,7 @@ from megatron.core.tensor_parallel.random import CudaRNGStatesTracker
 from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed
 from megatron.core.tensor_parallel.random import _CUDA_RNG_STATE_TRACKER
 from megatron.core.tensor_parallel.random import checkpoint
-from tests.test_utilities import Utils
+from tests.unit_tests.test_utilities import Utils
 import pytest
 import torch
 

tests/tensor_parallel/test_tensor_parallel_utils.py → tests/unit_tests/tensor_parallel/test_tensor_parallel_utils.py

 import torch
 import megatron.core.tensor_parallel.utils as util
 import megatron.core.parallel_state as ps
-from tests.test_utilities import Utils
+from tests.unit_tests.test_utilities import Utils
 
 rank = Utils.rank
 

tests/test_parallel_state.py → tests/unit_tests/test_parallel_state.py

 import torch
 import megatron.core.parallel_state as ps
 import pytest
-from tests.test_utilities import Utils
+from tests.unit_tests.test_utilities import Utils
 import os 
 
 rank = Utils.rank