reverting test matrix to debug node (#215)

azure-pipelines.yml

 
 jobs:
-- job: DeepSpeed_Tests
+- job: Default
   timeoutInMinutes: 360
   pool:
-    name: 'DS_testing'
+    name: 'GPU_testing'
 
   strategy:
     matrix:
-      PyTorch12:
+      Python36:
         python.version: '3.6'
-        cuda.version: '10.0'
-        pytorch.version: '1.2'
-        runmodeltests: false
-      PyTorch15:
-        python.version: '3.7'
-        cuda.version: '10.1'
-        pytorch.version: '1.5'
-        runmodeltests: true
+      #Python35:
+      #  python.version: '3.5'
+      #Python37:
+      #  python.version: '3.7'
+      #Python38:
+      #  python.version: '3.8'
 
-  variables:
-    conda_root: '/home/deepspeed/miniconda3'
-    conda_env: 'ds_test_py$(python.version)_cuda$(cuda.version)_pytorch$(pytorch.version)'
-    conda_env_path: '$(conda_root)/envs/$(conda_env)'
 
   steps:
-    # Unfortunately nvidia's nvcc_linux-64=<version> seems to install 10.1 regardless?
-    # Most of this complexity is a workaround to get the compiler toolchain to match the
-    # cudatoolkit runtime
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '$(python.version)'
+      addToPath: true
+      architecture: 'x64'
+    displayName: 'Use Python $(python.version)'
+
   - script: |
-      conda create --force --yes -n $(conda_env) python=$(python.version) cudatoolkit=$(cuda.version)
-      source $(conda_root)/bin/activate $(conda_env_path)
-      conda install --yes -c pytorch pytorch=$(pytorch.version) cudatoolkit=$(cuda.version)
-      conda install --yes conda
-      conda install --yes gxx_linux-64
-      conda install --yes -c conda-forge cudatoolkit-dev=$(cuda.version)
-    displayName: 'Setup environment python=$(python.version) pytorch=$(pytorch.version) cuda=$(cuda.version)'
+      python -m pip install --upgrade pip
+      pip install --user -r requirements.txt
+      ./install.sh --pip_sudo
+    displayName: 'Install dependencies'
 
   - script: |
-      source $(conda_root)/bin/activate $(conda_env_path)
-      python --version
-      which nvcc
-      nvcc --version
-      python -c "import torch; print('torch:', torch.__version__)"
-    displayName: 'Show environment'
+      pre-commit run --all-files
+    displayName: 'Formatting checks'
 
   - script: |
-      source $(conda_root)/bin/activate $(conda_env_path)
-      rm -rf third_party/apex/build/
-      rm -rf third_party/apex/dist/
-      rm -rf build/
-      rm -rf dist/
-      ./install.sh
-    displayName: 'Install DeepSpeed'
+      pip install --user pylint
+      pylint --exit-zero deepspeed/
+    displayName: 'Code linter'
 
   - script: |
-      source $(conda_root)/bin/activate $(conda_env_path)
       pytest --forked --verbose tests/unit/
     displayName: 'Unit tests'
 
   - script: |
-      source $(conda_root)/bin/activate $(conda_env_path)
       ln -s /data/Megatron-LM/data DeepSpeedExamples/Megatron-LM/
-      pip install -r DeepSpeedExamples/Megatron-LM/requirements.txt
+      pip install --user -r DeepSpeedExamples/Megatron-LM/requirements.txt
       cd tests/model/
-      rm -rf BingBertSquad/baseline
-      rm -rf Megatron_GPT2/baseline
       pytest -s run_sanity_check.py
-    condition: eq(variables['runmodeltests'], true)
     displayName: 'Model tests'
 
    #BingBertSquad logs
@@ -75,31 +57,35 @@ jobs:
       targetPath: '$(Build.SourcesDirectory)/tests/model/BingBertSquad/test/'
       artifactName: BingBertSquad_logs
     displayName: 'BingBertSquad log uploads'
-    condition: eq(variables['runmodeltests'], true)
+    condition: always()
 
+  # Megatron test logs
+  #- task: PublishPipelineArtifact@1
+  #  inputs:
+  #    targetPath: '$(Build.SourcesDirectory)/tests/model/Megatron_GPT2/test/'
+  #    artifactName: Megatron_GPT2_logs
+  #  displayName: 'Megatron GPT2 log uploads'
+  #  condition: always()
 
-- job: Code_Quality_Checks
-  pool:
-    name: 'DS_testing'
-  variables:
-    conda_root: '/home/deepspeed/miniconda3'
-    conda_env: 'ds_codetest'
-    conda_env_path: '$(conda_root)/envs/$(conda_env)'
+  #- task: PublishPipelineArtifact@1
+  #  inputs:
+  #    targetPath: '$(Build.SourcesDirectory)/tests/model/Megatron_GPT2/checkpoint_test_logs/'
+  #    artifactName: Megatron_GPT2_checkpoint_logs
+  #  displayName: 'Megatron GPT2 checkpoint log uploads'
+  #  condition: always()
 
-  steps:
-  - script: |
-      conda create --force --yes -n $(conda_env) python=3.7
-      source $(conda_root)/bin/activate $(conda_env_path)
-    displayName: 'Create code test environment'
 
-  - script: |
-      source $(conda_root)/bin/activate $(conda_env_path)
-      pip install pre-commit
-      pre-commit run --all-files
-    displayName: 'Formatting checks'
+  #BingBert logs
+  #- task: PublishPipelineArtifact@1
+  #  inputs:
+  #    targetPath: '$(Build.SourcesDirectory)/tests/model/bing_bert/pretrain_test/'
+  #    artifactName: BingBert_pretrain_logs
+  #  displayName: 'BingBert pretrain logs'
+  #  condition: always()
 
-  - script: |
-      source $(conda_root)/bin/activate $(conda_env_path)
-      pip install pylint
-      pylint --exit-zero deepspeed/
-    displayName: 'Code linter'
+  #- task: PublishPipelineArtifact@1
+  #  inputs:
+  #    targetPath: '$(Build.SourcesDirectory)/tests/model/bing_bert/checkpoint_test_logs/'
+  #    artifactName: BingBert_checkpoint_logs
+  #  displayName: 'BingBert checkpoint logs'
+  #  condition: always()