Skip to content

GitLab

Unverified Commit c7863d97 authored by Shaden Smith's avatar Shaden Smith Committed by GitHub
Browse files

Test matrix (#213)

parent 0be026e3
Hide whitespace changes
Inline Side-by-side
Showing with 67 additions and 53 deletions
azure-pipelines.yml
View file @ c7863d97
jobs: jobs:
- job: Default - job: DeepSpeed_Tests
timeoutInMinutes: 360 timeoutInMinutes: 360
pool: pool:
name: 'GPU_testing' name: 'DS_testing'
strategy: strategy:
matrix: matrix:
Python36: PyTorch12:
python.version: '3.6' python.version: '3.6'
#Python35: cuda.version: '10.0'
# python.version: '3.5' pytorch.version: '1.2'
#Python37: runmodeltests: false
# python.version: '3.7' PyTorch15:
#Python38: python.version: '3.7'
# python.version: '3.8' cuda.version: '10.1'
pytorch.version: '1.5'
runmodeltests: true
variables:
conda_root: '/home/deepspeed/miniconda3'
conda_env: 'ds_test_py$(python.version)_cuda$(cuda.version)_pytorch$(pytorch.version)'
conda_env_path: '$(conda_root)/envs/$(conda_env)'
steps: steps:
- task: UsePythonVersion@0 # Unfortunately nvidia's nvcc_linux-64=<version> seems to install 10.1 regardless?
inputs: # Most of this complexity is a workaround to get the compiler toolchain to match the
versionSpec: '$(python.version)' # cudatoolkit runtime
addToPath: true
architecture: 'x64'
displayName: 'Use Python $(python.version)'
- script: | - script: |
python -m pip install --upgrade pip conda create --force --yes -n $(conda_env) python=$(python.version) cudatoolkit=$(cuda.version)
pip install --user -r requirements.txt source $(conda_root)/bin/activate $(conda_env_path)
./install.sh --pip_sudo conda install --yes -c pytorch pytorch=$(pytorch.version) cudatoolkit=$(cuda.version)
displayName: 'Install dependencies' conda install --yes conda
conda install --yes gxx_linux-64
conda install --yes -c conda-forge cudatoolkit-dev=$(cuda.version)
displayName: 'Setup environment python=$(python.version) pytorch=$(pytorch.version) cuda=$(cuda.version)'
- script: | - script: |
pre-commit run --all-files source $(conda_root)/bin/activate $(conda_env_path)
displayName: 'Formatting checks' python --version
which nvcc
nvcc --version
python -c "import torch; print('torch:', torch.__version__)"
displayName: 'Show environment'
- script: | - script: |
pip install --user pylint source $(conda_root)/bin/activate $(conda_env_path)
pylint --exit-zero deepspeed/ rm -rf third_party/apex/build/
displayName: 'Code linter' rm -rf third_party/apex/dist/
rm -rf build/
rm -rf dist/
./install.sh
displayName: 'Install DeepSpeed'
- script: | - script: |
source $(conda_root)/bin/activate $(conda_env_path)
pytest --forked --verbose tests/unit/ pytest --forked --verbose tests/unit/
displayName: 'Unit tests' displayName: 'Unit tests'
- script: | - script: |
source $(conda_root)/bin/activate $(conda_env_path)
ln -s /data/Megatron-LM/data DeepSpeedExamples/Megatron-LM/ ln -s /data/Megatron-LM/data DeepSpeedExamples/Megatron-LM/
pip install --user -r DeepSpeedExamples/Megatron-LM/requirements.txt pip install -r DeepSpeedExamples/Megatron-LM/requirements.txt
cd tests/model/ cd tests/model/
rm -rf BingBertSquad/baseline
rm -rf Megatron_GPT2/baseline
pytest -s run_sanity_check.py pytest -s run_sanity_check.py
condition: eq(variables['runmodeltests'], true)
displayName: 'Model tests' displayName: 'Model tests'
#BingBertSquad logs #BingBertSquad logs
...@@ -57,35 +75,31 @@ jobs: ...@@ -57,35 +75,31 @@ jobs:
targetPath: '$(Build.SourcesDirectory)/tests/model/BingBertSquad/test/' targetPath: '$(Build.SourcesDirectory)/tests/model/BingBertSquad/test/'
artifactName: BingBertSquad_logs artifactName: BingBertSquad_logs
displayName: 'BingBertSquad log uploads' displayName: 'BingBertSquad log uploads'
condition: always() condition: eq(variables['runmodeltests'], true)
# Megatron test logs
#- task: PublishPipelineArtifact@1
# inputs:
# targetPath: '$(Build.SourcesDirectory)/tests/model/Megatron_GPT2/test/'
# artifactName: Megatron_GPT2_logs
# displayName: 'Megatron GPT2 log uploads'
# condition: always()
#- task: PublishPipelineArtifact@1 - job: Code_Quality_Checks
# inputs: pool:
# targetPath: '$(Build.SourcesDirectory)/tests/model/Megatron_GPT2/checkpoint_test_logs/' name: 'DS_testing'
# artifactName: Megatron_GPT2_checkpoint_logs variables:
# displayName: 'Megatron GPT2 checkpoint log uploads' conda_root: '/home/deepspeed/miniconda3'
# condition: always() conda_env: 'ds_codetest'
conda_env_path: '$(conda_root)/envs/$(conda_env)'
steps:
- script: |
conda create --force --yes -n $(conda_env) python=3.7
source $(conda_root)/bin/activate $(conda_env_path)
displayName: 'Create code test environment'
#BingBert logs - script: |
#- task: PublishPipelineArtifact@1 source $(conda_root)/bin/activate $(conda_env_path)
# inputs: pip install pre-commit
# targetPath: '$(Build.SourcesDirectory)/tests/model/bing_bert/pretrain_test/' pre-commit run --all-files
# artifactName: BingBert_pretrain_logs displayName: 'Formatting checks'
# displayName: 'BingBert pretrain logs'
# condition: always()
#- task: PublishPipelineArtifact@1 - script: |
# inputs: source $(conda_root)/bin/activate $(conda_env_path)
# targetPath: '$(Build.SourcesDirectory)/tests/model/bing_bert/checkpoint_test_logs/' pip install pylint
# artifactName: BingBert_checkpoint_logs pylint --exit-zero deepspeed/
# displayName: 'BingBert checkpoint logs' displayName: 'Code linter'
# condition: always()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment