Skip to content

GitLab

Unverified Commit 65a2ceb8 authored by Yuge Zhang's avatar Yuge Zhang Committed by GitHub
Browse files

[CI] Fast-test cross-platform speedup (#3364)

parent bb3d2986
Hide whitespace changes
Inline Side-by-side
Showing with 161 additions and 104 deletions
dependencies/recommended.txt
View file @ 65a2ceb8
# Recommended because some non-commonly-used modules/examples depend on those packages.
-f https://download.pytorch.org/whl/torch_stable.html
tensorflow
torch >= 1.6+cpu, != 1.7+cpu -f https://download.pytorch.org/whl/torch_stable.html
torchvision >= 0.8+cpu -f https://download.pytorch.org/whl/torch_stable.html
# PyTorch 1.7 has compatibility issue with model compression.
# Check for MacOS because this file is used on all platforms.
torch == 1.6.0+cpu ; sys_platform != "darwin"
torch == 1.6.0 ; sys_platform == "darwin"
torchvision == 0.7.0+cpu ; sys_platform != "darwin"
torchvision == 0.7.0 ; sys_platform == "darwin"
onnx
peewee
thop
graphviz
tensorflow
# the following content will be read by setup.py.
# please follow the logic in setup.py.
# SMAC
ConfigSpaceNNI
smac4nni
# BOHB
ConfigSpace==0.4.7
statsmodels==0.12.0
# PPOTuner
enum34
gym
dependencies/recommended_legacy.txt 0 → 100644
View file @ 65a2ceb8
-f https://download.pytorch.org/whl/torch_stable.html
tensorflow == 1.15.4
torch == 1.5.1+cpu
torchvision == 0.6.1+cpu
keras == 2.1.6
onnx
peewee
graphviz
dependencies/required_extra.txt 0 → 100644
View file @ 65a2ceb8
# the following content will be read by setup.py.
# please follow the logic in setup.py.
# SMAC
ConfigSpaceNNI
smac4nni
# BOHB
ConfigSpace==0.4.7
statsmodels==0.12.0
# PPOTuner
gym
pipelines/fast-test.yml
View file @ 65a2ceb8
......@@ -7,23 +7,19 @@ stages:
- job: docs
pool:
vmImage: ubuntu-latest
variables:
PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: 3.8
displayName: Configure Python version
- script: |
sudo apt-get install -y pandoc
sudo apt-get remove swig -y
sudo apt-get install swig3.0 -y
sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
displayName: Install apt packages
- task: Cache@2
inputs:
key: 'python | "$(Agent.OS)" | dependencies/*.txt'
......@@ -32,15 +28,14 @@ stages:
python
path: $(PIP_CACHE_DIR)
displayName: Cache pip packages
- script: |
set -e
python -m pip install -U -r dependencies/setup.txt
python -m pip install -r dependencies/develop.txt
python -m pip install -r dependencies/required.txt
python -m pip install -r dependencies/recommended.txt
python -m pip install -r dependencies/required_extra.txt
displayName: Install requirements
- script: |
cd docs/en_US
sphinx-build -M html . _build -W --keep-going -T
......@@ -49,22 +44,18 @@ stages:
- job: python
pool:
vmImage: ubuntu-latest
variables:
PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: 3.8
displayName: Configure Python version
- script: |
sudo apt-get remove swig -y
sudo apt-get install swig3.0 -y
sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
displayName: Install apt packages
- task: Cache@2
inputs:
key: 'python | "$(Agent.OS)" | dependencies/*.txt'
......@@ -73,19 +64,16 @@ stages:
python
path: $(PIP_CACHE_DIR)
displayName: Cache pip packages
- script: |
set -e
python -m pip install -U -r dependencies/setup.txt
python -m pip install -r dependencies/develop.txt
python -m pip install -r dependencies/required.txt
python -m pip install -r dependencies/recommended.txt
python -m pip install -r dependencies/required_extra.txt
displayName: Install requirements
- script: python -m pylint --rcfile pylintrc nni
displayName: pylint
- script: |
set -e
python -m flake8 nni --count --select=E9,F63,F72,F82 --show-source --statistics
......@@ -96,15 +84,22 @@ stages:
- job: typescript
pool:
vmImage: ubuntu-latest
variables:
YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn
steps:
- task: Cache@2
inputs:
key: 'yarn | "$(Agent.OS)" | ts/**/yarn.lock, !**/node_modules/**'
restoreKeys: |
yarn | "$(Agent.OS)"
path: $(YARN_CACHE_FOLDER)
displayName: Cache yarn packages
- script: |
set -e
cd ts/nni_manager
yarn
yarn eslint
displayName: ESLint (NNI Manager)
- script: |
set -e
cd ts/webui
......@@ -118,8 +113,9 @@ stages:
- job: ubuntu_latest
pool:
vmImage: ubuntu-latest
# This platform tests lint and doc first.
variables:
PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn
steps:
- task: UsePythonVersion@0
......@@ -127,11 +123,34 @@ stages:
versionSpec: 3.8
displayName: Configure Python version
- script: |
sudo apt-get install -y pandoc
sudo apt-get remove swig -y
sudo apt-get install swig3.0 -y
sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
displayName: Install apt packages
- task: Cache@2
inputs:
key: 'python | "$(Agent.OS)" | dependencies/*.txt'
restoreKeys: |
python | "$(Agent.OS)"
python
path: $(PIP_CACHE_DIR)
displayName: Cache pip packages
- task: Cache@2
inputs:
key: 'yarn | "$(Agent.OS)" | ts/**/yarn.lock, !**/node_modules/**'
restoreKeys: |
yarn | "$(Agent.OS)"
path: $(YARN_CACHE_FOLDER)
displayName: Cache yarn packages
- script: |
set -e
python -m pip install --upgrade pip setuptools
python -m pip install pytest coverage
python -m pip install pylint flake8
python -m pip install -U -r dependencies/setup.txt
python -m pip install -r dependencies/develop.txt
echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
displayName: Install Python tools
......@@ -141,41 +160,10 @@ stages:
- script: |
set -e
cd ts/nni_manager
yarn eslint
cd ../webui
yarn eslint
displayName: ESLint
# FIXME: temporarily fixed to pytorch 1.6 as 1.7 won't work with compression
- script: |
set -e
sudo apt-get install -y pandoc
python -m pip install --upgrade pygments
python -m pip install "torch==1.6.0+cpu" "torchvision==0.7.0+cpu" -f https://download.pytorch.org/whl/torch_stable.html
python -m pip install tensorflow
python -m pip install gym onnx peewee thop graphviz
python -m pip install sphinx sphinx-argparse sphinx-rtd-theme sphinxcontrib-websupport nbsphinx
sudo apt-get remove swig -y
sudo apt-get install swig3.0 -y
sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
python -m pip install -e .[SMAC,BOHB]
python -m pip install -r dependencies/recommended.txt
python -m pip install -e .[SMAC,BOHB,PPOTuner]
displayName: Install extra dependencies
- script: |
set -e
python -m pylint --rcfile pylintrc nni
python -m flake8 nni --count --select=E9,F63,F72,F82 --show-source --statistics
EXCLUDES=examples/trials/mnist-nas/*/mnist*.py,examples/trials/nas_cifar10/src/cifar10/general_child.py
python -m flake8 examples --count --exclude=$EXCLUDES --select=E9,F63,F72,F82 --show-source --statistics
displayName: pylint and flake8
- script: |
cd docs/en_US
sphinx-build -M html . _build -W --keep-going -T
displayName: Check Sphinx documentation
- script: |
set -e
cd test
......@@ -200,23 +188,48 @@ stages:
python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
displayName: Simple integration test
- job: ubuntu_legacy
pool:
vmImage: ubuntu-18.04
variables:
PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn
# This platform runs integration test first.
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: 3.6
displayName: Configure Python version
- script: |
sudo apt-get install -y pandoc
sudo apt-get remove swig -y
sudo apt-get install swig3.0 -y
sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
displayName: Install apt packages
- task: Cache@2
inputs:
key: 'python | "$(Agent.OS)" | legacy | dependencies/*.txt'
restoreKeys: |
python | "$(Agent.OS)"
python
path: $(PIP_CACHE_DIR)
displayName: Cache pip packages
- task: Cache@2
inputs:
key: 'yarn | "$(Agent.OS)" | legacy | ts/**/yarn.lock, !**/node_modules/**'
restoreKeys: |
yarn | "$(Agent.OS)"
path: $(YARN_CACHE_FOLDER)
displayName: Cache yarn packages
- script: |
set -e
python -m pip install --upgrade pip setuptools
python -m pip install pytest coverage
python -m pip install -U -r dependencies/setup.txt
python -m pip install -r dependencies/develop.txt
echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
displayName: Install Python tools
......@@ -226,12 +239,8 @@ stages:
- script: |
set -e
python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
python -m pip install tensorflow==1.15.4
python -m pip install keras==2.1.6
python -m pip install gym onnx peewee
sudo apt-get install swig -y
python -m pip install -e .[SMAC,BOHB]
python -m pip install -r dependencies/recommended_legacy.txt
python -m pip install -e .[SMAC,BOHB,PPOTuner]
displayName: Install extra dependencies
- script: |
......@@ -252,10 +261,12 @@ stages:
CI=true yarn test
displayName: TypeScript unit test
- job: macos
pool:
vmImage: macOS-10.15
variables:
PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn
# This platform runs TypeScript unit test first.
......@@ -265,11 +276,34 @@ stages:
versionSpec: 3.8
displayName: Configure Python version
- script: |
brew install swig@3
rm -f /usr/local/bin/swig
ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig
displayName: Install brew packages
- task: Cache@2
inputs:
key: 'python | "$(Agent.OS)" | dependencies/*.txt'
restoreKeys: |
python | "$(Agent.OS)"
python
path: $(PIP_CACHE_DIR)
displayName: Cache pip packages
- task: Cache@2
inputs:
key: 'yarn | "$(Agent.OS)" | ts/**/yarn.lock, !**/node_modules/**'
restoreKeys: |
yarn | "$(Agent.OS)"
path: $(YARN_CACHE_FOLDER)
displayName: Cache yarn packages
- script: |
set -e
python -m pip install -U -r dependencies/setup.txt
python -m pip install -r dependencies/develop.txt
echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
python -m pip install --upgrade pip setuptools wheel
python -m pip install pytest coverage
displayName: Install Python tools
- script: |
......@@ -285,13 +319,8 @@ stages:
- script: |
set -e
# pytorch Mac binary does not support CUDA, default is cpu version
python -m pip install torchvision==0.6.0 torch==1.5.0
python -m pip install tensorflow==2.3.1
brew install swig@3
rm -f /usr/local/bin/swig
ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig
python -m pip install -e .[SMAC]
python -m pip install -r dependencies/recommended.txt
python -m pip install -e .[SMAC,BOHB,PPOTuner]
displayName: Install extra dependencies
- script: |
......@@ -304,12 +333,12 @@ stages:
python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
displayName: Simple integration test
# FIXME: Windows UT is still under debugging
- job: windows
pool:
vmImage: windows-2019
variables:
PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn
# This platform runs Python unit test first.
......@@ -319,9 +348,27 @@ stages:
versionSpec: 3.8
displayName: Configure Python version
- task: Cache@2
inputs:
key: 'python | "$(Agent.OS)" | dependencies/*.txt'
restoreKeys: |
python | "$(Agent.OS)"
python
path: $(PIP_CACHE_DIR)
displayName: Cache pip packages
- task: Cache@2
inputs:
key: 'yarn | "$(Agent.OS)" | ts/**/yarn.lock, !**/node_modules/**'
restoreKeys: |
yarn | "$(Agent.OS)"
path: $(YARN_CACHE_FOLDER)
displayName: Cache yarn packages
- script: |
python -m pip install --upgrade pip setuptools
python -m pip install pytest coverage
set -e
python -m pip install -U -r dependencies/setup.txt
python -m pip install -r dependencies/develop.txt
displayName: Install Python tools
- script: |
......@@ -329,9 +376,7 @@ stages:
displayName: Install NNI
- script: |
python -m pip install scikit-learn==0.23.2
python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
python -m pip install tensorflow==2.3.1
python -m pip install -r dependencies/recommended.txt
displayName: Install extra dependencies
- script: |
......
setup.py
View file @ 65a2ceb8
......@@ -86,9 +86,9 @@ def _setup():
python_requires = '>=3.6',
install_requires = _read_requirements_txt('dependencies/required.txt'),
extras_require = {
'SMAC': _read_requirements_txt('dependencies/recommended.txt', 'SMAC'),
'BOHB': _read_requirements_txt('dependencies/recommended.txt', 'BOHB'),
'PPOTuner': _read_requirements_txt('dependencies/recommended.txt', 'PPOTuner')
'SMAC': _read_requirements_txt('dependencies/required_extra.txt', 'SMAC'),
'BOHB': _read_requirements_txt('dependencies/required_extra.txt', 'BOHB'),
'PPOTuner': _read_requirements_txt('dependencies/required_extra.txt', 'PPOTuner')
},
setup_requires = ['requests'],
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment