Skip to content

GitLab

Unverified Commit f11aea0d authored by Yuge Zhang's avatar Yuge Zhang Committed by GitHub
Browse files

[CI] Introduce multi-stage pipeline for speedup (#3313) 

* Refactor fast-test

* Fix yaml syntax error

* Fix syntax issues

* Update requirements

* Update requirements

* Fix pylint

* Fix pylint

* Fix pylint

* Cache pip packages for lint

* Add wheel as dependency

* Isolate setup dependencies

* Isolate docs pipeline

* Fix swig issue
parent e6629a76
Hide whitespace changes
Inline Side-by-side
Showing with 428 additions and 267 deletions
dependencies/develop.txt 0 → 100644
View file @ f11aea0d
pylint
flake8
sphinx
sphinx-argparse
sphinx-rtd-theme
sphinxcontrib-websupport
nbsphinx
pytest
coverage
dependencies/recommended.txt 0 → 100644
View file @ f11aea0d
tensorflow
torch >= 1.6+cpu, != 1.7+cpu -f https://download.pytorch.org/whl/torch_stable.html
torchvision >= 0.8+cpu -f https://download.pytorch.org/whl/torch_stable.html
onnx
peewee
thop
graphviz
tensorflow
# the following content will be read by setup.py.
# please follow the logic in setup.py.
# SMAC
ConfigSpaceNNI
smac4nni
# BOHB
ConfigSpace==0.4.7
statsmodels==0.12.0
# PPOTuner
enum34
gym
dependencies/required.txt 0 → 100644
View file @ f11aea0d
astor
hyperopt==0.1.2
json_tricks
netifaces
psutil
ruamel.yaml
requests
responses
schema
PythonWebHDFS
colorama
scikit-learn>=0.23.2
websockets
filelock
prettytable
dataclasses ; python_version < "3.7"
numpy < 1.19.4 ; sys_platform == "win32"
numpy < 1.20 ; sys_platform != "win32" and python_version < "3.7"
numpy ; sys.platform != "win32" and python_version >= "3.7"
scipy < 1.6 ; python_version < "3.7"
scipy ; python_version >= "3.7"
dependencies/setup.txt 0 → 100644
View file @ f11aea0d
pip
wheel
setuptools
nni/__init__.py
View file @ f11aea0d
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
# Licensed under the MIT license. # Licensed under the MIT license.
try: try:
from .version import __version__ from .version import __version__ # pylint: disable=import-error
except ModuleNotFoundError: except ModuleNotFoundError:
__version__ = '999.dev0' __version__ = '999.dev0'
......
nni/experiment/launcher.py
View file @ f11aea0d
...@@ -9,8 +9,8 @@ from typing import Optional, Tuple ...@@ -9,8 +9,8 @@ from typing import Optional, Tuple
import colorama import colorama
import nni_node # pylint: disable=import-error
import nni.runtime.protocol import nni.runtime.protocol
import nni_node
from .config import ExperimentConfig from .config import ExperimentConfig
from .config import convert from .config import convert
......
nni/tools/nnictl/launcher.py
View file @ f11aea0d
...@@ -12,7 +12,7 @@ from subprocess import Popen, check_call, CalledProcessError, PIPE, STDOUT ...@@ -12,7 +12,7 @@ from subprocess import Popen, check_call, CalledProcessError, PIPE, STDOUT
from nni.experiment.config import ExperimentConfig, convert from nni.experiment.config import ExperimentConfig, convert
from nni.tools.annotation import expand_annotations, generate_search_space from nni.tools.annotation import expand_annotations, generate_search_space
from nni.tools.package_utils import get_builtin_module_class_name from nni.tools.package_utils import get_builtin_module_class_name
import nni_node import nni_node # pylint: disable=import-error
from .launcher_utils import validate_all_content from .launcher_utils import validate_all_content
from .rest_utils import rest_put, rest_post, check_rest_server, check_response from .rest_utils import rest_put, rest_post, check_rest_server, check_response
from .url_utils import cluster_metadata_url, experiment_url, get_local_urls from .url_utils import cluster_metadata_url, experiment_url, get_local_urls
......
nni/tools/nnictl/nnictl_utils.py
View file @ f11aea0d
...@@ -15,7 +15,7 @@ from datetime import datetime, timezone ...@@ -15,7 +15,7 @@ from datetime import datetime, timezone
from subprocess import Popen from subprocess import Popen
from pyhdfs import HdfsClient from pyhdfs import HdfsClient
from nni.tools.annotation import expand_annotations from nni.tools.annotation import expand_annotations
import nni_node import nni_node # pylint: disable=import-error
from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response
from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url, metric_data_url from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url, metric_data_url
from .config_utils import Config, Experiments from .config_utils import Config, Experiments
......
pipelines/fast-test.yml
View file @ f11aea0d
# To reduce debug cost, steps are sorted differently on each platform, # To reduce debug cost, steps are sorted differently on each platform,
# so that a bug in any module will cause at least one platform to fail quickly. # so that a bug in any module will cause at least one platform to fail quickly.
jobs: stages:
- job: ubuntu_latest - stage: lint
pool: jobs:
vmImage: ubuntu-latest - job: docs
pool:
# This platform tests lint and doc first. vmImage: ubuntu-latest
steps: variables:
- task: UsePythonVersion@0 PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
inputs:
versionSpec: 3.8 steps:
displayName: Configure Python version - task: UsePythonVersion@0
inputs:
- script: | versionSpec: 3.8
set -e displayName: Configure Python version
python -m pip install --upgrade pip setuptools
python -m pip install pytest coverage - script: |
python -m pip install pylint flake8 sudo apt-get install -y pandoc
echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}" sudo apt-get remove swig -y
displayName: Install Python tools sudo apt-get install swig3.0 -y
sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
- script: | displayName: Install apt packages
python setup.py develop
displayName: Install NNI - task: Cache@2
inputs:
- script: | key: 'python | "$(Agent.OS)" | dependencies/*.txt'
set -e restoreKeys: |
cd ts/nni_manager python | "$(Agent.OS)"
yarn eslint python
cd ../webui path: $(PIP_CACHE_DIR)
yarn eslint displayName: Cache pip packages
displayName: ESLint
- script: |
# FIXME: temporarily fixed to pytorch 1.6 as 1.7 won't work with compression set -e
python -m pip install -U -r dependencies/setup.txt
- script: | python -m pip install -r dependencies/develop.txt
set -e python -m pip install -r dependencies/required.txt
sudo apt-get install -y pandoc python -m pip install -r dependencies/recommended.txt
python -m pip install --upgrade pygments displayName: Install requirements
python -m pip install "torch==1.6.0+cpu" "torchvision==0.7.0+cpu" -f https://download.pytorch.org/whl/torch_stable.html
python -m pip install tensorflow - script: |
python -m pip install gym onnx peewee thop graphviz cd docs/en_US
python -m pip install sphinx==3.3.1 sphinx-argparse==0.2.5 sphinx-rtd-theme==0.4.2 sphinxcontrib-websupport==1.1.0 nbsphinx sphinx-build -M html . _build -W --keep-going -T
sudo apt-get remove swig -y displayName: Sphinx # TODO: rstcheck
sudo apt-get install swig3.0 -y
sudo ln -s /usr/bin/swig3.0 /usr/bin/swig - job: python
python -m pip install -e .[SMAC,BOHB] pool:
displayName: Install extra dependencies vmImage: ubuntu-latest
- script: | variables:
set -e PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
python -m pylint --rcfile pylintrc nni
python -m flake8 nni --count --select=E9,F63,F72,F82 --show-source --statistics steps:
EXCLUDES=examples/trials/mnist-nas/*/mnist*.py,examples/trials/nas_cifar10/src/cifar10/general_child.py - task: UsePythonVersion@0
python -m flake8 examples --count --exclude=$EXCLUDES --select=E9,F63,F72,F82 --show-source --statistics inputs:
displayName: pylint and flake8 versionSpec: 3.8
displayName: Configure Python version
- script: |
cd docs/en_US - script: |
sphinx-build -M html . _build -W --keep-going -T sudo apt-get remove swig -y
displayName: Check Sphinx documentation sudo apt-get install swig3.0 -y
sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
- script: | displayName: Install apt packages
set -e
cd test - task: Cache@2
python -m pytest ut --ignore=ut/sdk/test_pruners.py \ inputs:
--ignore=ut/sdk/test_compressor_tf.py \ key: 'python | "$(Agent.OS)" | dependencies/*.txt'
--ignore=ut/sdk/test_compressor_torch.py restoreKeys: |
python -m pytest ut/sdk/test_pruners.py python | "$(Agent.OS)"
python -m pytest ut/sdk/test_compressor_tf.py python
python -m pytest ut/sdk/test_compressor_torch.py path: $(PIP_CACHE_DIR)
displayName: Python unit test displayName: Cache pip packages
- script: | - script: |
set -e set -e
cd ts/nni_manager python -m pip install -U -r dependencies/setup.txt
yarn test python -m pip install -r dependencies/develop.txt
cd ../nasui python -m pip install -r dependencies/required.txt
CI=true yarn test python -m pip install -r dependencies/recommended.txt
displayName: TypeScript unit test displayName: Install requirements
- script: |
cd test - script: python -m pylint --rcfile pylintrc nni
python nni_test/nnitest/run_tests.py --config config/pr_tests.yml displayName: pylint
displayName: Simple integration test
- script: |
set -e
- job: ubuntu_legacy python -m flake8 nni --count --select=E9,F63,F72,F82 --show-source --statistics
pool: EXCLUDES=examples/trials/mnist-nas/*/mnist*.py,examples/trials/nas_cifar10/src/cifar10/general_child.py
vmImage: ubuntu-18.04 python -m flake8 examples --count --exclude=$EXCLUDES --select=E9,F63,F72,F82 --show-source --statistics
displayName: flake8
# This platform runs integration test first.
- job: typescript
steps: pool:
- task: UsePythonVersion@0 vmImage: ubuntu-latest
inputs:
versionSpec: 3.6 steps:
displayName: Configure Python version - script: |
set -e
- script: | cd ts/nni_manager
set -e yarn
python -m pip install --upgrade pip setuptools yarn eslint
python -m pip install pytest coverage displayName: ESLint (NNI Manager)
echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
displayName: Install Python tools - script: |
set -e
- script: | cd ts/webui
python setup.py develop yarn
displayName: Install NNI yarn eslint
displayName: ESLint (WebUI)
- script: |
set -e
python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html - stage: test
python -m pip install tensorflow==1.15.4 jobs:
python -m pip install keras==2.1.6 - job: ubuntu_latest
python -m pip install gym onnx peewee pool:
sudo apt-get install swig -y vmImage: ubuntu-latest
python -m pip install -e .[SMAC,BOHB]
displayName: Install extra dependencies # This platform tests lint and doc first.
- script: | steps:
cd test - task: UsePythonVersion@0
python nni_test/nnitest/run_tests.py --config config/pr_tests.yml inputs:
displayName: Simple integration test versionSpec: 3.8
displayName: Configure Python version
- script: |
cd test - script: |
python -m pytest ut set -e
displayName: Python unit test python -m pip install --upgrade pip setuptools
python -m pip install pytest coverage
- script: | python -m pip install pylint flake8
set -e echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
cd ts/nni_manager displayName: Install Python tools
yarn test
cd ../nasui - script: |
CI=true yarn test python setup.py develop
displayName: TypeScript unit test displayName: Install NNI
- script: |
- job: macos set -e
pool: cd ts/nni_manager
vmImage: macOS-10.15 yarn eslint
cd ../webui
# This platform runs TypeScript unit test first. yarn eslint
displayName: ESLint
steps:
- task: UsePythonVersion@0 # FIXME: temporarily fixed to pytorch 1.6 as 1.7 won't work with compression
inputs:
versionSpec: 3.8 - script: |
displayName: Configure Python version set -e
sudo apt-get install -y pandoc
- script: | python -m pip install --upgrade pygments
set -e python -m pip install "torch==1.6.0+cpu" "torchvision==0.7.0+cpu" -f https://download.pytorch.org/whl/torch_stable.html
echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin" python -m pip install tensorflow
python -m pip install --upgrade pip setuptools wheel python -m pip install gym onnx peewee thop graphviz
python -m pip install pytest coverage python -m pip install sphinx sphinx-argparse sphinx-rtd-theme sphinxcontrib-websupport nbsphinx
displayName: Install Python tools sudo apt-get remove swig -y
sudo apt-get install swig3.0 -y
- script: | sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
python setup.py develop python -m pip install -e .[SMAC,BOHB]
displayName: Install NNI displayName: Install extra dependencies
- script: | - script: |
set -e set -e
export CI=true python -m pylint --rcfile pylintrc nni
(cd ts/nni_manager && yarn test) python -m flake8 nni --count --select=E9,F63,F72,F82 --show-source --statistics
(cd ts/nasui && yarn test) EXCLUDES=examples/trials/mnist-nas/*/mnist*.py,examples/trials/nas_cifar10/src/cifar10/general_child.py
displayName: TypeScript unit test python -m flake8 examples --count --exclude=$EXCLUDES --select=E9,F63,F72,F82 --show-source --statistics
displayName: pylint and flake8
- script: |
set -e - script: |
# pytorch Mac binary does not support CUDA, default is cpu version cd docs/en_US
python -m pip install torchvision==0.6.0 torch==1.5.0 sphinx-build -M html . _build -W --keep-going -T
python -m pip install tensorflow==2.3.1 displayName: Check Sphinx documentation
brew install swig@3
rm -f /usr/local/bin/swig - script: |
ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig set -e
python -m pip install -e .[SMAC] cd test
displayName: Install extra dependencies python -m pytest ut --ignore=ut/sdk/test_pruners.py \
--ignore=ut/sdk/test_compressor_tf.py \
- script: | --ignore=ut/sdk/test_compressor_torch.py
cd test python -m pytest ut/sdk/test_pruners.py
python -m pytest ut python -m pytest ut/sdk/test_compressor_tf.py
displayName: Python unit test python -m pytest ut/sdk/test_compressor_torch.py
displayName: Python unit test
- script: |
cd test - script: |
python nni_test/nnitest/run_tests.py --config config/pr_tests.yml set -e
displayName: Simple integration test cd ts/nni_manager
yarn test
cd ../nasui
# FIXME: Windows UT is still under debugging CI=true yarn test
displayName: TypeScript unit test
- job: windows
pool: - script: |
vmImage: windows-2019 cd test
python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
# This platform runs Python unit test first. displayName: Simple integration test
steps:
- task: UsePythonVersion@0 - job: ubuntu_legacy
inputs: pool:
versionSpec: 3.8 vmImage: ubuntu-18.04
displayName: Configure Python version
# This platform runs integration test first.
- script: |
python -m pip install --upgrade pip setuptools steps:
python -m pip install pytest coverage - task: UsePythonVersion@0
displayName: Install Python tools inputs:
versionSpec: 3.6
- script: | displayName: Configure Python version
python setup.py develop --no-user
displayName: Install NNI - script: |
set -e
- script: | python -m pip install --upgrade pip setuptools
python -m pip install scikit-learn==0.23.2 python -m pip install pytest coverage
python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
python -m pip install tensorflow==2.3.1 displayName: Install Python tools
displayName: Install extra dependencies
- script: |
- script: | python setup.py develop
cd test displayName: Install NNI
python -m pytest ut
displayName: Python unit test - script: |
set -e
- script: | python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
cd ts/nni_manager python -m pip install tensorflow==1.15.4
yarn test python -m pip install keras==2.1.6
displayName: TypeScript unit test python -m pip install gym onnx peewee
sudo apt-get install swig -y
- script: | python -m pip install -e .[SMAC,BOHB]
cd test displayName: Install extra dependencies
python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
displayName: Simple integration test - script: |
cd test
python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
displayName: Simple integration test
- script: |
cd test
python -m pytest ut
displayName: Python unit test
- script: |
set -e
cd ts/nni_manager
yarn test
cd ../nasui
CI=true yarn test
displayName: TypeScript unit test
- job: macos
pool:
vmImage: macOS-10.15
# This platform runs TypeScript unit test first.
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: 3.8
displayName: Configure Python version
- script: |
set -e
echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
python -m pip install --upgrade pip setuptools wheel
python -m pip install pytest coverage
displayName: Install Python tools
- script: |
python setup.py develop
displayName: Install NNI
- script: |
set -e
export CI=true
(cd ts/nni_manager && yarn test)
(cd ts/nasui && yarn test)
displayName: TypeScript unit test
- script: |
set -e
# pytorch Mac binary does not support CUDA, default is cpu version
python -m pip install torchvision==0.6.0 torch==1.5.0
python -m pip install tensorflow==2.3.1
brew install swig@3
rm -f /usr/local/bin/swig
ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig
python -m pip install -e .[SMAC]
displayName: Install extra dependencies
- script: |
cd test
python -m pytest ut
displayName: Python unit test
- script: |
cd test
python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
displayName: Simple integration test
# FIXME: Windows UT is still under debugging
- job: windows
pool:
vmImage: windows-2019
# This platform runs Python unit test first.
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: 3.8
displayName: Configure Python version
- script: |
python -m pip install --upgrade pip setuptools
python -m pip install pytest coverage
displayName: Install Python tools
- script: |
python setup.py develop --no-user
displayName: Install NNI
- script: |
python -m pip install scikit-learn==0.23.2
python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
python -m pip install tensorflow==2.3.1
displayName: Install extra dependencies
- script: |
cd test
python -m pytest ut
displayName: Python unit test
- script: |
cd ts/nni_manager
yarn test
displayName: TypeScript unit test
- script: |
cd test
python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
displayName: Simple integration test
trigger: trigger:
......
setup.py
View file @ f11aea0d
...@@ -55,31 +55,6 @@ from setuptools.command.develop import develop ...@@ -55,31 +55,6 @@ from setuptools.command.develop import develop
import setup_ts import setup_ts
dependencies = [
'astor',
'hyperopt==0.1.2',
'json_tricks',
'netifaces',
'psutil',
'ruamel.yaml',
'requests',
'responses',
'schema',
'PythonWebHDFS',
'colorama',
'scikit-learn>=0.23.2',
'websockets',
'filelock',
'prettytable',
'dataclasses ; python_version < "3.7"',
'numpy < 1.19.4 ; sys_platform == "win32"',
'numpy < 1.20 ; sys_platform != "win32" and python_version < "3.7"',
'numpy ; sys.platform != "win32" and python_version >= "3.7"',
'scipy < 1.6 ; python_version < "3.7"',
'scipy ; python_version >= "3.7"',
]
release = os.environ.get('NNI_RELEASE') release = os.environ.get('NNI_RELEASE')
def _setup(): def _setup():
...@@ -109,11 +84,11 @@ def _setup(): ...@@ -109,11 +84,11 @@ def _setup():
}, },
python_requires = '>=3.6', python_requires = '>=3.6',
install_requires = dependencies, install_requires = _read_requirements_txt('dependencies/required.txt'),
extras_require = { extras_require = {
'SMAC': ['ConfigSpaceNNI', 'smac4nni'], 'SMAC': _read_requirements_txt('dependencies/recommended.txt', 'SMAC'),
'BOHB': ['ConfigSpace==0.4.7', 'statsmodels==0.12.0'], 'BOHB': _read_requirements_txt('dependencies/recommended.txt', 'BOHB'),
'PPOTuner': ['enum34', 'gym'] 'PPOTuner': _read_requirements_txt('dependencies/recommended.txt', 'PPOTuner')
}, },
setup_requires = ['requests'], setup_requires = ['requests'],
...@@ -162,6 +137,23 @@ def _find_node_files(): ...@@ -162,6 +137,23 @@ def _find_node_files():
files.remove('__init__.py') files.remove('__init__.py')
return sorted(files) return sorted(files)
def _read_requirements_txt(file_path, section=None):
with open(file_path) as f:
lines = [line.strip() for line in f.readlines() if line.strip()] # remove whitespaces and empty lines
if section is None:
return [line for line in lines if not line.startswith('#')]
selected_lines = []
started = False
for line in lines:
if started:
if line.startswith('#'):
return selected_lines
else:
selected_lines.append(line)
elif line.startswith('# ' + section):
started = True
return selected_lines
def _using_conda_or_virtual_environment(): def _using_conda_or_virtual_environment():
return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta')) return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta'))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment