[CI] Introduce multi-stage pipeline for speedup (#3313)

* Refactor fast-test

* Fix yaml syntax error

* Fix syntax issues

* Update requirements

* Update requirements

* Fix pylint

* Fix pylint

* Fix pylint

* Cache pip packages for lint

* Add wheel as dependency

* Isolate setup dependencies

* Isolate docs pipeline

* Fix swig issue

dependencies/develop.txt

+pylint
+flake8
+sphinx
+sphinx-argparse
+sphinx-rtd-theme
+sphinxcontrib-websupport
+nbsphinx
+pytest
+coverage

dependencies/recommended.txt

+tensorflow
+torch >= 1.6+cpu, != 1.7+cpu -f https://download.pytorch.org/whl/torch_stable.html
+torchvision >= 0.8+cpu -f https://download.pytorch.org/whl/torch_stable.html
+onnx
+peewee
+thop
+graphviz
+tensorflow
+
+# the following content will be read by setup.py.
+# please follow the logic in setup.py.
+
+# SMAC
+ConfigSpaceNNI
+smac4nni
+
+# BOHB
+ConfigSpace==0.4.7
+statsmodels==0.12.0
+
+# PPOTuner
+enum34
+gym

dependencies/required.txt

+astor
+hyperopt==0.1.2
+json_tricks
+netifaces
+psutil
+ruamel.yaml
+requests
+responses
+schema
+PythonWebHDFS
+colorama
+scikit-learn>=0.23.2
+websockets
+filelock
+prettytable
+dataclasses ; python_version < "3.7"
+numpy < 1.19.4 ; sys_platform == "win32"
+numpy < 1.20 ; sys_platform != "win32" and python_version < "3.7"
+numpy ; sys.platform != "win32" and python_version >= "3.7"
+scipy < 1.6 ; python_version < "3.7"
+scipy ; python_version >= "3.7"

dependencies/setup.txt

+pip
+wheel
+setuptools

nni/__init__.py

@@ -2,7 +2,7 @@
 # Licensed under the MIT license.
 
 try:
-    from .version import __version__
+    from .version import __version__  # pylint: disable=import-error
 except ModuleNotFoundError:
     __version__ = '999.dev0'
 

nni/experiment/launcher.py

@@ -9,8 +9,8 @@ from typing import Optional, Tuple
 
 import colorama
 
+import nni_node  # pylint: disable=import-error
 import nni.runtime.protocol
-import nni_node
 
 from .config import ExperimentConfig
 from .config import convert

nni/tools/nnictl/launcher.py

@@ -12,7 +12,7 @@ from subprocess import Popen, check_call, CalledProcessError, PIPE, STDOUT
 from nni.experiment.config import ExperimentConfig, convert
 from nni.tools.annotation import expand_annotations, generate_search_space
 from nni.tools.package_utils import get_builtin_module_class_name
-import nni_node
+import nni_node  # pylint: disable=import-error
 from .launcher_utils import validate_all_content
 from .rest_utils import rest_put, rest_post, check_rest_server, check_response
 from .url_utils import cluster_metadata_url, experiment_url, get_local_urls

nni/tools/nnictl/nnictl_utils.py

@@ -15,7 +15,7 @@ from datetime import datetime, timezone
 from subprocess import Popen
 from pyhdfs import HdfsClient
 from nni.tools.annotation import expand_annotations
-import nni_node
+import nni_node  # pylint: disable=import-error
 from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response
 from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url, metric_data_url
 from .config_utils import Config, Experiments

pipelines/fast-test.yml

 # To reduce debug cost, steps are sorted differently on each platform,
 # so that a bug in any module will cause at least one platform to fail quickly.
 
-jobs:
-- job: ubuntu_latest
+stages:
+- stage: lint
+  jobs:
+  - job: docs
+    pool:
+      vmImage: ubuntu-latest
+
+    variables:
+      PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
+
+    steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: 3.8
+      displayName: Configure Python version
+
+    - script: |
+        sudo apt-get install -y pandoc
+        sudo apt-get remove swig -y
+        sudo apt-get install swig3.0 -y
+        sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
+      displayName: Install apt packages
+
+    - task: Cache@2
+      inputs:
+        key: 'python | "$(Agent.OS)" | dependencies/*.txt'
+        restoreKeys: | 
+          python | "$(Agent.OS)"
+          python
+        path: $(PIP_CACHE_DIR)
+      displayName: Cache pip packages
+
+    - script: |
+        set -e
+        python -m pip install -U -r dependencies/setup.txt
+        python -m pip install -r dependencies/develop.txt
+        python -m pip install -r dependencies/required.txt
+        python -m pip install -r dependencies/recommended.txt
+      displayName: Install requirements
+
+    - script: |
+        cd docs/en_US
+        sphinx-build -M html . _build -W --keep-going -T
+      displayName: Sphinx  # TODO: rstcheck
+
+  - job: python
+    pool:
+      vmImage: ubuntu-latest
+
+    variables:
+      PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
+
+    steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: 3.8
+      displayName: Configure Python version
+
+    - script: |
+        sudo apt-get remove swig -y
+        sudo apt-get install swig3.0 -y
+        sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
+      displayName: Install apt packages
+
+    - task: Cache@2
+      inputs:
+        key: 'python | "$(Agent.OS)" | dependencies/*.txt'
+        restoreKeys: | 
+          python | "$(Agent.OS)"
+          python
+        path: $(PIP_CACHE_DIR)
+      displayName: Cache pip packages
+
+    - script: |
+        set -e
+        python -m pip install -U -r dependencies/setup.txt
+        python -m pip install -r dependencies/develop.txt
+        python -m pip install -r dependencies/required.txt
+        python -m pip install -r dependencies/recommended.txt
+      displayName: Install requirements
+
+
+    - script: python -m pylint --rcfile pylintrc nni
+      displayName: pylint
+
+    - script: |
+        set -e
+        python -m flake8 nni --count --select=E9,F63,F72,F82 --show-source --statistics
+        EXCLUDES=examples/trials/mnist-nas/*/mnist*.py,examples/trials/nas_cifar10/src/cifar10/general_child.py
+        python -m flake8 examples --count --exclude=$EXCLUDES --select=E9,F63,F72,F82 --show-source --statistics
+      displayName: flake8
+
+  - job: typescript
+    pool:
+      vmImage: ubuntu-latest
+  
+    steps:
+    - script: |
+        set -e
+        cd ts/nni_manager
+        yarn
+        yarn eslint
+      displayName: ESLint (NNI Manager)
+    
+    - script: |
+        set -e
+        cd ts/webui
+        yarn
+        yarn eslint
+      displayName: ESLint (WebUI)
+
+
+- stage: test
+  jobs:
+  - job: ubuntu_latest
     pool:
       vmImage: ubuntu-latest
 
@@ -43,7 +156,7 @@ jobs:
         python -m pip install "torch==1.6.0+cpu" "torchvision==0.7.0+cpu" -f https://download.pytorch.org/whl/torch_stable.html
         python -m pip install tensorflow
         python -m pip install gym onnx peewee thop graphviz
-      python -m pip install sphinx==3.3.1 sphinx-argparse==0.2.5 sphinx-rtd-theme==0.4.2 sphinxcontrib-websupport==1.1.0 nbsphinx
+        python -m pip install sphinx sphinx-argparse sphinx-rtd-theme sphinxcontrib-websupport nbsphinx
         sudo apt-get remove swig -y
         sudo apt-get install swig3.0 -y
         sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
@@ -88,7 +201,7 @@ jobs:
       displayName: Simple integration test
 
 
-- job: ubuntu_legacy
+  - job: ubuntu_legacy
     pool:
       vmImage: ubuntu-18.04
 
@@ -140,7 +253,7 @@ jobs:
       displayName: TypeScript unit test
 
 
-- job: macos
+  - job: macos
     pool:
       vmImage: macOS-10.15
 
@@ -192,9 +305,9 @@ jobs:
       displayName: Simple integration test
 
 
-# FIXME: Windows UT is still under debugging
+  # FIXME: Windows UT is still under debugging
 
-- job: windows
+  - job: windows
     pool:
       vmImage: windows-2019
 

setup.py

@@ -55,31 +55,6 @@ from setuptools.command.develop import develop
 
 import setup_ts
 
-
-dependencies = [
-    'astor',
-    'hyperopt==0.1.2',
-    'json_tricks',
-    'netifaces',
-    'psutil',
-    'ruamel.yaml',
-    'requests',
-    'responses',
-    'schema',
-    'PythonWebHDFS',
-    'colorama',
-    'scikit-learn>=0.23.2',
-    'websockets',
-    'filelock',
-    'prettytable',
-    'dataclasses ; python_version < "3.7"',
-    'numpy < 1.19.4 ; sys_platform == "win32"',
-    'numpy < 1.20 ; sys_platform != "win32" and python_version < "3.7"',
-    'numpy ; sys.platform != "win32" and python_version >= "3.7"',
-    'scipy < 1.6 ; python_version < "3.7"',
-    'scipy ; python_version >= "3.7"',
-]
-
 release = os.environ.get('NNI_RELEASE')
 
 def _setup():
@@ -109,11 +84,11 @@ def _setup():
         },
 
         python_requires = '>=3.6',
-        install_requires = dependencies,
+        install_requires = _read_requirements_txt('dependencies/required.txt'),
         extras_require = {
-            'SMAC': ['ConfigSpaceNNI', 'smac4nni'],
-            'BOHB': ['ConfigSpace==0.4.7', 'statsmodels==0.12.0'],
-            'PPOTuner': ['enum34', 'gym']
+            'SMAC': _read_requirements_txt('dependencies/recommended.txt', 'SMAC'),
+            'BOHB': _read_requirements_txt('dependencies/recommended.txt', 'BOHB'),
+            'PPOTuner': _read_requirements_txt('dependencies/recommended.txt', 'PPOTuner')
         },
         setup_requires = ['requests'],
 
@@ -162,6 +137,23 @@ def _find_node_files():
         files.remove('__init__.py')
     return sorted(files)
 
+def _read_requirements_txt(file_path, section=None):
+    with open(file_path) as f:
+        lines = [line.strip() for line in f.readlines() if line.strip()]  # remove whitespaces and empty lines
+    if section is None:
+        return [line for line in lines if not line.startswith('#')]
+    selected_lines = []
+    started = False
+    for line in lines:
+        if started:
+            if line.startswith('#'):
+                return selected_lines
+            else:
+                selected_lines.append(line)
+        elif line.startswith('# ' + section):
+            started = True
+    return selected_lines
+
 def _using_conda_or_virtual_environment():
     return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta'))