Fix bugs found in integration test (#3214)

examples/nas/.gitignore

@@ -2,3 +2,4 @@ data
 checkpoints
 runs
 nni_auto_gen_search_space.json
+checkpoint.json

examples/nas/darts/search.py

 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+import json
 import logging
 import time
 from argparse import ArgumentParser
@@ -67,4 +68,6 @@ if __name__ == "__main__":
             unrolled=args.unrolled
         )
         trainer.fit()
+        final_architecture = trainer.export()
         print('Final architecture:', trainer.export())
+        json.dump(trainer.export(), open('checkpoint.json', 'w'))

nni/runtime/config.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+import os
+from pathlib import Path
+import shutil
+import sys
+
+import nni
+
+def get_config_directory() -> Path:
+    """
+    Get NNI config directory.
+    Create it if not exist.
+    """
+    if sys.prefix != sys.base_prefix or Path(sys.prefix, 'conda-meta').is_dir():
+        config_dir = Path(sys.prefix, 'nni')
+    elif sys.platform == 'win32':
+        config_dir = Path(os.environ['APPDATA'], 'nni')
+    else:
+        config_dir = Path.home() / '.config/nni'
+    config_dir.mkdir(parents=True, exist_ok=True)
+    return config_dir
+
+def get_config_file(name: str) -> Path:
+    """
+    Get an NNI config file.
+    Copy from `nni/runtime/default_config` if not exist.
+    """
+    config_file = get_config_directory() / name
+    if not config_file.exists():
+        default = Path(nni.__path__[0], 'runtime/default_config', name)
+        shutil.copyfile(default, config_file)
+    return config_file

nni/runtime/log.py

@@ -74,7 +74,8 @@ def _init_logger_trial() -> None:
     log_file = open(log_path, 'w')
     _setup_root_logger(StreamHandler(log_file), logging.INFO)
 
-    sys.stdout = _LogFileWrapper(log_file)
+    if trial_env_vars.NNI_PLATFORM == 'local':
+        sys.stdout = _LogFileWrapper(log_file)
 
 
 def _init_logger_standalone() -> None:

nni/tools/package_utils/__init__.py

@@ -7,7 +7,7 @@ import os
 from pathlib import Path
 import sys
 import ruamel.yaml as yaml
-import nni
+from nni.runtime.config import get_config_file
 
 ALGO_TYPES = ['tuners', 'assessors', 'advisors']
 
@@ -210,19 +210,7 @@ def _using_conda_or_virtual_environment():
     return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta'))
 
 def get_registered_algo_config_path():
-    # Find the path for registered_algorithms.yml for this nni installation,
-    # the registered_algorithms.yml is copied into this location in setup.py,
-    # so we need to ensure that we use the same logic as setup.py to find the location.
-
-    if _using_conda_or_virtual_environment():
-        nni_config_dir = os.path.join(sys.prefix, 'nni')
-    elif sys.platform == 'win32':
-        nni_config_dir = os.path.join(os.getenv('APPDATA'), 'nni')
-    else:
-        nni_config_dir = os.path.expanduser('~/.config/nni')
-    if not os.path.exists(nni_config_dir):
-        os.makedirs(nni_config_dir, exist_ok=True)
-    return os.path.join(nni_config_dir, 'registered_algorithms.yml')
+    return str(get_config_file('registered_algorithms.yml'))
 
 def read_registerd_algo_meta():
     config_file = get_registered_algo_config_path()

nni/tools/trial_tool/trial_keeper.py

@@ -92,7 +92,10 @@ def main_loop(args):
 
     # Notice: We don't appoint env, which means subprocess wil inherit current environment and that is expected behavior
     log_pipe_stdout = trial_syslogger_stdout.get_pipelog_reader()
-    _trial_process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout, preexec_fn=os.setsid)
+    if sys.platform == 'win32':
+        _trial_process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout)
+    else:
+        _trial_process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout, preexec_fn=os.setsid)
     nni_log(LogType.Info, 'Trial keeper spawns a subprocess (pid {0}) to run command: {1}'.format(_trial_process.pid,
                                                                                                   shlex.split(
                                                                                                       args.trial_command)))

pipelines/full-test-linux.yml

@@ -15,36 +15,33 @@ jobs:
       echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
       echo "##vso[task.setvariable variable=NNI_RELEASE]999.$(date -u +%Y%m%d%H%M%S)"
 
-      python3 -m pip install -U --upgrade pip setuptools
-      python3 -m pip install -U pytest
+      python3 -m pip install --upgrade pip setuptools
+      python3 -m pip install pytest
     displayName: Prepare
 
   - script: |
       set -e
       python3 setup.py build_ts
       python3 setup.py bdist_wheel -p manylinux1_x86_64
-      python3 -m pip install dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl
+      python3 -m pip install dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl[SMAC,BOHB,PPOTuner]
     displayName: Install NNI
 
   - script: |
       set -e
-      python3 -m pip install -U scikit-learn==0.23.2
-      python3 -m pip install -U torchvision==0.4.2
-      python3 -m pip install -U torch==1.3.1
-      python3 -m pip install -U keras==2.1.6
-      python3 -m pip install -U tensorflow==2.3.1 tensorflow-estimator==2.3.0
-      python3 -m pip install -U thop
+      python3 -m pip install scikit-learn==0.23.2
+      python3 -m pip install torchvision==0.4.2
+      python3 -m pip install torch==1.3.1
+      python3 -m pip install keras==2.1.6
+      python3 -m pip install tensorflow==2.3.1 tensorflow-estimator==2.3.0
+      python3 -m pip install thop
       sudo apt-get install swig -y
-      nnictl package install --name=SMAC
-      nnictl package install --name=BOHB
-      nnictl package install --name=PPOTuner
     displayName: Install extra dependencies
 
   - script: |
       set -e
       cd examples/tuners/customized_tuner
       python3 setup.py develop --user
-      nnictl package install .
+      nnictl algo register --meta meta_file.yml
     displayName: Install customized tuner
 
   - script: |

pipelines/full-test-windows.yml

@@ -12,39 +12,38 @@ jobs:
 
   steps:
   - script: |
-      python -m pip install -U --upgrade pip setuptools
-      python -m pip install -U pytest
+      python -m pip install --upgrade pip setuptools
+      python -m pip install pytest
     displayName: Install Python tools
-  
+
   - script: |
       python -m pip uninstall nni --yes
       set NNI_RELEASE=999.0
       python setup.py build_ts
       python setup.py bdist_wheel -p win_amd64
-      python -m pip install dist/nni-999.0-py3-none-win_amd64.whl
+      python -m pip install dist/nni-999.0-py3-none-win_amd64.whl[PPOTuner]
     displayName: Install NNI
-  
+
   - script: |
-      python -m pip install -U scikit-learn==0.23.2
-      python -m pip install -U keras==2.1.6
-      python -m pip install -U torchvision===0.4.1 torch===1.3.1 -f https://download.pytorch.org/whl/torch_stable.html
-      python -m pip install -U tensorflow==2.3.1 tensorflow-estimator==2.3.0
-      nnictl package install --name=PPOTuner
+      python -m pip install scikit-learn==0.23.2
+      python -m pip install keras==2.1.6
+      python -m pip install torchvision===0.4.1 torch===1.3.1 -f https://download.pytorch.org/whl/torch_stable.html
+      python -m pip install tensorflow==2.3.1 tensorflow-estimator==2.3.0
     displayName: Install extra dependencies
-  
+
   - script: |
       cd examples/tuners/customized_tuner
       python setup.py develop --user
-      nnictl package install .
+      nnictl algo register --meta meta_file.yml
     displayName: Install example customized tuner
-  
+
   - script: |
       cd test
       python -m pytest ut
       echo "TODO: TypeScript UT"
     displayName: Unit test
     continueOnError: true
-  
+
   - script: |
       cd test
       python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts local

pipelines/integration-test-frameworkcontroller.yml

+trigger: none
+pr: none
+schedules:
+- cron: 0 16 * * *
+  branches:
+    include: [ master ]
+
+# variables set on VSO: (mostly for security concern)
+#   manager_ip
+#   docker_hub_password
+
+jobs:
+- job: frameworkcontroller
+  pool: NNI CI KUBE CLI
+  timeoutInMinutes: 120
+
+  steps:
+  - script: |
+      echo "Working directory: ${PWD}"
+      echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
+      python3 test/vso_tools/generate_nni_version.py
+      python3 -m pip install --upgrade pip setuptools
+    displayName: Prepare
+
+  - script: |
+      set -e
+      python3 test/vso_tools/install_nni.py $(NNI_RELEASE) SMAC,BOHB
+      cd examples/tuners/customized_tuner
+      python3 setup.py develop --user
+      nnictl algo register --meta meta_file.yml
+    displayName: Install NNI
+
+  - script: |
+      set -e
+      docker login -u nnidev -p $(docker_hub_password)
+      docker build --build-arg NNI_RELEASE=$(NNI_RELEASE) -t nnidev/nni-nightly .
+      docker push nnidev/nni-nightly
+    displayName: Build and upload docker image
+
+  - script: |
+      set -e
+      cd test
+      python3 nni_test/nnitest/generate_ts_config.py \
+          --ts frameworkcontroller \
+          --keyvault_vaultname NNIKeyVault \
+          --keyvault_name AzureStorageAccountKey \
+          --azs_account nniazurestorage \
+          --azs_share nni \
+          --nni_docker_image nnidev/nni-nightly \
+          --nni_manager_ip $(manager_ip)
+      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts frameworkcontroller --exclude multi-phase
+    displayName: Integration test

pipelines/integration-test-kubeflow.yml

+trigger: none
+pr: none
+schedules:
+- cron: 0 16 * * *
+  branches:
+    include: [ master ]
+
+# variables set on VSO: (mostly for security concern)
+#   manager_ip
+#   docker_hub_password
+
+jobs:
+- job: kubeflow
+  pool: NNI CI KUBE CLI
+  timeoutInMinutes: 120
+
+  steps:
+  - script: |
+      export NNI_RELEASE=999.$(date -u +%Y%m%d%H%M%S)
+      echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
+      echo "##vso[task.setvariable variable=NNI_RELEASE]${NNI_RELEASE}"
+
+      echo "Working directory: ${PWD}"
+      echo "NNI version: ${NNI_RELEASE}"
+
+      python3 -m pip install --upgrade pip setuptools
+    displayName: Prepare
+
+  - script: |
+      set -e
+      python3 setup.py build_ts
+      python3 setup.py bdist_wheel -p manylinux1_x86_64
+      python3 -m pip install dist/nni-$(NNI_RELEASE)-py3-none-manylinux1_x86_64.whl[SMAC,BOHB]
+    displayName: Build and install NNI
+
+  - script: |
+      set -e
+      cd examples/tuners/customized_tuner
+      python3 setup.py develop --user
+      nnictl algo register --meta meta_file.yml
+    displayName: Install customized tuner
+
+  - script: |
+      set -e
+      docker login -u nnidev -p $(docker_hub_password)
+      docker build --build-arg NNI_RELEASE=$(NNI_RELEASE) -t nnidev/nni-nightly .
+      docker push nnidev/nni-nightly
+    displayName: Build and upload docker image
+
+  - script: |
+      set -e
+      cd test
+      python3 nni_test/nnitest/generate_ts_config.py \
+          --ts kubeflow \
+          --keyvault_vaultname NNIKeyVault \
+          --keyvault_name AzureStorageAccountKey \
+          --azs_account nniazurestorage \
+          --azs_share nni \
+          --nni_docker_image nnidev/nni-nightly \
+          --nni_manager_ip $(manager_ip)
+      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts kubeflow --exclude multi-phase
+    displayName: Integration test

pipelines/integration-test-openpai-linux.yml

@@ -26,37 +26,30 @@ jobs:
       echo "NNI version: ${NNI_RELEASE}"
       echo "Build docker image: $(build_docker_image)"
 
-      python3 -m pip install -U --upgrade pip setuptools
+      python3 -m pip install --upgrade pip setuptools
     displayName: Prepare
 
   - script: |
       set -e
       python3 setup.py build_ts
       python3 setup.py bdist_wheel -p manylinux1_x86_64
-      python3 -m pip install -U dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl
+      python3 -m pip install dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl[SMAC,BOHB]
     displayName: Build and install NNI
 
-  - script: |
-      set -e
-      sudo apt-get install swig -y
-      nnictl package install --name=SMAC
-      nnictl package install --name=BOHB
-    displayName: Install extra tuners
-
   - script: |
       set -e
       cd examples/tuners/customized_tuner
       python3 setup.py develop --user
-      nnictl package install .
+      nnictl algo register --meta meta_file.yml
     displayName: Install customized tuner
 
   - script: |
       set -e
       docker login -u nnidev -p $(docker_hub_password)
       echo '## Build docker image ##'
-      docker build --build-arg NNI_RELEASE=${NNI_RELEASE} -t nnidev/nni-it-pai:latest .
+      docker build --build-arg NNI_RELEASE=${NNI_RELEASE} -t nnidev/nni-nightly .
       echo '## Upload docker image ##'
-      docker push nnidev/nni-it-pai:latest
+      docker push nnidev/nni-nightly
     condition: eq(variables['build_docker_image'], 'true')
     displayName: Build and upload docker image
 
@@ -68,7 +61,7 @@ jobs:
           --pai_reuse false \
           --pai_host https://ne.openpai.org \
           --pai_user $(pai_user) \
-          --nni_docker_image nnidev/nni-it-pai:latest \
+          --nni_docker_image nnidev/nni-nightly \
           --pai_storage_config_name confignfs-data \
           --pai_token $(pai_token) \
           --nni_manager_nfs_mount_path /home/quzha/mnt-pai-ne/shinyang3 \

pipelines/integration-test-remote-l2l.yml

@@ -27,29 +27,17 @@ jobs:
       echo "Working directory: ${PWD}"
       echo "NNI version: ${NNI_RELEASE}"
 
-      python3 -m pip install -U --upgrade pip setuptools
+      python3 -m pip install --upgrade pip setuptools
     displayName: Prepare
 
   - script: |
       set -e
-      python3 setup.py build_ts
-      python3 setup.py bdist_wheel -p manylinux1_x86_64
-      python3 -m pip install dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl
-    displayName: Install NNI
-
-  - script: |
-      set -e
-      sudo apt-get install swig -y
-      nnictl package install --name=SMAC
-      nnictl package install --name=BOHB
-    displayName: Install extra tuners
+      python3 test/vso_tools/install_nni.py $(NNI_RELEASE) SMAC,BOHB
 
-  - script: |
-      set -e
       cd examples/tuners/customized_tuner
       python3 setup.py develop --user
-      nnictl package install .
-    displayName: Install customized tuner
+      nnictl algo register --meta meta_file.yml
+    displayName: Install NNI
 
   - task: CopyFilesOverSSH@0
     inputs:
@@ -60,6 +48,15 @@ jobs:
     displayName: Copy wheel to remote machine
     timeoutInMinutes: 10
 
+  - task: CopyFilesOverSSH@0
+    inputs:
+      sshEndpoint: $(worker)
+      contents: Dockerfile
+      targetFolder: /tmp/nnitest/$(Build.BuildId)
+      overwrite: true
+    displayName: Copy dockerfile to remote machine
+    timeoutInMinutes: 10
+
   - task: CopyFilesOverSSH@0
     inputs:
       sshEndpoint: $(worker)
@@ -73,10 +70,8 @@ jobs:
     inputs:
       sshEndpoint: $(worker)
       runOptions: commands
-      commands: |
-        python3 /tmp/nnitest/$(Build.BuildId)/test/nni_test/nnitest/remote_docker.py --mode start --name $(Build.BuildId) --image nni/nni
-        echo "##vso[task.setvariable variable=docker_port]$(cat /tmp/nnitest/$(Build.BuildId)/port)"
-    displayName: Start docker
+      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/vso_tools/start_docker.py $(NNI_RELEASE) $(Build.BuildId) $(password_in_docker)
+    displayName: Install NNI and run docker on Linux worker
 
   - script: |
       cd test
@@ -108,5 +103,6 @@ jobs:
     inputs:
       sshEndpoint: $(worker)
       runOptions: commands
-      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/nni_test/nnitest/remote_docker.py --mode stop --name $(Build.BuildId)
+      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/vso_tools/stop_docker.py $(Build.BuildId)
+    condition: always()
     displayName: Stop docker

pipelines/integration-test-remote-l2w.yml

+trigger: none
+pr: none
+schedules:
+- cron: 0 16 * * *
+  branches:
+    include: [ master ]
+
+variables:
+  worker: remote_nni-ci-gpu-04-w
+
+# variables set on VSO: (for security concern)
+#   manager_ip
+#   worker_ip
+#   worker_port
+#   worker_password
+
+jobs:
+- job: remote_linux2windows
+  pool: NNI CI REMOTE CLI
+  timeoutInMinutes: 120
+
+  steps:
+  - script: |
+      export NNI_RELEASE=999.$(date -u +%Y%m%d%H%M%S)
+      echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
+      echo "##vso[task.setvariable variable=NNI_RELEASE]${NNI_RELEASE}"
+      echo "Working directory: ${PWD}"
+      echo "NNI version: ${NNI_RELEASE}"
+      echo "Build ID: $(Build.BuildId)"
+      python3 -m pip install --upgrade pip setuptools
+      python3 setup.py clean --all
+    displayName: Prepare on Linux manager
+
+  - task: CopyFilesOverSSH@0
+    inputs:
+      sshEndpoint: $(worker)
+      targetFolder: /tmp/nnitest/$(Build.BuildId)
+      overwrite: true
+    displayName: Copy source files to Windows worker
+    timeoutInMinutes: 10
+
+  - task: SSH@0
+    inputs:
+      sshEndpoint: $(worker)
+      runOptions: commands
+      commands: |
+        conda activate l2w & python -m pip install --upgrade pip setuptools
+        conda activate l2w & python /tmp/nnitest/$(Build.BuildId)/test/vso_tools/install_nni.py $(NNI_RELEASE)
+      failOnStdErr: false
+    displayName: Install NNI on Windows worker
+
+  - script: |
+      set -e
+      python3 setup.py build_ts
+      python3 setup.py bdist_wheel -p manylinux1_x86_64
+      python3 -m pip install dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl[SMAC,BOHB]
+    displayName: Install NNI on Linux manager
+
+  - script: |
+      set -e
+      cd examples/tuners/customized_tuner
+      python3 setup.py develop --user
+      nnictl algo register --meta meta_file.yml
+    displayName: Install customized tuner
+
+  - script: |
+      set -e
+      cd test
+      python3 nni_test/nnitest/generate_ts_config.py \
+          --ts remote \
+          --remote_user AzureUser \
+          --remote_host $(worker_ip) \
+          --remote_port $(worker_port) \
+          --remote_pwd $(worker_password) \
+          --nni_manager_ip $(manager_ip)
+      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote
+    displayName: Integration test
+
+  - task: SSH@0
+    inputs:
+      sshEndpoint: $(worker)
+      runOptions: commands
+      commands: rm -rf /tmp/nnitest/$(Build.BuildId)
+    condition: always()
+    displayName: Clean up on Windows worker

pipelines/integration-test-remote-w2l.yml

+trigger: none
+pr: none
+schedules:
+- cron: 0 16 * * *
+  branches:
+    include: [ master ]
+
+variables:
+  worker: remote_nni-ci-gpu-03
+
+# variables set on VSO:
+#   manager_ip
+#   worker_ip
+#   password_in_docker
+
+jobs:
+- job: remote_windows2linux
+  pool: NNI CI WINDOWS2
+  timeoutInMinutes: 120
+
+  steps:
+  - script:
+      python test/vso_tools/generate_nni_version.py
+      python -m pip install --upgrade pip setuptools
+      python setup.py clean --all
+    displayName: Prepare on Windows manager
+
+  - task: CopyFilesOverSSH@0
+    inputs:
+      sshEndpoint: $(worker)
+      targetFolder: /tmp/nnitest/$(Build.BuildId)
+      overwrite: true
+    displayName: Copy source files to Linux worker
+    timeoutInMinutes: 10
+
+  - script: |
+      python test/vso_tools/install_nni.py $(NNI_RELEASE)
+
+      cd examples/tuners/customized_tuner
+      python setup.py develop --user
+      nnictl algo register --meta meta_file.yml
+    displayName: Install NNI on Windows manager
+
+  - task: SSH@0
+    inputs:
+      sshEndpoint: $(worker)
+      runOptions: commands
+      commands: |
+        python3 /tmp/nnitest/$(Build.BuildId)/test/vso_tools/build_wheel.py $(NNI_RELEASE)
+        python3 /tmp/nnitest/$(Build.BuildId)/test/vso_tools/start_docker.py $(NNI_RELEASE) $(Build.BuildId) $(password_in_docker)
+      failOnStdErr: false
+    displayName: Install NNI and run docker on Linux worker
+
+  - powershell: |
+      cd test
+      python nni_test/nnitest/generate_ts_config.py `
+          --ts remote `
+          --remote_reuse false `
+          --remote_user nni `
+          --remote_host $(worker_ip) `
+          --remote_port $(docker_port) `
+          --remote_pwd $(password_in_docker) `
+          --nni_manager_ip $(manager_ip)
+      Get-Content config/training_service.yml
+      python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote --exclude cifar10
+    displayName: Integration test
+
+  - task: SSH@0
+    inputs:
+      sshEndpoint: $(worker)
+      runOptions: commands
+      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/vso_tools/stop_docker.py $(Build.BuildId)
+    displayName: Stop docker

pipelines/release.yml

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+jobs:
+- job: validate_version_number
+  pool:
+    vmImage: Ubuntu 18.04
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: 3.8
+    displayName: Configure Python version
+
+  - script: |
+      export BRANCH_TAG=`git describe --tags --abbrev=0`
+      echo $BRANCH_TAG
+      if [[ $BRANCH_TAG = v$(NNI_RELEASE) && $(NNI_RELEASE) =~ ^v[0-9](.[0-9])+$ ]]; then
+        echo 'Build version match branch tag'
+      else
+        echo 'Build version does not match branch tag'
+        exit 1
+      fi
+    condition: eq( variables['build_type'], 'release' )
+    displayName: Validate release version number and branch tag
+
+  - script: |
+      echo $(NNI_RELEASE)
+      if [[ $(NNI_RELEASE) =~ ^[0-9](.[0-9])+a[0-9]$ ]]; then
+        echo 'Valid prerelease version $(NNI_RELEASE)'
+        echo `git describe --tags --abbrev=0`
+      else
+        echo 'Invalid build version $(NNI_RELEASE)'
+        exit 1
+      fi
+    condition: ne( variables['build_type'], 'rerelease' )
+    displayName: Validate prerelease version number
+
+- job: linux
+  dependsOn: validate_version_number
+  condition: succeeded()
+  pool:
+    vmImage: Ubuntu 18.04
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: 3.8
+    displayName: Configure Python version
+
+  - script: |
+      python -m pip install --upgrade pip setuptools twine
+      python test/vso_tools/build_wheel.py $(NNI_RELEASE)
+
+      if [ $(build_type) = 'release' ]
+        echo 'uploading release package to pypi...'
+        python -m twine upload -u nni -p $(pypi_password) dist/*
+      then
+      else
+        echo 'uploading prerelease package to testpypi...'
+        python -m twine upload -u nni -p $(pypi_password) --repository-url https://test.pypi.org/legacy/ dist/*
+      fi
+    displayName: Build and upload wheel
+
+  - script: |
+      if [ $(build_type) = 'release' ]
+      then
+        docker login -u msranni -p $(docker_hub_password)
+        export IMAGE_NAME=msranni/nni
+      else
+        docker login -u nnidev -p $(docker_hub_password)
+        export IMAGE_NAME=nnidev/nni-test
+      fi
+
+      echo "## Building ${IMAGE_NAME}:$(NNI_RELEASE) ##"
+      docker build --build-arg NNI_RELEASE=$(NNI_RELEASE) -t ${IMAGE_NAME} .
+      docker tag ${IMAGE_NAME} ${IMAGE_NAME}:$(NNI_RELEASE)
+      docker push ${IMAGE_NAME}
+      docker push ${IMAGE_NAME}:$(NNI_RELEASE)
+    displayName: Build and upload docker image
+
+- job: macos
+  dependsOn: validate_version_number
+  condition: succeeded()
+  pool:
+    vmImage: macOS-10.15
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: 3.8
+    displayName: Configure Python version
+
+  - script: |
+      python -m pip install --upgrade pip setuptools twine
+      python test/vso_tools/build_wheel.py $(NNI_RELEASE)
+
+      if [ $(build_type) = 'release' ]
+        echo '## uploading to pypi ##'
+        python -m twine upload -u nni -p $(pypi_password) dist/*
+      then
+      else
+        echo '## uploading to testpypi ##'
+        python -m twine upload -u nni -p $(pypi_password) --repository-url https://test.pypi.org/legacy/ dist/*
+      fi
+    displayName: Build and upload wheel
+
+- job: windows
+  dependsOn: validate_version_number
+  condition: succeeded()
+  pool:
+    vmImage: windows-2019
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: 3.8
+    displayName: Configure Python version
+
+  - powershell: |
+      python -m pip install --upgrade pip setuptools twine
+      python test/vso_tools/build_wheel.py $(NNI_RELEASE)
+
+      if($env:BUILD_TYPE -eq 'release'){
+        Write-Host '## uploading to pypi ##'
+        python -m twine upload -u nni -p $(pypi_password) dist/*
+      }
+      else{
+        Write-Host '## uploading to testpypi ##'
+        python -m twine upload -u nni -p $(pypi_password) --repository-url https://test.pypi.org/legacy/ dist/*
+      }
+    displayName: Build and upload wheel

setup.py

@@ -105,7 +105,7 @@ def _setup():
 
         packages = _find_python_packages(),
         package_data = {
-            'nni': _find_requirements_txt(),  # must do this manually due to setuptools issue #1806
+            'nni': _find_requirements_txt() + _find_default_config(),  # setuptools issue #1806
             'nni_node': _find_node_files()  # note: this does not work before building
         },
 
@@ -139,7 +139,7 @@ def _setup():
 def _find_python_packages():
     packages = []
     for dirpath, dirnames, filenames in os.walk('nni'):
-        if '/__pycache__' not in dirpath and '/.mypy_cache' not in dirpath:
+        if '/__pycache__' not in dirpath and '/.mypy_cache' not in dirpath and '/default_config' not in dirpath:
             packages.append(dirpath.replace('/', '.'))
     return sorted(packages) + ['nni_node']
 
@@ -150,9 +150,12 @@ def _find_requirements_txt():
             requirement_files.append(os.path.join(dirpath[len('nni/'):], 'requirements.txt'))
     return requirement_files
 
+def _find_default_config():
+    return ['runtime/default_config/' + name for name in os.listdir('nni/runtime/default_config')]
+
 def _find_node_files():
     if not os.path.exists('nni_node'):
-        if release and 'build_ts' not in sys.argv:
+        if release and 'build_ts' not in sys.argv and 'clean' not in sys.argv:
             sys.exit('ERROR: To build a release version, run "python setup.py build_ts" first')
         return []
     files = []
@@ -166,20 +169,6 @@ def _find_node_files():
 def _using_conda_or_virtual_environment():
     return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta'))
 
-def _copy_data_files():
-    # after installation, nni needs to find this location in nni.tools.package_utils.get_registered_algo_config_path
-    # since we can not import nni here, we need to ensure get_registered_algo_config_path use the same
-    # logic here to retrieve registered_algorithms.yml
-    if _using_conda_or_virtual_environment():
-        nni_config_dir = os.path.join(sys.prefix, 'nni')
-    elif sys.platform == 'win32':
-        nni_config_dir = os.path.join(os.getenv('APPDATA'), 'nni')
-    else:
-        nni_config_dir = os.path.expanduser('~/.config/nni')
-    if not os.path.exists(nni_config_dir):
-        os.makedirs(nni_config_dir)
-    shutil.copyfile('./deployment/registered_algorithms.yml', os.path.join(nni_config_dir, 'registered_algorithms.yml'))
-
 class BuildTs(Command):
     description = 'build TypeScript modules'
 
@@ -200,7 +189,6 @@ class Build(build):
             sys.exit('Please set environment variable "NNI_RELEASE=<release_version>"')
         if os.path.islink('nni_node/main.js'):
             sys.exit('A development build already exists. Please uninstall NNI and run "python3 setup.py clean --all".')
-        _copy_data_files()
         super().run()
 
 class Develop(develop):
@@ -226,7 +214,6 @@ class Develop(develop):
     def run(self):
         if not self.skip_ts:
             setup_ts.build(release=None)
-        _copy_data_files()
         super().run()
 
 class Clean(clean):

test/nni_test/nnitest/run_tests.py

@@ -249,7 +249,7 @@ def run(args):
             wait_for_port_available(8080, 180)
         else:
             wait_for_port_available(8080, 30)
-        print('{}Testing: {}{}'.format(GREEN, name, CLEAR))
+        print('## {}Testing: {}{} ##'.format(GREEN, name, CLEAR))
         begin_time = time.time()
 
         run_test_case(test_case_config, it_config, args)

test/scripts/nas.sh

@@ -21,7 +21,7 @@ fi
 echo "testing darts..."
 cd $EXAMPLE_DIR/darts
 python3 search.py --epochs 1 --channels 2 --layers 4
-python3 retrain.py --arc-checkpoint ./checkpoints/epoch_0.json --layers 4 --epochs 1
+python3 retrain.py --arc-checkpoint ./checkpoint.json --layers 4 --epochs 1
 
 echo "testing enas..."
 cd $EXAMPLE_DIR/enas

test/vso_tools/_common.py

+from pathlib import Path
+import os
+import subprocess
+import sys
+
+nni_root = str(Path(__file__).parents[2])
+
+def build_wheel():
+    python = sys.executable
+    version = sys.argv[1]
+    os_spec = {
+        'linux': 'manylinux1_x86_64',
+        'darwin': 'macosx_10_9_x86_64',
+        'win32': 'win_amd64'
+    }[sys.platform]
+
+    run_command(f'{python} setup.py clean --all')
+    run_command(f'{python} setup.py build_ts', NNI_RELEASE=version)
+    run_command(f'{python} setup.py bdist_wheel -p {os_spec}', NNI_RELEASE=version)
+
+    return f'dist/nni-{version}-py3-none-{os_spec}.whl'
+
+def run_command(command, **extra_environ):
+    print('# run command:', command)
+    if isinstance(command, str):
+        command = command.split()
+    subprocess.run(
+        command,
+        stderr = subprocess.STDOUT,  # azure will highlight stderr, which is annoying
+        cwd = nni_root,
+        check = True,
+        env = {**os.environ, **extra_environ}
+    )
+
+def set_variable(key, value):
+    print('# set variable:', key, value)
+    print(f'##vso[task.setvariable variable={key}]{value}')