Merge pull request #1030 from Microsoft/v0.7

V0.7 merge back to master

README.md

@@ -100,28 +100,47 @@ Targeting at openness and advancing state-of-art technology, [Microsoft Research
 We encourage researchers and students leverage these projects to accelerate the AI development and research.
 
 ## **Install & Verify**
+If you choose NNI Windows local mode and you use powershell to run script for the first time, you need to **run powershell as administrator** with this command first:
+```bash
+    Set-ExecutionPolicy -ExecutionPolicy Unrestricted
+```
 
 **Install through pip** 	
-* We support Linux and MacOS in current stage, Ubuntu 16.04 or higher, along with MacOS 10.14.1 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`.	
+* We support Linux, MacOS and Windows(local mode) in current stage, Ubuntu 16.04 or higher, MacOS 10.14.1 along with Windows 10.1809 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`.
 
+Linux and MacOS
 ```bash
     python3 -m pip install --upgrade nni
 ```
+Windows
+```bash
+    python -m pip install --upgrade nni
+```
 Note:
 
 * `--user` can be added if you want to install NNI in your home directory, which does not require any special privileges.
+* Currently NNI on Windows only support local mode. Anaconda is highly recommanded to install NNI on Windows. 
 * If there is any error like `Segmentation fault`, please refer to [FAQ](docs/en_US/FAQ.md)
 
 **Install through source code**
-* We support Linux (Ubuntu 16.04 or higher), MacOS (10.14.1) in our current stage. 
+* We support Linux (Ubuntu 16.04 or higher), MacOS (10.14.1) and Windows local mode (10.1809) in our current stage. 
+
+Linux and MacOS
 * Run the following commands in an environment that has `python >= 3.5`, `git` and `wget`.
 ```bash	
     git clone -b v0.7 https://github.com/Microsoft/nni.git
     cd nni	
     source install.sh	
 ```
-
+Windows
+* Run the following commands in an environment that has `python >=3.5`, `git` and `powershell`    
+```bash
+  git clone -b v0.7 https://github.com/Microsoft/nni.git
+  cd nni
+  powershell ./install.ps1
+```
 For the system requirements of NNI, please refer to [Install NNI](docs/en_US/Installation.md)  
+For NNI Windows local mode, please refer to [NNI Windows local mode](docs/en_US/WindowsLocalMode.md) 
 
 **Verify install**	
 
@@ -130,11 +149,16 @@ The following example is an experiment built on TensorFlow. Make sure you have *
 ```bash	
     git clone -b v0.7 https://github.com/Microsoft/nni.git
 ```
+Linux and MacOS
 * Run the mnist example.
 ```bash
     nnictl create --config nni/examples/trials/mnist/config.yml
 ```
-
+Windows
+* Run the mnist example.
+```bash
+    nnictl create --config nni/examples/trials/mnist/config_windows.yml
+```
 * Wait for the message `INFO: Successfully started experiment!` in the command line. This message indicates that your experiment has been successfully started. You can explore the experiment using the `Web UI url`.
 
 ```

deployment/deployment-pipelines.yml

@@ -172,7 +172,7 @@ jobs:
     condition: eq( variables['upload_package'], 'true')
     displayName: 'upload nni package to pypi/testpypi'
 
-- job: 'Build_upload_nni_windows'
+- job: 'Build_upload_nni_win32'
   dependsOn: version_number_validation
   condition: succeeded()
   pool:
@@ -182,35 +182,78 @@ jobs:
       Python36:
         PYTHON_VERSION: '3.6'
   steps:
-  - script: |
-      python -m pip install --upgrade pip setuptools --user
-      python -m pip install twine --user
+  - powershell: |
+      python -m pip install --upgrade pip setuptools
+      python -m pip install twine
     displayName: 'Install twine'
 
-  - script: |
-      cd deployment/pypi
-      if [ $(build_type) = 'prerelease' ]
-      then
+  - powershell: |
+      cd deployment\pypi
+      if($env:BUILD_TYPE -eq 'prerelease'){
         # NNI build scripts (powershell) uses branch tag as package version number
-        git tag $(build_version)
-        echo 'building prerelease package...'
-        powershell.exe ./install.ps1 -version_ts $True
-      else
-        echo 'building release package...'
-        powershell.exe ./install.ps1
-      fi
+        git tag $env:BUILD_VERSION
+        Write-Host 'building prerelease package...'
+        .\install.ps1 -version_os 32 -version_ts $True
+      }
+      else{
+        Write-Host 'building release package...'
+        .\install.ps1 -version_os 32 -version_ts $False
+      }
     condition: eq( variables['upload_package'], 'true')
     displayName: 'build nni bdsit_wheel'
 
-  - script: |
-      cd deployment/pypi
-      if [ $(build_type) = 'prerelease' ]
-      then
-        echo 'uploading prerelease package to testpypi...'
-        python -m twine upload -u $(testpypi_user) -p $(testpypi_pwd) --repository-url https://test.pypi.org/legacy/ dist/*
-      else
-        echo 'uploading release package to pypi...'
-        python -m twine upload -u $(pypi_user) -p $(pypi_pwd) dist/*
-      fi
+  - powershell: |
+      cd deployment\pypi
+      if($env:BUILD_TYPE -eq 'prerelease'){
+        Write-Host 'uploading prerelease package to testpypi...'
+        python -m twine upload -u $env:TESTPYPI_USER -p $env:TESTPYPI_PWD --repository-url https://test.pypi.org/legacy/ dist/*
+      }
+      else{
+        Write-Host 'uploading release package to pypi...'
+        python -m twine upload -u $env:PYPI_USER -p $env:PYPI_PWD dist/*
+      }
+    condition: eq( variables['upload_package'], 'true')
+    displayName: 'upload nni package to pypi/testpypi'
+
+- job: 'Build_upload_nni_win_amd64'
+  dependsOn: version_number_validation
+  condition: succeeded()
+  pool:
+    vmImage: 'vs2017-win2016'
+  strategy:
+    matrix:
+      Python36:
+        PYTHON_VERSION: '3.6'
+  steps:
+  - powershell: |
+      python -m pip install --upgrade pip setuptools
+      python -m pip install twine
+    displayName: 'Install twine'
+
+  - powershell: |
+      cd deployment\pypi
+      if($env:BUILD_TYPE -eq 'prerelease'){
+        # NNI build scripts (powershell) uses branch tag as package version number
+        git tag $env:BUILD_VERSION
+        Write-Host 'building prerelease package...'
+        .\install.ps1 -version_os 64 -version_ts $True
+      }
+      else{
+        Write-Host 'building release package...'
+        .\install.ps1 -version_os 64 -version_ts $False
+      }
+    condition: eq( variables['upload_package'], 'true')
+    displayName: 'build nni bdsit_wheel'
+
+  - powershell: |
+      cd deployment\pypi
+      if($env:BUILD_TYPE -eq 'prerelease'){
+        Write-Host 'uploading prerelease package to testpypi...'
+        python -m twine upload -u $env:TESTPYPI_USER -p $env:TESTPYPI_PWD --repository-url https://test.pypi.org/legacy/ dist/*
+      }
+      else{
+        Write-Host 'uploading release package to pypi...'
+        python -m twine upload -u $env:PYPI_USER -p $env:PYPI_PWD dist/*
+      }
     condition: eq( variables['upload_package'], 'true')
     displayName: 'upload nni package to pypi/testpypi'
\ No newline at end of file

deployment/pypi/README.md

@@ -47,15 +47,14 @@ This is the PyPI build and upload tool for NNI project.
   powershell
   Python >= 3.5
   Pip
-  Node.js
   Yarn
-  tar
   ```
 
 * __How to build__
  
+  parameter `version_os` is used to build for Windows 64-bit or 32-bit. 
   ```bash
-  powershell ./install.ps1
+  powershell ./install.ps1 -version_os [64/32]
   ```
 
 * __How to upload__

deployment/pypi/clean.ps1

@@ -4,4 +4,4 @@ Remove-Item $CWD\build -Recurse -Force
 Remove-Item $CWD\dist -Recurse -Force
 Remove-Item $CWD\nni -Recurse -Force
 Remove-Item $CWD\nni.egg-info -Recurse -Force
-Remove-Item $CWD\node-$OS_SPEC-x64 -Recurse -Force
\ No newline at end of file
+Remove-Item $CWD\node-$OS_SPEC -Recurse -Force
\ No newline at end of file

deployment/pypi/install.ps1

-param([bool]$version_ts=$false)
+param([int]$version_os, [bool]$version_ts=$false)
+[System.Net.ServicePointManager]::DefaultConnectionLimit = 100
 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
 $CWD = $PWD
 
 $OS_SPEC = "windows"
-$WHEEL_SPEC = "win_amd64"
+if($version_os -eq 64){
+    $OS_VERSION = 'win64'
+    $WHEEL_SPEC = 'win_amd64'
+}
+else{
+    $OS_VERSION = 'win32'
+    $WHEEL_SPEC = 'win32'
+}
 
 $TIME_STAMP = date -u "+%y%m%d%H%M"
 $NNI_VERSION_VALUE = git describe --tags --abbrev=0
@@ -17,18 +25,21 @@ if($version_ts){
 
 $NNI_VERSION_TEMPLATE = "999.0.0-developing"
 
-python -m pip install --user --upgrade setuptools wheel
+python -m pip install --upgrade setuptools wheel
 
-$nodeUrl = "https://aka.ms/nni/nodejs-download/win64"
-$NNI_NODE_ZIP = "$CWD\node-$OS_SPEC-x64.zip"
-$NNI_NODE_FOLDER = "$CWD\node-$OS_SPEC-x64"
+$nodeUrl = "https://aka.ms/nni/nodejs-download/" + $OS_VERSION
+$NNI_NODE_ZIP = "$CWD\node-$OS_SPEC.zip"
+$NNI_NODE_FOLDER = "$CWD\node-$OS_SPEC"
+$unzipNodeDir = "node-v*"
 (New-Object Net.WebClient).DownloadFile($nodeUrl, $NNI_NODE_ZIP)
 if(Test-Path $NNI_NODE_FOLDER){
     Remove-Item $NNI_NODE_FOLDER -Recurse -Force
 }
-New-Item $NNI_NODE_FOLDER -ItemType Directory
-cmd /c tar -xf $NNI_NODE_ZIP -C $NNI_NODE_FOLDER --strip-components 1
+Expand-Archive $NNI_NODE_ZIP -DestinationPath $CWD
+$unzipNodeDir = Get-ChildItem "$CWD\$unzipNodeDir"
+Rename-Item $unzipNodeDir $NNI_NODE_FOLDER
 
+$env:PATH = $NNI_NODE_FOLDER+';'+$env:PATH
 cd $CWD\..\..\src\nni_manager
 yarn
 yarn build

deployment/pypi/setup.py

@@ -34,7 +34,7 @@ else:
 
 data_files = [('bin', ['node-{}-x64/bin/node'.format(os_type.lower())])]
 if os_type == 'Windows':
-    data_files = [('.\Scripts', ['node-{}-x64/node.exe'.format(os_type.lower())])]
+    data_files = [('.\Scripts', ['node-{}/node.exe'.format(os_type.lower())])]
 
 for (dirpath, dirnames, filenames) in walk('./nni'):
     files = [path.normpath(path.join(dirpath, filename)) for filename in filenames]

docs/en_US/Builtin_Tuner.md

@@ -132,6 +132,8 @@ tuner:
 
 > Builtin Tuner Name: **SMAC**
 
+**Please note that SMAC doesn't support running on windows currently. The specific reason can be referred to this [github issue](https://github.com/automl/SMAC3/issues/483).**
+
 **Installation**
 
 SMAC need to be installed by following command before first use.

docs/en_US/Installation.md

 # Installation of NNI
 
-Currently we support installation on Linux, Mac and Windows.
+Currently we support installation on Linux, Mac and Windows(local mode).
 
 ## **Installation on Linux & Mac**
 
@@ -25,10 +25,14 @@ Currently we support installation on Linux, Mac and Windows.
   You can also install NNI in a docker image. Please follow the instructions [here](https://github.com/Microsoft/nni/tree/master/deployment/docker/README.md) to build NNI docker image. The NNI docker image can also be retrieved from Docker Hub through the command `docker pull msranni/nni:latest`.
 
 ## **Installation on Windows** 
-
+  When you use powershell to run script for the first time, you need **run powershell as administrator** with this command:
+  ```bash
+  Set-ExecutionPolicy -ExecutionPolicy Unrestricted
+  ```
+  Anaconda is highly recommanded.
 * __Install NNI through pip__
 
-  Prerequisite: `python >= 3.5`
+  Prerequisite: `python(64-bit) >= 3.5`
   ```bash
   python -m pip install --upgrade nni
   ```
@@ -36,13 +40,9 @@ Currently we support installation on Linux, Mac and Windows.
 * __Install NNI through source code__
 
   Prerequisite: `python >=3.5`, `git`, `powershell`    
-  When you use powershell to run script for the first time, you need run powershell as Administrator with this command:
+  you can install nni as administrator or current user as follows:
   ```bash
-  Set-ExecutionPolicy -ExecutionPolicy Unrestricted
-  ```
-  Then you can install nni as administrator or current user as follows:
-  ```bash
-  git clone https://github.com/Microsoft/nni.git
+  git clone -b v0.7 https://github.com/Microsoft/nni.git
   cd nni
   powershell ./install.ps1
   ```
@@ -73,7 +73,7 @@ Below are the minimum system requirements for NNI on macOS. Due to potential pro
 |**Internet**|Boardband internet connection|
 |**Resolution**|1024 x 768 minimum display resolution|
 
-Below are the minimum system requirements for NNI on Windows. Due to potential programming changes, the minimum system requirements for NNI may change over time.
+Below are the minimum system requirements for NNI on Windows, Windows 10.1809 is well tested and recommend. Due to potential programming changes, the minimum system requirements for NNI may change over time.
 
 ||Minimum Requirements|Recommended Specifications|
 |---|---|---|

docs/en_US/QuickStart.md

@@ -2,15 +2,18 @@
 
 ## Installation
 
-We support Linux and MacOS in current stage, Ubuntu 16.04 or higher and MacOS 10.14.1 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`.
-
+We support Linux MacOS and Windows(local mode) in current stage, Ubuntu 16.04 or higher, MacOS 10.14.1 and Windows 10.1809 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`.
+#### Linux and MacOS
 ```bash
     python3 -m pip install --upgrade nni
 ```
-
+#### Windows
+```bash
+    python -m pip install --upgrade nni
+```
 Note:
 
-* `--user` can be added if you want to install NNI in your home directory, which does not require any special privileges.
+* For Linux and MacOS `--user` can be added if you want to install NNI in your home directory, which does not require any special privileges.
 * If there is any error like `Segmentation fault`, please refer to [FAQ](FAQ.md)
 * For the `system requirements` of NNI, please refer to [Install NNI](Installation.md)
 
@@ -124,16 +127,27 @@ trial:
   codeDir: .
   gpuNum: 0
 ```
+Note:
+* **For Windows, you need to change trial command `python3` to `python`**
 
 *Implemented code directory: [config.yml](https://github.com/Microsoft/nni/tree/master/examples/trials/mnist/config.yml)*
 
 All the codes above are already prepared and stored in [examples/trials/mnist/](https://github.com/Microsoft/nni/tree/master/examples/trials/mnist).
 
+If you choose Windows local mode and use powershell to run script for the first time, you need run powershell as administrator with this command
+```bash
+    Set-ExecutionPolicy -ExecutionPolicy Unrestricted
+```
 When these things are done, **run the config.yml file from your command line to start the experiment**.
 
 ```bash
     nnictl create --config nni/examples/trials/mnist/config.yml
 ```
+If you use windows local mode and forget to change the trial command `python3` to `python` in config.yml, **then run the config_windows.yml file from your command line to start the experiment**.
+
+```bash
+    nnictl create --config nni/examples/trials/mnist/config_windows.yml
+```
 
 Note: **nnictl** is a command line tool, which can be used to control experiments, such as start/stop/resume an experiment, start/stop NNIBoard, etc. Click [here](NNICTLDOC.md) for more usage of `nnictl`
 

docs/en_US/RELEASE.md

 # ChangeLog
 
+## Release 0.7 - 4/29/2018
+
+### Major Features
+* [Support NNI on Windows](./WindowsLocalMode.md)
+    * NNI running on windows for local mode
+* [New advisor: BOHB](./bohbAdvisor.md)
+    * Support a new advisor BOHB, which is a robust and efficient hyperparameter tuning algorithm, combines the advantages of Bayesian optimization and Hyperband
+* [Support import and export experiment data through nnictl](./NNICTLDOC.md#experiment)
+    * Generate analysis results report after the experiment execution
+    * Support import data to tuner and advisor for tuning
+* [Designated gpu devices for NNI trial jobs](./ExperimentConfig.md#localConfig)
+    * Specify GPU devices for NNI trial jobs by gpuIndices configuration, if gpuIndices is set in experiment configuration file, only the specified GPU devices are used for NNI trial jobs.
+* Web Portal enhancement
+    * Decimal format of metrics other than default on the Web UI
+    * Hints in WebUI about Multi-phase
+    * Enable copy/paste for hyperparameters as python dict
+    * Enable early stopped trials data for tuners.
+* NNICTL provide better error message
+    * nnictl provide more meaningful error message for yaml file format error
+
+### Bug fix
+* Unable to kill all python threads after nnictl stop in async dispatcher mode
+* nnictl --version does not work with make dev-instal
+* All trail jobs status stays on 'waiting' for long time on PAI platform
+
+## Release 0.6 - 4/2/2019
+### Major Features
+* [Version checking](https://github.com/Microsoft/nni/blob/master/docs/en_US/PAIMode.md#version-check)
+	* check whether the version is consistent between nniManager and trialKeeper
+* [Report final metrics for early stop job](https://github.com/Microsoft/nni/issues/776)
+	* If includeIntermediateResults is true, the last intermediate result of the trial that is early stopped by assessor is sent to tuner as final result. The default value of includeIntermediateResults is false.
+* [Separate Tuner/Assessor](https://github.com/Microsoft/nni/issues/841)
+	* Adds two pipes to separate message receiving channels for tuner and assessor.
+* Make log collection feature configurable
+* Add intermediate result graph for all trials
+
+### Bug fix
+* [Add shmMB config key for PAI](https://github.com/Microsoft/nni/issues/842)
+* Fix the bug that doesn't show any result if metrics is dict
+* Fix the number calculation issue for float types in hyperband
+* Fix a bug in the search space conversion in SMAC tuner
+* Fix the WebUI issue when parsing experiment.json with illegal format
+* Fix cold start issue in Metis Tuner
+
 ## Release 0.5.2 - 3/4/2019
 ### Improvements
 * Curve fitting assessor performance improvement.

docs/en_US/WindowsLocalMode.md

+# Windows Local Mode (experimental feature)
+Currently we only support local mode on Windows. Windows 10.1809 is well tested and recommended.
+
+## **Installation on Windows**
+
+  **Anaconda python(64-bit) is highly recommended.**  
+
+When you use powershell to run script for the first time, you need **run powershell as administrator** with this command:
+```bash
+Set-ExecutionPolicy -ExecutionPolicy Unrestricted
+```
+
+* __Install NNI through pip__
+
+  Prerequisite: `python(64-bit) >= 3.5`
+  ```bash
+  python -m pip install --upgrade nni
+  ```
+
+* __Install NNI through source code__
+
+  Prerequisite: `python >=3.5`, `git`, `powershell`
+  ```bash
+  git clone -b v0.7 https://github.com/Microsoft/nni.git
+  cd nni
+  powershell ./install.ps1
+  ```
+
+When these things are done, run the **config_windows.yml** file from your command line to start the experiment.
+
+```bash
+    nnictl create --config nni/examples/trials/mnist/config_windows.yml
+```
+For other examples you need to change trial command `python3` into `python` in each example yaml.
+
+## **Frequent met errors and answers**
+
+### simplejson failed when installing nni
+Make sure C++ 14.0 compiler installed.
+>builging 'simplejson._speedups' extension error: [WinError 3] The system cannot find the path specified
+
+### Fail to run powershell when install nni from source
+If you run powershell script for the first time and did not set the execution policies for executing the script, you will meet this error below. Try to run powershell as administrator with this command first:
+```bash
+Set-ExecutionPolicy -ExecutionPolicy Unrestricted
+```
+>...cannot be loaded because running scripts is disabled on this system.
+
+### Trial failed with missing DLL in cmd or powershell
+This error caused by missing LIBIFCOREMD.DLL and LIBMMD.DLL and fail to install scipy. Anaconda python is highly recommended. If you use official python, make sure you have one of `Visual Studio`, `MATLAB`, `MKL` and `Intel Distribution for Python` installed on Windows before running nni. If not, try to install one of the softwares above or change to use Anaconda python(64-bit).
+>ImportError: DLL load failed
+
+### Trial failed on webUI
+Please check the trial log file stderr for more details. If there is no such file and nni is installed through pip, then you need to run powershell as administrator with this command first:
+```bash
+Set-ExecutionPolicy -ExecutionPolicy Unrestricted
+```
+If there is a stderr file, please check out. Two possible cases are as follows:
+* forget to change the trial command `python3` into `python` in each experiment yaml.
+* forget to install experiment dependencies such as tensorflow, keras and so on.
+
+### Support tuner on Windows
+* SMAC is not supported
+* BOHB is supported, make sure C++ 14.0 compiler and dependencies installed successfully.
+
+Note:
+
+* If there is any error like `Segmentation fault`, please refer to [FAQ](FAQ.md)

examples/trials/mnist-cascading-search-space/mnist.py

@@ -145,7 +145,7 @@ def parse_init_json(data):
         if value == 'Empty':
             params[key] = ['Empty']
         else:
-            params[key] = [value[0], value[1]['_value'], value[1]['_value']]
+            params[key] = [value[0], value[1], value[1]]
     return params
 
 

examples/trials/mnist-cascading-search-space/sample.json

-{"layer2": "Empty", "layer8": ["Conv", {"_index": 0, "_value": 2}], "layer3": ["Avg_pool", {"_index": 2,     "_value": 5}], "layer0": ["Max_pool", {"_index": 2, "_value": 5}], "layer1": ["Conv", {"_index": 0, "_value": 2}], "layer6": ["Max_pool",     {"_index": 1, "_value": 3}], "layer7": ["Max_pool", {"_index": 2, "_value": 5}], "layer9": ["Conv", {"_index": 0, "_value": 2}], "layer4":     ["Avg_pool", {"_index": 1, "_value": 3}], "layer5": ["Avg_pool", {"_index": 2, "_value": 5}]}
+{
+    "layer2": "Empty", 
+    "layer8": ["Conv",  2], 
+    "layer3": ["Avg_pool", 5], 
+    "layer0": ["Max_pool", 5], 
+    "layer1": ["Conv", 2], 
+    "layer6": ["Max_pool", 3], 
+    "layer7": ["Max_pool", 5], 
+    "layer9": ["Conv", 2], 
+    "layer4": ["Avg_pool", 3], 
+    "layer5": ["Avg_pool", 5]
+}

examples/trials/mnist/config_windows.yml

+authorName: default
+experimentName: example_mnist
+trialConcurrency: 1
+maxExecDuration: 1h
+maxTrialNum: 10
+#choice: local, remote, pai
+trainingServicePlatform: local
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+trial:
+  command: python mnist.py
+  codeDir: .
+  gpuNum: 0

install.ps1

@@ -3,8 +3,14 @@
 $install_node = $true
 $install_yarn = $true
 
+if([Environment]::Is64BitOperatingSystem){
+    $OS_VERSION = 'win64'
+}
+else{
+    $OS_VERSION = 'win32'
+}
 # nodejs
-$nodeUrl = "https://aka.ms/nni/nodejs-download/win64"
+$nodeUrl = "https://aka.ms/nni/nodejs-download/" + $OS_VERSION
 $yarnUrl = "https://yarnpkg.com/latest.tar.gz"
 $unzipNodeDir = "node-v*"
 $unzipYarnDir = "yarn-v*"

src/nni_manager/training_service/remote_machine/gpuScheduler.ts

@@ -134,6 +134,13 @@ export class GPUScheduler {
                     rmMeta.gpuReservation = new Map<number, string>();
                 }
                 const designatedGpuIndices: Set<number> | undefined = parseGpuIndices(rmMeta.gpuIndices);
+                if (designatedGpuIndices !== undefined) {
+                    for (const gpuIndex of designatedGpuIndices) {
+                        if (gpuIndex >= rmMeta.gpuSummary.gpuCount) {
+                            throw new Error(`Specified GPU index not found: ${gpuIndex}`);
+                        }
+                    }
+                }
                 this.log.debug(`designated gpu indices: ${designatedGpuIndices}`);
                 rmMeta.gpuSummary.gpuInfos.forEach((gpuInfo: GPUInfo) => {
                     // if the GPU has active process, OR be reserved by a job,
@@ -179,5 +186,4 @@ export class GPUScheduler {
             }
         };
     }
-
 }

src/sdk/pynni/nni/bohb_advisor/bohb_advisor.py

@@ -595,6 +595,9 @@ class BOHB(MsgDispatcherBase):
             _params = trial_info["parameter"]
             assert "value" in trial_info
             _value = trial_info['value']
+            if not _value:
+                logger.info("Useless trial data, value is %s, skip this trial data." %_value)
+                continue
             budget_exist_flag = False
             barely_params = dict()
             for keys in _params:

src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py

@@ -164,6 +164,11 @@ class GridSearchTuner(Tuner):
             _completed_num += 1
             assert "parameter" in trial_info
             _params = trial_info["parameter"]
+            assert "value" in trial_info
+            _value = trial_info['value']
+            if not _value:
+                logger.info("Useless trial data, value is %s, skip this trial data." %_value)
+                continue
             _params_tuple = convert_dict2tuple(_params)
             self.supplement_data[_params_tuple] = True
         logger.info("Successfully import data to grid search tuner.")

src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py

@@ -139,19 +139,50 @@ def json2vals(in_x, vals, out_y, name=ROOT):
         for i, temp in enumerate(in_x):
             json2vals(temp, vals[i], out_y, name + '[%d]' % i)
 
+def _add_index(in_x, parameter):
+    """
+    change parameters in NNI format to parameters in hyperopt format(This function also support nested dict.).
+    For example, receive parameters like:
+        {'dropout_rate': 0.8, 'conv_size': 3, 'hidden_size': 512}
+    Will change to format in hyperopt, like:
+        {'dropout_rate': 0.8, 'conv_size': {'_index': 1, '_value': 3}, 'hidden_size': {'_index': 1, '_value': 512}}
+    """
+    if TYPE not in in_x: # if at the top level
+        out_y = dict()
+        for key, value in parameter.items():
+            out_y[key] = _add_index(in_x[key], value)
+        return out_y
+    elif isinstance(in_x, dict):
+        value_type = in_x[TYPE]
+        value_format = in_x[VALUE]
+        if value_type == "choice":
+            choice_name = parameter[0] if isinstance(parameter, list) else parameter
+            for pos, item in enumerate(value_format): # here value_format is a list
+                if isinstance(item, list): # this format is ["choice_key", format_dict]
+                    choice_key = item[0]
+                    choice_value_format = item[1]
+                    if choice_key == choice_name:
+                        return {INDEX: pos, VALUE: [choice_name, _add_index(choice_value_format, parameter[1])]}
+                elif choice_name == item:
+                    return {INDEX: pos, VALUE: item}
+        else:
+            return parameter
 
 def _split_index(params):
     """
     Delete index infromation from params
     """
-    result = {}
+    if isinstance(params, list):
+        return [params[0], _split_index(params[1])]
+    elif isinstance(params, dict):
+        if INDEX in params.keys():
+            return _split_index(params[VALUE])
+        result = dict()
         for key in params:
-        if isinstance(params[key], dict):
-            value = params[key][VALUE]
-        else:
-            value = params[key]
-        result[key] = value
+            result[key] = _split_index(params[key])
         return result
+    else:
+        return params
 
 
 class HyperoptTuner(Tuner):
@@ -373,8 +404,11 @@ class HyperoptTuner(Tuner):
             _params = trial_info["parameter"]
             assert "value" in trial_info
             _value = trial_info['value']
+            if not _value:
+                logger.info("Useless trial data, value is %s, skip this trial data." %_value)
+                continue
             self.supplement_data_num += 1
             _parameter_id = '_'.join(["ImportData", str(self.supplement_data_num)])
-            self.total_data[_parameter_id] = _params
+            self.total_data[_parameter_id] = _add_index(in_x=self.json, parameter=_params)
             self.receive_trial_result(parameter_id=_parameter_id, parameters=_params, value=_value)
         logger.info("Successfully import data to TPE/Anneal tuner.")

src/sdk/pynni/nni/metis_tuner/metis_tuner.py

@@ -65,7 +65,7 @@ class MetisTuner(Tuner):
     https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/
     """
 
-    def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=True,
+    def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=False,
                  selection_num_starting_points=600, cold_start_num=10, exploration_probability=0.9):
         """
         Parameters
@@ -417,6 +417,9 @@ class MetisTuner(Tuner):
             _params = trial_info["parameter"]
             assert "value" in trial_info
             _value = trial_info['value']
+            if not _value:
+                logger.info("Useless trial data, value is %s, skip this trial data." %_value)
+                continue
             self.supplement_data_num += 1
             _parameter_id = '_'.join(["ImportData", str(self.supplement_data_num)])
             self.total_data.append(_params)