Merge branch 'master' of https://github.com/microsoft/nni into dev-retiarii

README.md

@@ -5,7 +5,7 @@
 -----------
 
 [![MIT licensed](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE)
-[![Build Status](https://msrasrg.visualstudio.com/NNIOpenSource/_apis/build/status/integration-test-local?branchName=master)](https://msrasrg.visualstudio.com/NNIOpenSource/_build/latest?definitionId=17&branchName=master)
+[![Build Status](https://msrasrg.visualstudio.com/NNIOpenSource/_apis/build/status/full%20test%20-%20linux?branchName=master)](https://msrasrg.visualstudio.com/NNIOpenSource/_build/latest?definitionId=62&branchName=master)
 [![Issues](https://img.shields.io/github/issues-raw/Microsoft/nni.svg)](https://github.com/Microsoft/nni/issues?q=is%3Aissue+is%3Aopen)
 [![Bugs](https://img.shields.io/github/issues/Microsoft/nni/bug.svg)](https://github.com/Microsoft/nni/issues?q=is%3Aissue+is%3Aopen+label%3Abug)
 [![Pull Requests](https://img.shields.io/github/issues-pr-raw/Microsoft/nni.svg)](https://github.com/Microsoft/nni/pulls?q=is%3Apr+is%3Aopen)

azure-pipelines.yml

@@ -34,8 +34,7 @@ jobs:
       python3 -m pip install gym onnx peewee thop --user
       python3 -m pip install sphinx==1.8.3 sphinx-argparse==0.2.5 sphinx-markdown-tables==0.0.9 sphinx-rtd-theme==0.4.2 sphinxcontrib-websupport==1.1.0 recommonmark==0.5.0 nbsphinx --user
       sudo apt-get install swig -y
-      nnictl package install --name=SMAC
-      nnictl package install --name=BOHB
+      python3 -m pip install -e .[SMAC,BOHB]
     displayName: 'Install dependencies'
   - script: |
       cd test
@@ -73,8 +72,7 @@ jobs:
       python3 -m pip install keras==2.1.6 --user
       python3 -m pip install gym onnx peewee --user
       sudo apt-get install swig -y
-      nnictl package install --name=SMAC
-      nnictl package install --name=BOHB
+      python3 -m pip install -e .[SMAC,BOHB]
     displayName: 'Install dependencies'
   - script: |
       set -e

deployment/registered_algorithms.yml

+advisors:
+- builtinName: Hyperband
+  classArgsValidator: nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.HyperbandClassArgsValidator
+  className: nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.Hyperband
+  source: nni
+- builtinName: BOHB
+  classArgsValidator: nni.algorithms.hpo.bohb_advisor.bohb_advisor.BOHBClassArgsValidator
+  className: nni.algorithms.hpo.bohb_advisor.bohb_advisor.BOHB
+  source: nni
+assessors:
+- builtinName: Medianstop
+  classArgsValidator: nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopClassArgsValidator
+  className: nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopAssessor
+  source: nni
+- builtinName: Curvefitting
+  classArgsValidator: nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingClassArgsValidator
+  className: nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingAssessor
+  source: nni
+tuners:
+- builtinName: PPOTuner
+  classArgsValidator: nni.algorithms.hpo.ppo_tuner.ppo_tuner.PPOClassArgsValidator
+  className: nni.algorithms.hpo.ppo_tuner.ppo_tuner.PPOTuner
+  source: nni
+- builtinName: SMAC
+  classArgsValidator: nni.algorithms.hpo.smac_tuner.smac_tuner.SMACClassArgsValidator
+  className: nni.algorithms.hpo.smac_tuner.smac_tuner.SMACTuner
+  source: nni
+- builtinName: TPE
+  classArgs:
+    algorithm_name: tpe
+  classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator
+  className: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner
+  source: nni
+- acceptClassArgs: false
+  builtinName: Random
+  classArgs:
+    algorithm_name: random_search
+  classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator
+  className: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner
+  source: nni
+- builtinName: Anneal
+  classArgs:
+    algorithm_name: anneal
+  classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator
+  className: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner
+  source: nni
+- builtinName: Evolution
+  classArgsValidator: nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionClassArgsValidator
+  className: nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner
+  source: nni
+- acceptClassArgs: false
+  builtinName: BatchTuner
+  className: nni.algorithms.hpo.batch_tuner.batch_tuner.BatchTuner
+  source: nni
+- acceptClassArgs: false
+  builtinName: GridSearch
+  className: nni.algorithms.hpo.gridsearch_tuner.gridsearch_tuner.GridSearchTuner
+  source: nni
+- builtinName: NetworkMorphism
+  classArgsValidator: nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismClassArgsValidator
+  className: nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner
+  source: nni
+- builtinName: MetisTuner
+  classArgsValidator: nni.algorithms.hpo.metis_tuner.metis_tuner.MetisClassArgsValidator
+  className: nni.algorithms.hpo.metis_tuner.metis_tuner.MetisTuner
+  source: nni
+- builtinName: GPTuner
+  classArgsValidator: nni.algorithms.hpo.gp_tuner.gp_tuner.GPClassArgsValidator
+  className: nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner
+  source: nni
+- builtinName: PBTTuner
+  classArgsValidator: nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTClassArgsValidator
+  className: nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTTuner
+  source: nni
+- builtinName: RegularizedEvolutionTuner
+  classArgsValidator: nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.EvolutionClassArgsValidator
+  className: nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.RegularizedEvolutionTuner
+  source: nni

docs/archive_en_US/NAS/Benchmarks.md

 # NAS Benchmarks
 
+[TOC]
+
 ```eval_rst
 ..  toctree::
     :hidden:
@@ -9,7 +11,7 @@
 
 ## Introduction
 
-To imporve the reproducibility of NAS algorithms as well as reducing computing resource requirements, researchers proposed a series of NAS benchmarks such as [NAS-Bench-101](https://arxiv.org/abs/1902.09635), [NAS-Bench-201](https://arxiv.org/abs/2001.00326), [NDS](https://arxiv.org/abs/1905.13214), etc. NNI provides a query interface for users to acquire these benchmarks. Within just a few lines of code, researcher are able to evaluate their NAS algorithms easily and fairly by utilizing these benchmarks.
+To imporve the reproducibility of NAS algorithms as well as reducing computing resource requirements, researchers proposed a series of NAS benchmarks such as [NAS-Bench-101](https://arxiv.org/abs/1902.09635), [NAS-Bench-201](https://arxiv.org/abs/2001.00326), [NDS](https://arxiv.org/abs/1905.13214), [NLP](https://arxiv.org/abs/2006.07116), etc. NNI provides a query interface for users to acquire these benchmarks. Within just a few lines of code, researcher are able to evaluate their NAS algorithms easily and fairly by utilizing these benchmarks.
 
 ## Prerequisites
 
@@ -27,7 +29,7 @@ cd nni/examples/nas/benchmarks
 ```
 Replace `${NNI_VERSION}` with a released version name or branch name, e.g., `v1.9`.
 
-2. Install dependencies via `pip3 install -r xxx.requirements.txt`. `xxx` can be `nasbench101`, `nasbench201` or `nds`.
+2. Install dependencies via `pip3 install -r xxx.requirements.txt`. `xxx` can be `nasbench101`, `nasbench201`, `nds` or `nlp`.
 3. Generate the database via `./xxx.sh`. The directory that stores the benchmark file can be configured with `NASBENCHMARK_DIR` environment variable, which defaults to `~/.nni/nasbenchmark`. Note that the NAS-Bench-201 dataset will be downloaded from a google drive.
 
 Please make sure there is at least 10GB free disk space and note that the conversion process can take up to hours to complete.
@@ -109,7 +111,7 @@ _On Network Design Spaces for Visual Recognition_ released trial statistics of o
 
 Instead of storing results obtained with different configurations in separate files, we dump them into one single database to enable comparison in multiple dimensions. Specifically, we use `model_family` to distinguish model types, `model_spec` for all hyper-parameters needed to build this model, `cell_spec` for detailed information on operators and connections if it is a NAS cell, `generator` to denote the sampling policy through which this configuration is generated. Refer to API documentation for details.
 
-## Available Operators
+### Available Operators
 
 Here is a list of available operators used in NDS.
 
@@ -158,3 +160,22 @@ Here is a list of available operators used in NDS.
 
 .. autoclass:: nni.nas.benchmarks.nds.NdsIntermediateStats
 ```
+
+## NLP
+
+[Paper link](https://arxiv.org/abs/2006.07116)     [Open-source](https://github.com/fmsnew/nas-bench-nlp-release)
+
+The paper "NAS-Bench-NLP: Neural Architecture Search Benchmark for Natural Language Processing"  have provided search space of recurrent neural networks on the text datasets and trained 14k architectures within it, and have conducted both intrinsic and extrinsic evaluation of the trained models using datasets for semantic relatedness and language understanding evaluation. There are 2 datasets - PTB and wikitext-2. In the end, the precomputed results(ptb_single_run + ptb_multi_run + wikitext-2) can be utilized. 
+
+### API Documentation
+
+```eval_rst
+.. autofunction:: nni.nas.benchmarks.nlp.query_nlp_trial_stats
+
+.. autoclass:: nni.nas.benchmarks.nlp.NlpTrialConfig
+
+.. autoclass:: nni.nas.benchmarks.nlp.NlpTrialStats
+
+.. autoclass:: nni.nas.benchmarks.nlp.NlpIntermediateStats
+```
+

docs/archive_en_US/TrainingService/AdaptDLMode.md

@@ -52,6 +52,7 @@ trialConcurrency: 2
 maxTrialNum: 2
 
 trial:
+  namespace: <k8s_namespace>
   adaptive: false # optional.
   image: <image_tag>
   imagePullSecrets:  # optional
@@ -66,7 +67,7 @@ trial:
     path: /
     containerMountPath: /nfs
   checkpoint: # optional
-    storageClass: microk8s-hostpath
+    storageClass: dfs
     storageSize: 1Gi
 ```
 
@@ -79,18 +80,22 @@ IP address of the machine with NNI manager (NNICTL) that launches NNI experiment
 * **logCollection**: *Recommended* to set as `http`. It will collect the trial logs on cluster back to your machine via http.
 * **tuner**: It supports the Tuun tuner and all NNI built-in tuners (only except for the checkpoint feature of the NNI PBT tuners).
 * **trial**: It defines the specs of an `adl` trial.
-    * **adaptive**: (*Optional*) Boolean for AdaptDL trainer. While `true`, it the job is preemptible and adaptive.
-    * **image**: Docker image for the trial
-    * **imagePullSecret**: (*Optional*) If you are using a private registry,
-    you need to provide the secret to successfully pull the image.
-    * **codeDir**: the working directory of the container. `.` means the default working directory defined by the image.
-    * **command**: the bash command to start the trial
-    * **gpuNum**: the number of GPUs requested for this trial. It must be non-negative integer.
-    * **cpuNum**: (*Optional*) the number of CPUs requested for this trial.  It must be non-negative integer.
-    * **memorySize**: (*Optional*) the size of memory requested for this trial. It must follow the Kubernetes
-    [default format](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory).
-    * **nfs**: (*Optional*) mounting external storage. For more information about using NFS please check the below paragraph.
-    * **checkpoint** (*Optional*) [storage settings](https://kubernetes.io/docs/concepts/storage/storage-classes/) for AdaptDL internal checkpoints. You can keep it optional if you are not dev users.
+  * **namespace**: (*Optional*) Kubernetes namespace to launch the trials. Default to `default` namespace.
+  * **adaptive**: (*Optional*) Boolean for AdaptDL trainer. While `true`, it the job is preemptible and adaptive.
+  * **image**: Docker image for the trial
+  * **imagePullSecret**: (*Optional*) If you are using a private registry,
+  you need to provide the secret to successfully pull the image.
+  * **codeDir**: the working directory of the container. `.` means the default working directory defined by the image.
+  * **command**: the bash command to start the trial
+  * **gpuNum**: the number of GPUs requested for this trial. It must be non-negative integer.
+  * **cpuNum**: (*Optional*) the number of CPUs requested for this trial.  It must be non-negative integer.
+  * **memorySize**: (*Optional*) the size of memory requested for this trial. It must follow the Kubernetes
+  [default format](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory).
+  * **nfs**: (*Optional*) mounting external storage. For more information about using NFS please check the below paragraph.
+  * **checkpoint**: (*Optional*) storage settings for model checkpoints.
+    * **storageClass**: check [Kubernetes storage documentation](https://kubernetes.io/docs/concepts/storage/storage-classes/) for how to use the appropriate `storageClass`.
+    * **storageSize**: this value should be large enough to fit your model's checkpoints, or it could cause disk quota exceeded error.
+
 
 ### NFS Storage
 

docs/archive_en_US/Tuner/BohbAdvisor.md

@@ -46,7 +46,7 @@ The sampling procedure (using Multidimensional KDE to guide selection) is summar
 BOHB advisor requires the [ConfigSpace](https://github.com/automl/ConfigSpace) package. ConfigSpace can be installed using the following command.
 
 ```bash
-nnictl package install --name=BOHB
+pip install nni[BOHB]
 ```
 
 To use BOHB, you should add the following spec in your experiment's YAML config file:

docs/archive_en_US/Tuner/BuiltinTuner.md

@@ -12,7 +12,7 @@ Currently, we support the following algorithms:
 |[__Random Search__](#Random)|In Random Search for Hyper-Parameter Optimization show that Random Search might be surprisingly simple and effective. We suggest that we could use Random Search as the baseline when we have no knowledge about the prior distribution of hyper-parameters. [Reference Paper](http://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf)|
 |[__Anneal__](#Anneal)|This simple annealing algorithm begins by sampling from the prior, but tends over time to sample from points closer and closer to the best ones observed. This algorithm is a simple variation on the random search that leverages smoothness in the response surface. The annealing rate is not adaptive.|
 |[__Naïve Evolution__](#Evolution)|Naïve Evolution comes from Large-Scale Evolution of Image Classifiers. It randomly initializes a population-based on search space. For each generation, it chooses better ones and does some mutation (e.g., change a hyperparameter, add/remove one layer) on them to get the next generation. Naïve Evolution requires many trials to work, but it's very simple and easy to expand new features. [Reference paper](https://arxiv.org/pdf/1703.01041.pdf)|
-|[__SMAC__](#SMAC)|SMAC is based on Sequential Model-Based Optimization (SMBO). It adapts the most prominent previously used model class (Gaussian stochastic process models) and introduces the model class of random forests to SMBO, in order to handle categorical parameters. The SMAC supported by NNI is a wrapper on the SMAC3 GitHub repo. Notice, SMAC needs to be installed by `nnictl package` command. [Reference Paper,](https://www.cs.ubc.ca/~hutter/papers/10-TR-SMAC.pdf) [GitHub Repo](https://github.com/automl/SMAC3)|
+|[__SMAC__](#SMAC)|SMAC is based on Sequential Model-Based Optimization (SMBO). It adapts the most prominent previously used model class (Gaussian stochastic process models) and introduces the model class of random forests to SMBO, in order to handle categorical parameters. The SMAC supported by NNI is a wrapper on the SMAC3 GitHub repo. Notice, SMAC needs to be installed by `pip install nni[SMAC]` command. [Reference Paper,](https://www.cs.ubc.ca/~hutter/papers/10-TR-SMAC.pdf) [GitHub Repo](https://github.com/automl/SMAC3)|
 |[__Batch tuner__](#Batch)|Batch tuner allows users to simply provide several configurations (i.e., choices of hyper-parameters) for their trial code. After finishing all the configurations, the experiment is done. Batch tuner only supports the type choice in search space spec.|
 |[__Grid Search__](#GridSearch)|Grid Search performs an exhaustive searching through a manually specified subset of the hyperparameter space defined in the searchspace file. Note that the only acceptable types of search space are choice, quniform, randint. |
 |[__Hyperband__](#Hyperband)|Hyperband tries to use limited resources to explore as many configurations as possible and returns the most promising ones as a final result. The basic idea is to generate many configurations and run them for a small number of trials. The half least-promising configurations are thrown out, the remaining are further trained along with a selection of new configurations. The size of these populations is sensitive to resource constraints (e.g. allotted search time). [Reference Paper](https://arxiv.org/pdf/1603.06560.pdf)|
@@ -27,7 +27,9 @@ Currently, we support the following algorithms:
 
 Using a built-in tuner provided by the NNI SDK requires one to declare the  **builtinTunerName** and **classArgs** in the `config.yml` file. In this part, we will introduce each tuner along with information about usage and suggested scenarios, classArg requirements, and an example configuration.
 
-Note: Please follow the format when you write your `config.yml` file. Some built-in tuners need to be installed using `nnictl package`, like SMAC.
+Note: Please follow the format when you write your `config.yml` file. Some built-in tuners have 
+dependencies need to be installed using `pip install nni[<tuner>]`, like SMAC's dependencies can
+be installed using `pip install nni[SMAC]`.
 
 <a name="TPE"></a>
 
@@ -144,10 +146,10 @@ tuner:
 
 **Installation**
 
-SMAC needs to be installed by following command before the first usage. As a reminder, `swig` is required for SMAC: for Ubuntu `swig` can be installed with `apt`.
+SMAC has dependencies need to be installed by following command before the first usage. As a reminder, `swig` is required for SMAC: for Ubuntu `swig` can be installed with `apt`.
 
 ```bash
-nnictl package install --name=SMAC
+pip install nni[SMAC]
 ```
 
 **Suggested scenario**
@@ -340,7 +342,7 @@ tuner:
 BOHB advisor requires [ConfigSpace](https://github.com/automl/ConfigSpace) package. ConfigSpace can be installed using the following command.
 
 ```bash
-nnictl package install --name=BOHB
+pip install nni[BOHB]
 ```
 
 **Suggested scenario**

docs/archive_en_US/Tuner/InstallCustomizedTuner.md

-# How to install customized tuner as a builtin tuner
+# How to register a customized tuner as a builtin tuner
 
-You can following below steps to install a customized tuner in `nni/examples/tuners/customized_tuner` as a builtin tuner.
+You can following below steps to register a customized tuner in `nni/examples/tuners/customized_tuner` as a builtin tuner.
 
-## Prepare installation source and install package
+## Install the customized tuner package into python environment
 
-There are 2 options to install this customized tuner:
+There are 2 options to install the package into python environment:
 
 ### Option 1: install from directory
 
-Step 1: From `nni/examples/tuners/customized_tuner` directory, run:
+From `nni/examples/tuners/customized_tuner` directory, run:
 
 `python setup.py develop`
 
 This command will build the `nni/examples/tuners/customized_tuner` directory as a pip installation source.
 
-Step 2: Run command:
-
-`nnictl package install ./`
 
 ### Option 2: install from whl file
 
@@ -28,16 +25,22 @@ This command build a whl file which is a pip installation source.
 
 Step 2: Run command:
 
-`nnictl package install dist/demo_tuner-0.1-py3-none-any.whl`
+`pip install dist/demo_tuner-0.1-py3-none-any.whl`
+
+## Register the customized tuner as builtin tuner:
+
+Run following command:
+
+`nnictl algo register --meta meta_file.yml`
 
-## Check the installed package
+## Check the registered builtin algorithms
 
-Then run command `nnictl package list`, you should be able to see that demotuner is installed:
+Then run command `nnictl algo list`, you should be able to see that demotuner is installed:
 ```
 +-----------------+------------+-----------+--------=-------------+------------------------------------------+
-|      Name       |    Type    | Installed |      Class Name      |               Module Name                |
+|      Name       |    Type    |  source   |      Class Name      |               Module Name                |
 +-----------------+------------+-----------+----------------------+------------------------------------------+
-| demotuner       | tuners     | Yes       | DemoTuner            | demo_tuner                               |
+| demotuner       | tuners     |  user     | DemoTuner            | demo_tuner                               |
 +-----------------+------------+-----------+----------------------+------------------------------------------+
 ```
 

docs/archive_en_US/Tutorial/InstallCustomizedAlgos.md

-**How to install customized algorithms as builtin tuners, assessors and advisors**
+**How to register customized algorithms as builtin tuners, assessors and advisors**
 ===
 
 ## Overview
 
-NNI provides a lot of [builtin tuners](../Tuner/BuiltinTuner.md), [advisors](../Tuner/HyperbandAdvisor.md) and [assessors](../Assessor/BuiltinAssessor.md) can be used directly for Hyper Parameter Optimization, and some extra algorithms can be installed via `nnictl package install --name <name>` after NNI is installed. You can check these extra algorithms via `nnictl package list` command.
+NNI provides a lot of [builtin tuners](../Tuner/BuiltinTuner.md), [advisors](../Tuner/HyperbandAdvisor.md) and [assessors](../Assessor/BuiltinAssessor.md) can be used directly for Hyper Parameter Optimization, and some extra algorithms can be registered via `nnictl algo register --meta <path_to_meta_file>` after NNI is installed. You can check builtin algorithms via `nnictl algo list` command.
 
 NNI also provides the ability to build your own customized tuners, advisors and assessors. To use the customized algorithm, users can simply follow the spec in experiment config file to properly reference the algorithm, which has been illustrated in the tutorials of [customized tuners](../Tuner/CustomizeTuner.md)/[advisors](../Tuner/CustomizeAdvisor.md)/[assessors](../Assessor/CustomizeAssessor.md).
 
@@ -13,8 +13,8 @@ tuner:
   builtinTunerName: mytuner
 ```
 
-## Install customized algorithms as builtin tuners, assessors and advisors
-You can follow below steps to build a customized tuner/assessor/advisor, and install it into NNI as builtin algorithm.
+## Register customized algorithms as builtin tuners, assessors and advisors
+You can follow below steps to build a customized tuner/assessor/advisor, and register it into NNI as builtin algorithm.
 
 ### 1. Create a customized tuner/assessor/advisor
 Reference following instructions to create:
@@ -48,56 +48,43 @@ class MedianstopClassArgsValidator(ClassArgsValidator):
 ```
 The validator will be invoked before experiment is started to check whether the classArgs fields are valid for your customized algorithms.
 
-### 3. Prepare package installation source
-In order to be installed as builtin tuners, assessors and advisors, the customized algorithms need to be packaged as installable source which can be recognized by `pip` command, under the hood nni calls `pip` command to install the package.
-Besides being a common pip source, the package needs to provide meta information in the `classifiers` field.
-Format of classifiers field is a following:
-```
-NNI Package :: <type> :: <builtin name> :: <full class name of tuner> :: <full class name of class args validator>
-```
-* `type`: type of algorithms, could be one of `tuner`, `assessor`, `advisor`
-* `builtin name`: builtin name used in experiment configuration file
-* `full class name of tuner`: tuner class name, including its module name, for example: `demo_tuner.DemoTuner`
-* `full class name of class args validator`: class args validator class name, including its module name, for example: `demo_tuner.MyClassArgsValidator`
-
-Following is an example of classfiers in package's `setup.py`:
+### 3. Install your customized algorithms into python environment
+Firstly, the customized algorithms need to be prepared as a python package. Then you can install the package into python environment via:
+* Run command `python setup.py develop` from the package directory, this command will install the package in development mode, this is recommended if your algorithm is under development.
+* Run command `python setup.py bdist_wheel` from the package directory, this command build a whl file which is a pip installation source. Then run `pip install <wheel file>` to install it.
 
-```python
-    classifiers = [
-        'Programming Language :: Python :: 3',
-        'License :: OSI Approved :: MIT License',
-        'Operating System :: ',
-        'NNI Package :: tuner :: demotuner :: demo_tuner.DemoTuner :: demo_tuner.MyClassArgsValidator'
-    ],
-```
 
-Once you have the meta info in `setup.py`, you can build your pip installation source via:
-* Run command `python setup.py develop` from the package directory, this command will build the directory as a pip installation source.
-* Run command `python setup.py bdist_wheel` from the package directory, this command build a whl file which is a pip installation source.
+### 4. Prepare meta file
 
-NNI will look for the classifier starts with `NNI Package` to retrieve the package meta information while the package being installed with `nnictl package install <source>` command.
+Create a yaml file with following keys as meta file:
+* `algoType`: type of algorithms, could be one of `tuner`, `assessor`, `advisor`
+* `builtinName`: builtin name used in experiment configuration file
+* `className`: tuner class name, including its module name, for example: `demo_tuner.DemoTuner`
+* `classArgsValidator`: class args validator class name, including its module name, for example: `demo_tuner.MyClassArgsValidator`
 
-Reference [customized tuner example](../Tuner/InstallCustomizedTuner.md) for a full example.
+Following is an example of the yaml file:
 
-### 4. Install customized algorithms package into NNI
-
-If your installation source is prepared as a directory with `python setup.py develop`, you can install the package by following command:
-
-`nnictl package install <installation source directory>`
+```yaml
+algoType: tuner
+builtinName: demotuner
+className: demo_tuner.DemoTuner
+classArgsValidator: demo_tuner.MyClassArgsValidator
 
-For example:
+```
 
-`nnictl package install nni/examples/tuners/customized_tuner/`
+### 5. Register customized algorithms into NNI
+Run following command to register the customized algorithms as builtin algorithms in NNI:
 
-If your installation source is prepared as a whl file with `python setup.py bdist_wheel`, you can install the package by following command:
+```bash
+nnictl algo register --meta <path_to_meta_file>
+```
+The `<path_to_meta_file>` is the path to the yaml file your created in above section.
 
-`nnictl package install <whl file path>`
 
-For example:
+Reference [customized tuner example](../Tuner/InstallCustomizedTuner.md) for a full example.
 
-`nnictl package install nni/examples/tuners/customized_tuner/dist/demo_tuner-0.1-py3-none-any.whl`
 
-## 5. Use the installed builtin algorithms in experiment
+## 6. Use the installed builtin algorithms in experiment
 Once your customized algorithms is installed, you can use it in experiment configuration file the same way as other builtin tuners/assessors/advisors, for example:
 
 ```yaml
@@ -109,56 +96,42 @@ tuner:
 ```
 
 
-## Manage packages using `nnictl package`
+## Manage builtin algorithms using `nnictl algo`
 
-### List installed packages
+### List builtin algorithms
 
-Run following command to list the installed packages:
+Run following command to list the registered builtin algorithms:
 
-```
-nnictl package list
-+-----------------+------------+-----------+--------=-------------+------------------------------------------+
-|      Name       |    Type    | Installed |      Class Name      |               Module Name                |
-+-----------------+------------+-----------+----------------------+------------------------------------------+
-| demotuner       | tuners     | Yes       | DemoTuner            | demo_tuner                               |
-| SMAC            | tuners     | No        | SMACTuner            | nni.smac_tuner.smac_tuner                |
-| PPOTuner        | tuners     | No        | PPOTuner             | nni.ppo_tuner.ppo_tuner                  |
-| BOHB            | advisors   | Yes       | BOHB                 | nni.bohb_advisor.bohb_advisor            |
-+-----------------+------------+-----------+----------------------+------------------------------------------+
-```
-
-Run following command to list all packages, including the builtin packages can not be uninstalled.
-
-```
-nnictl package list --all
+```bash
+nnictl algo list
 +-----------------+------------+-----------+--------=-------------+------------------------------------------+
-|      Name       |    Type    | Installed |      Class Name      |               Module Name                |
+|      Name       |    Type    | Source    |      Class Name      |               Module Name                |
 +-----------------+------------+-----------+----------------------+------------------------------------------+
-| TPE             | tuners     | Yes       | HyperoptTuner        | nni.hyperopt_tuner.hyperopt_tuner        |
-| Random          | tuners     | Yes       | HyperoptTuner        | nni.hyperopt_tuner.hyperopt_tuner        |
-| Anneal          | tuners     | Yes       | HyperoptTuner        | nni.hyperopt_tuner.hyperopt_tuner        |
-| Evolution       | tuners     | Yes       | EvolutionTuner       | nni.evolution_tuner.evolution_tuner      |
-| BatchTuner      | tuners     | Yes       | BatchTuner           | nni.batch_tuner.batch_tuner              |
-| GridSearch      | tuners     | Yes       | GridSearchTuner      | nni.gridsearch_tuner.gridsearch_tuner    |
-| NetworkMorphism | tuners     | Yes       | NetworkMorphismTuner | nni.networkmorphism_tuner.networkmo...   |
-| MetisTuner      | tuners     | Yes       | MetisTuner           | nni.metis_tuner.metis_tuner              |
-| GPTuner         | tuners     | Yes       | GPTuner              | nni.gp_tuner.gp_tuner                    |
-| PBTTuner        | tuners     | Yes       | PBTTuner             | nni.pbt_tuner.pbt_tuner                  |
-| SMAC            | tuners     | No        | SMACTuner            | nni.smac_tuner.smac_tuner                |
-| PPOTuner        | tuners     | No        | PPOTuner             | nni.ppo_tuner.ppo_tuner                  |
-| Medianstop      | assessors  | Yes       | MedianstopAssessor   | nni.medianstop_assessor.medianstop_...   |
-| Curvefitting    | assessors  | Yes       | CurvefittingAssessor | nni.curvefitting_assessor.curvefitt...   |
-| Hyperband       | advisors   | Yes       | Hyperband            | nni.hyperband_advisor.hyperband_adv...   |
-| BOHB            | advisors   | Yes       | BOHB                 | nni.bohb_advisor.bohb_advisor            |
+| TPE             | tuners     | nni       | HyperoptTuner        | nni.hyperopt_tuner.hyperopt_tuner        |
+| Random          | tuners     | nni       | HyperoptTuner        | nni.hyperopt_tuner.hyperopt_tuner        |
+| Anneal          | tuners     | nni       | HyperoptTuner        | nni.hyperopt_tuner.hyperopt_tuner        |
+| Evolution       | tuners     | nni       | EvolutionTuner       | nni.evolution_tuner.evolution_tuner      |
+| BatchTuner      | tuners     | nni       | BatchTuner           | nni.batch_tuner.batch_tuner              |
+| GridSearch      | tuners     | nni       | GridSearchTuner      | nni.gridsearch_tuner.gridsearch_tuner    |
+| NetworkMorphism | tuners     | nni       | NetworkMorphismTuner | nni.networkmorphism_tuner.networkmo...   |
+| MetisTuner      | tuners     | nni       | MetisTuner           | nni.metis_tuner.metis_tuner              |
+| GPTuner         | tuners     | nni       | GPTuner              | nni.gp_tuner.gp_tuner                    |
+| PBTTuner        | tuners     | nni       | PBTTuner             | nni.pbt_tuner.pbt_tuner                  |
+| SMAC            | tuners     | nni       | SMACTuner            | nni.smac_tuner.smac_tuner                |
+| PPOTuner        | tuners     | nni       | PPOTuner             | nni.ppo_tuner.ppo_tuner                  |
+| Medianstop      | assessors  | nni       | MedianstopAssessor   | nni.medianstop_assessor.medianstop_...   |
+| Curvefitting    | assessors  | nni       | CurvefittingAssessor | nni.curvefitting_assessor.curvefitt...   |
+| Hyperband       | advisors   | nni       | Hyperband            | nni.hyperband_advisor.hyperband_adv...   |
+| BOHB            | advisors   | nni       | BOHB                 | nni.bohb_advisor.bohb_advisor            |
 +-----------------+------------+-----------+----------------------+------------------------------------------+
 ```
 
-### Uninstall package
+### Unregister builtin algorithms
 
 Run following command to uninstall an installed package:
 
-`nnictl package uninstall <builtin name>`
+`nnictl algo unregister <builtin name>`
 
 For example:
 
-`nnictl package uninstall demotuner`
+`nnictl algo unregister demotuner`

docs/archive_en_US/Tutorial/InstallationWin.md

@@ -123,7 +123,7 @@ If there is a stderr file, please check it. Two possible cases are:
 
 ### Fail to use BOHB on Windows
 
-Make sure a C++ 14.0 compiler is installed when trying to run `nnictl package install --name=BOHB` to install the dependencies.
+Make sure a C++ 14.0 compiler is installed when trying to run `pip install nni[BOHB]` to install the dependencies.
 
 ### Not supported tuner on Windows
 

docs/archive_en_US/Tutorial/Nnictl.md

@@ -21,7 +21,7 @@ nnictl support commands:
 * [nnictl log](#log)
 * [nnictl webui](#webui)
 * [nnictl tensorboard](#tensorboard)
-* [nnictl package](#package)
+* [nnictl algo](#algo)
 * [nnictl ss_gen](#ss_gen)
 * [nnictl --version](#version)
 
@@ -769,112 +769,89 @@ Debug mode will disable version check function in Trialkeeper.
   |------|------|------ |------|
   |id|  False| |ID of the experiment you want to set|
 
-<a name="package"></a>
+<a name="algo"></a>
 
-### Manage package
+### Manage builtin algorithms
 
-* __nnictl package install__
+* __nnictl algo register__
   * Description
 
-    Install a package (customized algorithms or nni provided algorithms) as builtin tuner/assessor/advisor.
+    Register customized algorithms as builtin tuner/assessor/advisor.
 
   * Usage
 
     ```bash
-    nnictl package install --name <package name>
+    nnictl algo register --meta <path_to_meta_file>
     ```
+    `<path_to_meta_file>` is the path to the meta data file in yml format, which has following keys:
+    * `algoType`: type of algorithms, could be one of `tuner`, `assessor`, `advisor`
+    * `builtinName`: builtin name used in experiment configuration file
+    * `className`: tuner class name, including its module name, for example: `demo_tuner.DemoTuner`
+    * `classArgsValidator`: class args validator class name, including its module name, for example: `demo_tuner.MyClassArgsValidator`
 
-    The available `<package name>` can be checked via `nnictl package list` command.
-
-    or
-
-    ```bash
-    nnictl package install <installation source>
-    ```
-
-    Reference [Install customized algorithms](InstallCustomizedAlgos.md) to prepare the installation source.
 
   * Example
 
-    > Install SMAC tuner
+    > Install a customized tuner in nni examples
 
     ```bash
-    nnictl package install --name SMAC
+    cd nni/examples/tuners/customized_tuner
+    python3 setup.py develop
+    nnictl algo register --meta meta_file.yml
     ```
 
-    > Install a customized tuner
-
-    ```bash
-    nnictl package install nni/examples/tuners/customized_tuner/dist/demo_tuner-0.1-py3-none-any.whl
-    ```
 
-
-* __nnictl package show__
+* __nnictl algo show__
 
   * Description
 
-    Show the detailed information of specified packages.
+    Show the detailed information of specified registered algorithms.
 
   * Usage
 
     ```bash
-    nnictl package show <package name>
+    nnictl algo show <builtinName>
     ```
 
   * Example
 
     ```bash
-    nnictl package show SMAC
+    nnictl algo show SMAC
     ```
 
-* __nnictl package list__
+* __nnictl algo list__
   * Description
 
-    List the installed/all packages.
+    List the registered builtin algorithms
 
   * Usage
 
     ```bash
-    nnictl package list [OPTIONS]
+    nnictl algo list
     ```
 
-  * Options
-
-  |Name, shorthand|Required|Default|Description|
-  |------|------|------ |------|
-  |--all|  False| |List all packages|
-
   * Example
 
-    > List installed packages
-
-    ```bash
-    nnictl package list
-    ```
-
-    > List all packages
-
     ```bash
-    nnictl package list --all
+    nnictl algo list
     ```
 
-* __nnictl package uninstall__
+* __nnictl algo unregister__
 
   * Description
 
-    Uninstall a package.
+    Unregister a registered customized builtin algorithms. The NNI provided builtin algorithms can not be unregistered.
 
   * Usage
 
     ```bash
-    nnictl package uninstall <package name>
+    nnictl algo unregister <builtinName>
     ```
 
   * Example
-    Uninstall SMAC package
 
     ```bash
-    nnictl package uninstall SMAC
+    nnictl algo unregister demotuner
     ```
 
 

docs/archive_en_US/Tutorial/SearchSpaceSpec.md

@@ -89,4 +89,4 @@ Known Limitations:
 
 * Note that for nested search space:
 
-    * Only Random Search/TPE/Anneal/Evolution tuner supports nested search space
+    * Only Random Search/TPE/Anneal/Evolution/Grid Search tuner supports nested search space

docs/en_US/TrainingService/AdaptDLMode.rst

@@ -72,7 +72,7 @@ Here is a template configuration specification to use AdaptDL as a training serv
        path: /
        containerMountPath: /nfs
      checkpoint: # optional
-       storageClass: microk8s-hostpath
+       storageClass: dfs
        storageSize: 1Gi
 
 Those configs not mentioned below, are following the
@@ -86,6 +86,7 @@ Those configs not mentioned below, are following the
 * **tuner**\ : It supports the Tuun tuner and all NNI built-in tuners (only except for the checkpoint feature of the NNI PBT tuners).
 * **trial**\ : It defines the specs of an ``adl`` trial.
 
+  * **namespace**\: (*Optional*\ ) Kubernetes namespace to launch the trials. Default to ``default`` namespace.
   * **adaptive**\ : (*Optional*\ ) Boolean for AdaptDL trainer. While ``true``\ , it the job is preemptible and adaptive.
   * **image**\ : Docker image for the trial
   * **imagePullSecret**\ : (*Optional*\ ) If you are using a private registry,
@@ -97,7 +98,10 @@ Those configs not mentioned below, are following the
   * **memorySize**\ : (*Optional*\ ) the size of memory requested for this trial. It must follow the Kubernetes
     `default format <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory>`__.
   * **nfs**\ : (*Optional*\ ) mounting external storage. For more information about using NFS please check the below paragraph.
-  * **checkpoint** (*Optional*\ ) `storage settings <https://kubernetes.io/docs/concepts/storage/storage-classes/>`__ for AdaptDL internal checkpoints. You can keep it optional if you are not dev users.
+  * **checkpoint** (*Optional*\ ) storage settings for model checkpoints.
+
+    * **storageClass**\ : check `Kubernetes storage documentation <https://kubernetes.io/docs/concepts/storage/storage-classes/>`__ for how to use the appropriate ``storageClass``.
+    * **storageSize**\ : this value should be large enough to fit your model's checkpoints, or it could cause "disk quota exceeded" error.
 
 NFS Storage
 ^^^^^^^^^^^

docs/en_US/Tutorial/SearchSpaceSpec.rst

@@ -255,4 +255,4 @@ Known Limitations:
   Note that for nested search space:
 
 
-  * Only Random Search/TPE/Anneal/Evolution tuner supports nested search space
+  * Only Random Search/TPE/Anneal/Evolution/Grid Search tuner supports nested search space

examples/nas/benchmarks/.gitignore

@@ -2,3 +2,4 @@ nasbench_full.tfrecord
 a.pth
 data.zip
 nds_data
+nlp_data
\ No newline at end of file

examples/nas/benchmarks/nlp.requirements.txt

+peewee

examples/nas/benchmarks/nlp.sh

+#!/bin/bash
+set -e
+
+if [ -z "${NASBENCHMARK_DIR}" ]; then
+    NASBENCHMARK_DIR=~/.nni/nasbenchmark
+fi
+
+
+mkdir -p nlp_data
+cd nlp_data
+echo "Downloading NLP[1/3] wikitext2_data.zip..."
+if [ -f "wikitext2_data.zip" ]; then
+    echo "wikitext2_data.zip found. Skip download."
+else
+    wget -O wikitext2_data.zip https://github.com/fmsnew/nas-bench-nlp-release/blob/master/train_logs_wikitext-2/logs.zip?raw=true
+fi
+echo "Downloading NLP[2/3] ptb_single_run_data.zip..."
+if [ -f "ptb_single_run_data.zip" ]; then
+    echo "ptb_single_run_data.zip found. Skip download."
+else
+    wget -O ptb_single_run_data.zip https://github.com/fmsnew/nas-bench-nlp-release/blob/master/train_logs_single_run/logs.zip?raw=true
+fi
+echo "Downloading NLP[3/3] ptb_multi_runs_data.zip..."
+if [ -f "ptb_multi_runs_data.zip" ]; then
+    echo "ptb_multi_runs_data.zip found. Skip download."
+else
+    wget -O ptb_multi_runs_data.zip https://github.com/fmsnew/nas-bench-nlp-release/blob/master/train_logs_multi_runs/logs.zip?raw=true
+fi
+echo "### there exits duplicate log_files in ptb_single_run_data.zip and ptb_multi_run_data.zip, you can ignore all or replace all ###"
+unzip -q wikitext2_data.zip
+unzip -q ptb_single_run_data.zip
+unzip -q ptb_multi_runs_data.zip
+cd ..
+
+echo "Generating database..."
+rm -f ${NASBENCHMARK_DIR}/nlp.db ${NASBENCHMARK_DIR}/nlp.db-journal
+mkdir -p ${NASBENCHMARK_DIR}
+python3 -m nni.nas.benchmarks.nlp.db_gen nlp_data
+rm -rf nlp_data