Merge pull request #242 from microsoft/master

merge master

docs/zh_CN/conf.py

@@ -28,7 +28,7 @@ author = 'Microsoft'
 # The short X.Y version
 version = ''
 # The full version, including alpha/beta/rc tags
-release = 'v1.4'
+release = 'v1.5'
 
 # -- General configuration ---------------------------------------------------
 

docs/zh_CN/nas.rst

@@ -26,5 +26,7 @@
     SPOS <NAS/SPOS>
     CDARTS <NAS/CDARTS>
     ProxylessNAS <NAS/Proxylessnas>
+    TextNAS <NAS/TextNAS>
     自定义 NAS 算法 <NAS/Advanced>
+    NAS 可视化 <NAS/Visualization>
     API 参考 <NAS/NasReference>

docs/zh_CN/training_services.rst

@@ -9,4 +9,4 @@ NNI 支持的训练平台介绍
     OpenPAI Yarn 模式<./TrainingService/PaiYarnMode>
     Kubeflow<./TrainingService/KubeflowMode>
     FrameworkController<./TrainingService/FrameworkControllerMode>
-    OpenPAI<./TrainingService/DLTSMode>
+    DLTS<./TrainingService/DLTSMode>

examples/nas/textnas/README_zh_CN.md

@@ -42,4 +42,8 @@ python search.py
 
 ## 重新训练
 
-待完成
+```
+sh run_retrain.sh
+```
+
+默认情况下，脚本会重新训练 SST-2 数据集上作者所提供的网络结构。

examples/trials/sklearn/regression/main.py

@@ -24,9 +24,9 @@ import numpy as np
 from sklearn.metrics import r2_score
 from sklearn.preprocessing import StandardScaler
 from sklearn.linear_model import LinearRegression
-from sklearn.svm import SVR
-from sklearn.neighbors import KNeighborsRegressor
-from sklearn.tree import DecisionTreeRegressor
+from sklearn.linear_model import Ridge
+from sklearn.linear_model import Lars
+from sklearn.linear_model import ARDRegression
 
 LOG = logging.getLogger('sklearn_regression')
 
@@ -55,24 +55,18 @@ def get_model(PARAMS):
     '''Get model according to parameters'''
     model_dict = {
         'LinearRegression': LinearRegression(),
-        'SVR': SVR(),
-        'KNeighborsRegressor': KNeighborsRegressor(),
-        'DecisionTreeRegressor': DecisionTreeRegressor()
+        'Ridge': Ridge(),
+        'Lars': Lars(),
+        'ARDRegression': ARDRegression()
+
     }
     if not model_dict.get(PARAMS['model_name']):
         LOG.exception('Not supported model!')
         exit(1)
 
     model = model_dict[PARAMS['model_name']]
+    model.normalize = bool(PARAMS['normalize'])
 
-    try:
-        if PARAMS['model_name'] == 'SVR':
-            model.kernel = PARAMS['svr_kernel']
-        elif PARAMS['model_name'] == 'KNeighborsRegressor':
-            model.weights = PARAMS['knr_weights']
-    except Exception as exception:
-        LOG.exception(exception)
-        raise
     return model
 
 def run(X_train, X_test, y_train, y_test, model):

examples/trials/sklearn/regression/search_space.json

 {
-    "model_name":{"_type":"choice","_value":["LinearRegression", "SVR", "KNeighborsRegressor", "DecisionTreeRegressor"]},
-    "svr_kernel": {"_type":"choice","_value":["linear", "poly", "rbf"]},
-    "knr_weights": {"_type":"choice","_value":["uniform", "distance"]}
+    "model_name":{"_type":"choice","_value":["LinearRegression", "Lars", "Ridge", "ARDRegression"]},
+    "normalize": {"_type":"choice","_value":["true", "false"]}
 }
\ No newline at end of file

src/webui/src/components/overview/Progress.tsx

 import * as React from 'react';
-import { Stack, Callout, Link, IconButton, FontWeights, mergeStyleSets, getId, getTheme, StackItem } from 'office-ui-fabric-react';
+import {
+    Stack, Callout, Link, IconButton, FontWeights, mergeStyleSets,
+    getId, getTheme, StackItem, TooltipHost
+} from 'office-ui-fabric-react';
 import axios from 'axios';
-import { MANAGER_IP } from '../../static/const';
+import { MANAGER_IP, CONCURRENCYTOOLTIP } from '../../static/const';
 import { EXPERIMENT, TRIALS } from '../../static/datamodel';
 import { convertTime } from '../../static/function';
 import ConcurrencyInput from './NumInput';
 import ProgressBar from './ProgressItem';
 import LogDrawer from '../Modals/LogDrawer';
 import MessageInfo from '../Modals/MessageInfo';
+import { infoIcon } from "../Buttons/Icon";
 import '../../static/style/progress.scss';
 import '../../static/style/probar.scss';
 interface ProgressProps {
@@ -97,7 +101,7 @@ class Progressed extends React.Component<ProgressProps, ProgressState> {
     hideSucceedInfo = (): void => {
         this.setState(() => ({ isShowSucceedInfo: false }));
     }
-    
+
     /**
      * info: message content
      * typeInfo: message type: success | error...
@@ -264,7 +268,9 @@ class Progressed extends React.Component<ProgressProps, ProgressState> {
                     </span>
                     <span style={itemStyles}>
                         {/* modify concurrency */}
-                        <p>Concurrency</p>
+                        <TooltipHost content={CONCURRENCYTOOLTIP}>
+                        <p className="cursor">Concurrency<span className="progress-info">{infoIcon}</span></p>
+                        </TooltipHost>
                         <ConcurrencyInput value={this.props.concurrency} updateValue={this.editTrialConcurrency} />
                     </span>
                     <span style={itemStyles} className="basic colorOfbasic"></span>

src/webui/src/components/overview/SuccessTable.tsx

@@ -4,6 +4,7 @@ import DefaultMetric from '../public-child/DefaultMetric';
 import Details from './Details';
 import { convertDuration } from '../../static/function';
 import { TRIALS } from '../../static/datamodel';
+import { DETAILTABS } from '../stateless-component/NNItabs';
 import '../../static/style/succTable.scss';
 import '../../static/style/openRow.scss';
 
@@ -55,6 +56,13 @@ class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState>
         return items.slice(0).sort((a: T, b: T) => ((isSortedDescending ? a[key] < b[key] : a[key] > b[key]) ? 1 : -1));
     }
 
+    tooltipStr = (
+        <div>
+            <p>The experiment is running, please wait for the final metric patiently.</p>
+            <div className="link">You could also find status of trial job with <span>{DETAILTABS}</span> button.</div>
+        </div>
+    );
+
     columns = [
         {
             name: 'Trial No.',
@@ -125,9 +133,10 @@ class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState>
 
     render(): React.ReactNode {
         const { columns, source } = this.state;
+        const isNoneData = (source.length === 0) ? true : false;
+        
         return (
             <div id="succTable">
-                {/* TODO: [style] lineHeight question */}
                 <DetailsList
                     columns={columns}
                     items={source}
@@ -135,7 +144,9 @@ class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState>
                     compact={true}
                     onRenderRow={this.onRenderRow}
                     selectionMode={0} // close selector function
+                    className="succTable"
                 />
+                {isNoneData && <div className="succTable-tooltip">{this.tooltipStr}</div>}
             </div>
         );
     }

src/webui/src/static/const.ts

@@ -35,8 +35,10 @@ const COLUMN = ['Trial No.', 'ID', 'Duration', 'Status', 'Default', OPERATION];
 // all choice column !dictory final
 const COLUMNPro = ['Trial No.', 'ID', 'Start Time', 'End Time', 'Duration', 'Status',
 'Intermediate result', 'Default', OPERATION];
+const  CONCURRENCYTOOLTIP = 'Trial concurrency is the number of trials running concurrently.';
+
 export {
     MANAGER_IP, DOWNLOAD_IP, trialJobStatus, COLUMNPro, WEBUIDOC,
     CONTROLTYPE, MONACO, COLUMN, DRAWEROPTION, OPERATION,
-    METRIC_GROUP_UPDATE_THRESHOLD, METRIC_GROUP_UPDATE_SIZE,
+    METRIC_GROUP_UPDATE_THRESHOLD, METRIC_GROUP_UPDATE_SIZE, CONCURRENCYTOOLTIP
 };

src/webui/src/static/style/progress.scss

@@ -48,6 +48,12 @@
       margin-left: 6px;
     }
   }
+
+  &-info{
+    margin-left: 4px;
+    position: relative;
+    top: 2px;
+  }
 }
 
 /* basic experiment message style */
@@ -101,3 +107,7 @@
     border: 2px solid #e6e6e6;
     border-radius: 0 12px 12px 0 !important;
 }
+
+.cursor, .cursor:hover{
+  cursor: pointer;
+}
\ No newline at end of file

src/webui/src/static/style/succTable.scss

 #succTable{
     height: 404px;
     overflow-y: scroll;
+    position: relative;
+    .succTable-tooltip{
+        position: absolute;
+        top: 40%;
+        left: 17%;
+        .link{
+            margin-left: 15px;
+            a{
+                font-weight: 500;
+                color: blue;
+            }
+        }
+    }
 }
\ No newline at end of file

src/webui/src/static/style/tableStatus.css

@@ -15,7 +15,7 @@
     color:	#EB0716;
 }
 .WAITING{
-    color:	#008B8B;
+    color: #FDC401;
 }
 .EARLY_STOPPED{
     color: 	#FFA500;

test/config/naive_test/README_zh_CN.md

@@ -5,7 +5,6 @@
 * 如果没有问题，最终会打印绿色的 `PASS`。
 
 ## 详细说明
-
 * 这是测试 Trial 和 Tuner、Assessor 之间通信的测试用例。
 * Trial 会收到整数 `x` 作为参数，并返回 `x`, `x²`, `x³`, ... , `x¹⁰` 作为指标。
 * Tuner 会简单的生成自然数序列，并将收到的指标输出到 `tuner_result.txt`。
@@ -14,7 +13,6 @@
 * 当 Experiment 结束时，也表示用例成功执行，可以在 nni_manager.log 文件中找到 `Experiment done`。
 
 ## 问题
-
 * 使用了私有 API 来检测是否 Tuner 和 Assessor 成功结束。
 * RESTful 服务的输出未测试。
 * 远程计算机训练平台没有被测试。
\ No newline at end of file

test/nni_test/nnitest/run_tests.py

@@ -70,7 +70,7 @@ def run_test_case(test_case_config, it_config, args):
 
     try:
         launch_test(new_config_file, args.ts, test_case_config)
-        invoke_validator(test_case_config, args.nni_source_dir)
+        invoke_validator(test_case_config, args.nni_source_dir, args.ts)
     finally:
         stop_command = get_command(test_case_config, 'stopCommand')
         print('Stop command:', stop_command, flush=True)
@@ -80,7 +80,7 @@ def run_test_case(test_case_config, it_config, args):
         if os.path.exists(new_config_file):
             os.remove(new_config_file)
 
-def invoke_validator(test_case_config, nni_source_dir):
+def invoke_validator(test_case_config, nni_source_dir, training_service):
     validator_config = test_case_config.get('validator')
     if validator_config is None or validator_config.get('class') is None:
         return
@@ -88,7 +88,13 @@ def invoke_validator(test_case_config, nni_source_dir):
     validator = validators.__dict__[validator_config.get('class')]()
     kwargs = validator_config.get('kwargs', {})
     print('kwargs:', kwargs)
-    validator(REST_ENDPOINT, get_experiment_dir(EXPERIMENT_URL), nni_source_dir, **kwargs)
+    experiment_id = get_experiment_id(EXPERIMENT_URL)
+    try:
+        validator(REST_ENDPOINT, get_experiment_dir(EXPERIMENT_URL), nni_source_dir, **kwargs)
+    except:
+        print_experiment_log(experiment_id=experiment_id)
+        print_trial_job_log(training_service, TRIAL_JOBS_URL)
+        raise
 
 def get_max_values(config_file):
     experiment_config = get_yml_content(config_file)
@@ -117,7 +123,7 @@ def launch_test(config_file, training_service, test_case_config):
 
     proc = subprocess.run(shlex.split(launch_command))
 
-    assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
+    assert proc.returncode == 0, 'launch command failed with code %d' % proc.returncode
 
     # set experiment ID into variable
     exp_var_name = test_case_config.get('setExperimentIdtoVar')
@@ -134,24 +140,30 @@ def launch_test(config_file, training_service, test_case_config):
 
     bg_time = time.time()
     print(str(datetime.datetime.now()), ' waiting ...', flush=True)
-    while True:
+    try:
+        # wait restful server to be ready
         time.sleep(3)
-        waited_time = time.time() - bg_time
-        if  waited_time > max_duration + 10:
-            print('waited: {}, max_duration: {}'.format(waited_time, max_duration))
-            break
-        status = get_experiment_status(STATUS_URL)
-        if status in ['DONE', 'ERROR']:
-            print('experiment status:', status)
-            break
-        num_failed = len(get_failed_trial_jobs(TRIAL_JOBS_URL))
-        if num_failed > 0:
-            print('failed jobs: ', num_failed)
-            break
-
+        experiment_id = get_experiment_id(EXPERIMENT_URL)
+        while True:
+            waited_time = time.time() - bg_time
+            if  waited_time > max_duration + 10:
+                print('waited: {}, max_duration: {}'.format(waited_time, max_duration))
+                break
+            status = get_experiment_status(STATUS_URL)
+            if status in ['DONE', 'ERROR']:
+                print('experiment status:', status)
+                break
+            num_failed = len(get_failed_trial_jobs(TRIAL_JOBS_URL))
+            if num_failed > 0:
+                print('failed jobs: ', num_failed)
+                break
+            time.sleep(3)
+    except:
+        print_experiment_log(experiment_id=experiment_id)
+        raise
     print(str(datetime.datetime.now()), ' waiting done', flush=True)
     if get_experiment_status(STATUS_URL) == 'ERROR':
-        print_experiment_log(EXPERIMENT_URL)
+        print_experiment_log(experiment_id=experiment_id)
 
     trial_stats = get_trial_stats(TRIAL_JOBS_URL)
     print(json.dumps(trial_stats, indent=4), flush=True)

test/nni_test/nnitest/utils.py

@@ -10,6 +10,7 @@ import subprocess
 import requests
 import time
 import ruamel.yaml as yaml
+import shlex
 
 EXPERIMENT_DONE_SIGNAL = 'Experiment done'
 
@@ -65,14 +66,16 @@ def get_experiment_id(experiment_url):
     experiment_id = requests.get(experiment_url).json()['id']
     return experiment_id
 
-def get_experiment_dir(experiment_url):
+def get_experiment_dir(experiment_url=None, experiment_id=None):
     '''get experiment root directory'''
-    experiment_id = get_experiment_id(experiment_url)
+    assert any([experiment_url, experiment_id])
+    if experiment_id is None:
+        experiment_id = get_experiment_id(experiment_url)
     return os.path.join(os.path.expanduser('~'), 'nni', 'experiments', experiment_id)
 
-def get_nni_log_dir(experiment_url):
+def get_nni_log_dir(experiment_url=None, experiment_id=None):
     '''get nni's log directory from nni's experiment url'''
-    return os.path.join(get_experiment_dir(experiment_url), 'log')
+    return os.path.join(get_experiment_dir(experiment_url, experiment_id), 'log')
 
 def get_nni_log_path(experiment_url):
     '''get nni's log path from nni's experiment url'''
@@ -125,12 +128,17 @@ def print_trial_job_log(training_service, trial_jobs_url):
         for log_file in log_files:
             print_file_content(os.path.join(trial_log_dir, log_file))
 
-def print_experiment_log(experiment_url):
-    log_dir = get_nni_log_dir(experiment_url)
+def print_experiment_log(experiment_id):
+    log_dir = get_nni_log_dir(experiment_id=experiment_id)
     for log_file in ['dispatcher.log', 'nnimanager.log']:
         filepath = os.path.join(log_dir, log_file)
         print_file_content(filepath)
 
+    print('nnictl log stderr:')
+    subprocess.run(shlex.split('nnictl log stderr {}'.format(experiment_id)))
+    print('nnictl log stdout:')
+    subprocess.run(shlex.split('nnictl log stdout {}'.format(experiment_id)))
+
 def parse_max_duration_time(max_exec_duration):
     unit = max_exec_duration[-1]
     time = max_exec_duration[:-1]

test/pipelines/pipelines-it-frameworkcontroller.yml

@@ -11,6 +11,10 @@ jobs:
 
   - script: |
       cd deployment/pypi
+      if [ -d ./dist ]
+      then
+        rm -rf ./dist/*
+      fi
       echo 'building prerelease package...'
       make build
       ls $(Build.SourcesDirectory)/deployment/pypi/dist/
@@ -28,6 +32,7 @@ jobs:
     displayName: 'Install dependencies for integration tests in frameworkcontroller mode'
 
   - script: |
+      set -e
       if [ $(build_docker_img) = 'true' ]
       then
         cd deployment/pypi
@@ -35,7 +40,7 @@ jobs:
 
         echo 'updating docker file for installing nni from local...'
         # update Dockerfile to install NNI in docker image from whl file built in last step
-        sed -ie 's/RUN python3 -m pip --no-cache-dir install nni/COPY .\/dist\/* .\nRUN python3 -m pip install nni-*.whl/' ../docker/Dockerfile
+        sed -ie 's/RUN python3 -m pip --no-cache-dir install nni/COPY .\/dist\/* .\/\nRUN python3 -m pip install nni-*.whl/' ../docker/Dockerfile
         cat ../docker/Dockerfile
         export IMG_TAG=`date -u +%y%m%d%H%M`
         docker build -f ../docker/Dockerfile -t $(test_docker_img_name):$IMG_TAG .

test/pipelines/pipelines-it-kubeflow.yml

@@ -11,6 +11,10 @@ jobs:
 
   - script: |
       cd deployment/pypi
+      if [ -d ./dist ]
+      then
+        rm -rf ./dist/*
+      fi
       echo 'building prerelease package...'
       make build
       ls $(Build.SourcesDirectory)/deployment/pypi/dist/
@@ -28,6 +32,7 @@ jobs:
     displayName: 'Install dependencies for integration tests in Kubeflow mode'
 
   - script: |
+      set -e
       if [ $(build_docker_img) = 'true' ]
       then
         cd deployment/pypi
@@ -35,7 +40,7 @@ jobs:
 
         echo 'updating docker file for installing nni from local...'
         # update Dockerfile to install NNI in docker image from whl file built in last step
-        sed -ie 's/RUN python3 -m pip --no-cache-dir install nni/COPY .\/dist\/* .\nRUN python3 -m pip install nni-*.whl/' ../docker/Dockerfile
+        sed -ie 's/RUN python3 -m pip --no-cache-dir install nni/COPY .\/dist\/* .\/\nRUN python3 -m pip install nni-*.whl/' ../docker/Dockerfile
         cat ../docker/Dockerfile
         export IMG_TAG=`date -u +%y%m%d%H%M`
         docker build -f ../docker/Dockerfile -t $(test_docker_img_name):$IMG_TAG .

test/pipelines/pipelines-it-pai-windows.yml

@@ -9,6 +9,10 @@ jobs:
 
   - script: |
       cd deployment/pypi
+      if [ -d ./dist ]
+      then
+        rm -rf ./dist/*
+      fi
       echo 'building prerelease package...'
       make build
       ls $(Build.SourcesDirectory)/deployment/pypi/dist/
@@ -16,13 +20,14 @@ jobs:
     displayName: 'build nni bdsit_wheel'
 
   - script: |
+      set -e
       if [ $(build_docker_img) = 'true' ]
       then
         cd deployment/pypi
         docker login -u $(docker_hub_user) -p $(docker_hub_pwd)
         echo 'updating docker file for installing nni from local...'
         # update Dockerfile to install NNI in docker image from whl file built in last step
-        sed -ie 's/RUN python3 -m pip --no-cache-dir install nni/COPY .\/dist\/* .\nRUN python3 -m pip install nni-*.whl/' ../docker/Dockerfile
+        sed -ie 's/RUN python3 -m pip --no-cache-dir install nni/COPY .\/dist\/* .\/\nRUN python3 -m pip install nni-*.whl/' ../docker/Dockerfile
         cat ../docker/Dockerfile
         export IMG_TAG=`date -u +%y%m%d%H%M`
 

test/pipelines/pipelines-it-pai.yml

@@ -11,6 +11,10 @@ jobs:
 
   - script: |
       cd deployment/pypi
+      if [ -d ./dist ]
+      then
+        rm -rf ./dist/*
+      fi
       echo 'building prerelease package...'
       make build
       ls $(Build.SourcesDirectory)/deployment/pypi/dist/
@@ -35,7 +39,7 @@ jobs:
         docker login -u $(docker_hub_user) -p $(docker_hub_pwd)
         echo 'updating docker file for installing nni from local...'
         # update Dockerfile to install NNI in docker image from whl file built in last step
-        sed -ie 's/RUN python3 -m pip --no-cache-dir install nni/COPY .\/dist\/* .\nRUN python3 -m pip install nni-*.whl/' ../docker/Dockerfile
+        sed -ie 's/RUN python3 -m pip --no-cache-dir install nni/COPY .\/dist\/* .\/\nRUN python3 -m pip install nni-*.whl/' ../docker/Dockerfile
         cat ../docker/Dockerfile
         export IMG_TAG=`date -u +%y%m%d%H%M`