Merge pull request #206 from microsoft/master

merge master

src/sdk/pynni/nni/nas_utils.py

@@ -32,7 +32,7 @@ def classic_mode(
     '''Execute the chosen function and inputs directly.
     In this mode, the trial code is only running the chosen subgraph (i.e., the chosen ops and inputs),
     without touching the full model graph.'''
-    if trial._params is None:
+    if trial.get_current_parameter() is None:
         trial.get_next_parameter()
     mutable_block = trial.get_current_parameter(mutable_id)
     chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"]
@@ -118,7 +118,7 @@ def oneshot_mode(
     The difference is that oneshot mode does not receive subgraph.
     Instead, it uses dropout to randomly dropout inputs and ops.'''
     # NNI requires to get_next_parameter before report a result. But the parameter will not be used in this mode
-    if trial._params is None:
+    if trial.get_current_parameter() is None:
         trial.get_next_parameter()
     optional_inputs = list(optional_inputs.values())
     inputs_num = len(optional_inputs)

src/sdk/pynni/nni/smartparam.py

@@ -189,6 +189,6 @@ else:
         raise RuntimeError('Unrecognized mode: %s' % mode)
 
     def _get_param(key):
-        if trial._params is None:
+        if trial.get_current_parameter() is None:
             trial.get_next_parameter()
         return trial.get_current_parameter(key)

src/sdk/pynni/nni/trial.py

@@ -50,10 +50,12 @@ def get_next_parameter():
         return None
     return _params['parameters']
 
-def get_current_parameter(tag):
+def get_current_parameter(tag=None):
     global _params
     if _params is None:
         return None
+    if tag is None:
+        return _params['parameters']
     return _params['parameters'][tag]
 
 def get_experiment_id():

src/sdk/pynni/nni/tuner.py

@@ -57,19 +57,22 @@ class Tuner(Recoverable):
 
     def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
         """Invoked when a trial reports its final result. Must override.
+        By default this only reports results of algorithm-generated hyper-parameters.
+        Use `accept_customized_trials()` to receive results from user-added parameters.
         parameter_id: int
         parameters: object created by 'generate_parameters()'
-        reward: object reported by trial
+        value: object reported by trial
+        customized: bool, true if the trial is created from web UI, false if generated by algorithm
+        trial_job_id: str, only available in multiphase mode.
         """
         raise NotImplementedError('Tuner: receive_trial_result not implemented')
 
-    def receive_customized_trial_result(self, parameter_id, parameters, value, **kwargs):
-        """Invoked when a trial added by WebUI reports its final result. Do nothing by default.
-        parameter_id: int
-        parameters: object created by user
-        value: object reported by trial
+    def accept_customized_trials(self, accept=True):
+        """Enable or disable receiving results of user-added hyper-parameters. 
+        By default `receive_trial_result()` will only receive results of algorithm-generated hyper-parameters.
+        If tuners want to receive those of customized parameters as well, they can call this function in `__init__()`.
         """
-        _logger.info('Customized trial job %s ignored by tuner', parameter_id)
+        self._accept_customized = accept
 
     def trial_end(self, parameter_id, success, **kwargs):
         """Invoked when a trial is completed or terminated. Do nothing by default.

src/sdk/pynni/tests/test_tuner.py

@@ -34,6 +34,7 @@ class NaiveTuner(Tuner):
         self.param = 0
         self.trial_results = [ ]
         self.search_space = None
+        self.accept_customized_trials()
 
     def generate_parameters(self, parameter_id, **kwargs):
         # report Tuner's internal states to generated parameters,
@@ -45,13 +46,9 @@ class NaiveTuner(Tuner):
             'search_space': self.search_space
         }
 
-    def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
+    def receive_trial_result(self, parameter_id, parameters, value, customized, **kwargs):
         reward = extract_scalar_reward(value)
-        self.trial_results.append((parameter_id, parameters['param'], reward, False))
-
-    def receive_customized_trial_result(self, parameter_id, parameters, value):
-        reward = extract_scalar_reward(value)
-        self.trial_results.append((parameter_id, parameters['param'], reward, True))
+        self.trial_results.append((parameter_id, parameters['param'], reward, customized))
 
     def update_search_space(self, search_space):
         self.search_space = search_space

src/webui/src/components/Modal/Compare.tsx

@@ -107,6 +107,7 @@ class Compare extends React.Component<CompareProps, {}> {
     initColumn = () => {
         const { compareRows } = this.props;
         const idList: Array<string> = [];
+        const sequenceIdList: Array<number> = [];
         const durationList: Array<number> = [];
 
         const parameterList: Array<object> = [];
@@ -117,6 +118,7 @@ class Compare extends React.Component<CompareProps, {}> {
         Object.keys(compareRows).map(item => {
             const temp = compareRows[item];
             idList.push(temp.id);
+            sequenceIdList.push(temp.sequenceId);
             durationList.push(temp.duration);
             parameterList.push(temp.description.parameters);
         });
@@ -124,13 +126,21 @@ class Compare extends React.Component<CompareProps, {}> {
             <table className="compare">
                 <tbody>
                     <tr>
-                        <td />
+                        <td className="column">Id</td>
                         {Object.keys(idList).map(key => {
                             return (
                                 <td className="value idList" key={key}>{idList[key]}</td>
                             );
                         })}
                     </tr>
+                    <tr>
+                        <td className="column">Trial No.</td>
+                        {Object.keys(sequenceIdList).map(key => {
+                            return (
+                                <td className="value idList" key={key}>{sequenceIdList[key]}</td>
+                            );
+                        })}
+                    </tr>
                     <tr>
                         <td className="column">Default metric</td>
                         {Object.keys(compareRows).map(index => {

src/webui/src/components/SlideBar.tsx

@@ -223,6 +223,16 @@ class SlideBar extends React.Component<SliderProps, SliderState> {
                     <span>{DETAILTABS}</span>
                 </Col>
                 <Col span={16} className="desktop-right">
+                    <span>
+                        <Button
+                            className="fresh"
+                            type="ghost"
+                        >
+                            <a target="_blank" href="https://nni.readthedocs.io/en/latest/Tutorial/WebUI.html">
+                                <Icon type="question" /><span>Help</span>
+                            </a>
+                        </Button>
+                    </span>
                     <span>{this.select()}</span>
                     <span>
                         <Dropdown
@@ -238,9 +248,9 @@ class SlideBar extends React.Component<SliderProps, SliderState> {
                                 {
                                     menuVisible
                                         ?
-                                    <Icon type="up" className="margin-icon"/>
+                                        <Icon type="up" className="margin-icon" />
                                         :
-                                    <Icon type="down" className="margin-icon"/>
+                                        <Icon type="down" className="margin-icon" />
                                 }
                             </a>
                         </Dropdown>

src/webui/src/components/TrialsDetail.tsx

@@ -171,7 +171,9 @@ class TrialsDetail extends React.Component<TrialsDetailProps, TrialDetailState>
                             status: status,
                             duration: duration,
                             acc: acc,
-                            description: desc
+                            description: desc,
+                            startTime: begin,
+                            endTime: (end !== undefined) ? end : undefined
                         });
                     });
                     // update search data result

src/webui/src/components/trial-detail/DefaultMetricPoint.tsx

@@ -85,7 +85,9 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
             });
             // deal with best metric line
             const bestCurve: Array<number | object>[] = []; // best curve data source
+            if (lineListDefault[0] !== undefined) {
                 bestCurve.push([lineListDefault[0][0], lineListDefault[0][1], accSource[0].searchSpace]);
+            }
             if (optimize === 'maximize') {
                 for (let i = 1; i < lineListDefault.length; i++) {
                     const val = lineListDefault[i][1];

src/webui/src/components/trial-detail/Para.tsx

@@ -128,13 +128,22 @@ class Para extends React.Component<ParaProps, ParaState> {
 
     hyperParaPic = (source: Array<TableObj>, searchSpace: string) => {
         // filter succeed trials [{}, {}, {}]
-        const dataSource: Array<TableObj> = source.filter(filterByStatus);
+        const origin = source.filter(filterByStatus);
+        const dataSource: Array<TableObj> = JSON.parse(JSON.stringify(origin));
         const lenOfDataSource: number = dataSource.length;
         const accPara: Array<number> = [];
         // specific value array
         const eachTrialParams: Array<string> = [];
         // experiment interface search space obj
         const searchRange = searchSpace !== undefined ? JSON.parse(searchSpace) : '';
+        // nest search space
+        let isNested: boolean = false;
+        Object.keys(searchRange).map(item => {
+            if (typeof searchRange[item]._value[0] === 'object') {
+                isNested = true;
+                return;
+            }
+        });
         const dimName = Object.keys(searchRange);
         if (this._isMounted === true) {
             this.setState(() => ({ dimName: dimName }));
@@ -143,6 +152,7 @@ class Para extends React.Component<ParaProps, ParaState> {
         const parallelAxis: Array<Dimobj> = [];
         // search space range and specific value [only number]
         let i = 0;
+        if (isNested === false) {
             for (i; i < dimName.length; i++) {
                 const searchKey = searchRange[dimName[i]];
                 switch (searchKey._type) {
@@ -155,7 +165,6 @@ class Para extends React.Component<ParaProps, ParaState> {
                             min: searchKey._value[0]
                         });
                         break;
-
                     case 'randint':
                         parallelAxis.push({
                             dim: i,
@@ -164,7 +173,6 @@ class Para extends React.Component<ParaProps, ParaState> {
                             max: searchKey._value[1],
                         });
                         break;
-
                     case 'choice':
                         const data: Array<string> = [];
                         for (let j = 0; j < searchKey._value.length; j++) {
@@ -209,13 +217,66 @@ class Para extends React.Component<ParaProps, ParaState> {
                             });
                         }
                         break;
-
                     default:
                         parallelAxis.push({
                             dim: i,
                             name: dimName[i]
                         });
-
+                }
+            }
+        } else {
+            for (i; i < dimName.length; i++) {
+                const searchKey = searchRange[dimName[i]];
+                switch (searchKey._type) {
+                    case 'choice':
+                        const data: Array<string> = [];
+                        let j = 0;
+                        for (j; j < searchKey._value.length; j++) {
+                            const item = searchKey._value[j];
+                            Object.keys(item).map(key => {
+                                if (key !== '_name' && key !== '_type') {
+                                    Object.keys(item[key]).map(index => {
+                                        if (index !== '_type') {
+                                            const realChoice = item[key][index];
+                                            Object.keys(realChoice).map(m => {
+                                                data.push(`${item._name}_${realChoice[m]}`);
+                                            });
+                                        }
+                                    });
+                                }
+                            });
+                        }
+                        data.push('null');
+                        parallelAxis.push({
+                            dim: i,
+                            name: dimName[i],
+                            type: 'category',
+                            data: data,
+                            boundaryGap: true,
+                            axisLine: {
+                                lineStyle: {
+                                    type: 'dotted', // axis type,solid dashed dotted
+                                    width: 1
+                                }
+                            },
+                            axisTick: {
+                                show: true,
+                                interval: 0,
+                                alignWithLabel: true,
+                            },
+                            axisLabel: {
+                                show: true,
+                                interval: 0,
+                                // rotate: 30
+                            },
+                        });
+                        break;
+                    default:
+                        parallelAxis.push({
+                            dim: i,
+                            name: dimName[i]
+                        });
+                }
             }
         }
         parallelAxis.push({
@@ -283,6 +344,23 @@ class Para extends React.Component<ParaProps, ParaState> {
                     }
                 }
             });
+            // nested search space, deal data
+            if (isNested !== false) {
+                eachTrialParams.forEach(element => {
+                    Object.keys(element).forEach(key => {
+                        let item = element[key];
+                        if (typeof item === 'object') {
+                            Object.keys(item).forEach(index => {
+                                if (index !== '_name') {
+                                    element[key] = `${item._name}_${item[index]}`;
+                                } else {
+                                    element[key] = 'null';
+                                }
+                            });
+                        }
+                    });
+                });
+            }
             if (this._isMounted) {
                 // if not return final result
                 const maxVal = accPara.length === 0 ? 1 : Math.max(...accPara);

src/webui/src/components/trial-detail/TableList.tsx

@@ -182,7 +182,7 @@ class TableList extends React.Component<TableListProps, TableListState> {
 
     // checkbox for coloumn
     selectedColumn = (checkedValues: Array<string>) => {
-        // 7: because have seven common column, "Intermediate count" is not shown by default
+        // 7: because have seven common column, "Intermediate count" is hidden by default
         let count = 7;
         const want: Array<object> = [];
         const finalKeys: Array<string> = [];
@@ -191,6 +191,8 @@ class TableList extends React.Component<TableListProps, TableListState> {
             switch (checkedValues[m]) {
                 case 'Trial No.':
                 case 'ID':
+                case 'StartTime':
+                case 'EndTime':
                 case 'Duration':
                 case 'Status':
                 case 'Operation':
@@ -347,6 +349,50 @@ class TableList extends React.Component<TableListProps, TableListState> {
                         }
                     });
                     break;
+                case 'StartTime':
+                    showColumn.push({
+                        title: 'StartTime',
+                        dataIndex: 'startTime',
+                        key: 'startTime',
+                        width: 160,
+                        render: (text: string, record: TableObj) => {
+                            const start = record.startTime !== undefined ? record.startTime : -1;
+                            return (
+                                <span>
+                                    {
+                                        start !== -1
+                                            ?
+                                            new Date(start).toLocaleString('en-US')
+                                            :
+                                            '--'
+                                    }
+                                </span>
+                            );
+                        },
+                    });
+                    break;
+                case 'EndTime':
+                    showColumn.push({
+                        title: 'EndTime',
+                        dataIndex: 'endTime',
+                        key: 'endTime',
+                        width: 160,
+                        render: (text: string, record: TableObj) => {
+                            const end = record.endTime !== undefined ? record.endTime : -1;
+                            return (
+                                <span>
+                                    {
+                                        end !== -1
+                                            ?
+                                            new Date(end).toLocaleString('en-US')
+                                            :
+                                            '--'
+                                    }
+                                </span>
+                            );
+                        },
+                    });
+                    break;
                 case 'Duration':
                     showColumn.push({
                         title: 'Duration',

src/webui/src/static/const.ts

@@ -34,21 +34,29 @@ const COLUMN_INDEX = [
         index: 2
     },
     {
-        name: 'Duration',
+        name: 'StartTime',
         index: 3
     },
     {
-        name: 'Status',
+        name: 'EndTime',
         index: 4
     },
     {
-        name: 'Intermediate count',
+        name: 'Duration',
         index: 5
     },
     {
-        name: 'Default',
+        name: 'Status',
         index: 6
     },
+    {
+        name: 'Intermediate count',
+        index: 7
+    },
+    {
+        name: 'Default',
+        index: 8
+    },
     {
         name: 'Operation',
         index: 10000
@@ -57,7 +65,8 @@ const COLUMN_INDEX = [
 // defatult selected column
 const COLUMN = ['Trial No.', 'ID', 'Duration', 'Status', 'Default', 'Operation'];
 // all choice column !dictory final
-const COLUMNPro = ['Trial No.', 'ID', 'Duration', 'Status', 'Intermediate count', 'Default', 'Operation'];
+const COLUMNPro = ['Trial No.', 'ID', 'StartTime', 'EndTime', 'Duration', 'Status',
+'Intermediate count', 'Default', 'Operation'];
 export {
     MANAGER_IP, DOWNLOAD_IP, trialJobStatus, COLUMNPro,
     CONTROLTYPE, MONACO, COLUMN, COLUMN_INDEX, DRAWEROPTION

src/webui/src/static/interface.ts

@@ -8,6 +8,8 @@ interface TableObj {
     acc?: FinalType; // draw accuracy graph
     description: Parameters;
     color?: string;
+    startTime?: number;
+    endTime?: number;
 }
 
 interface SearchSpace {

src/webui/src/static/style/slideBar.scss

@@ -9,10 +9,11 @@ $drowHoverBgColor: #e2e2e2;
     border: none;
     color: #fff;
     font-size: 16px;
-    padding: 0;
+    padding: 0 8px;
 }
 .fresh:hover{
     color: #fff;
+    background-color: #006cb5;
 }
 
 .dropdown{

src/webui/src/static/style/table.scss

@@ -115,6 +115,7 @@
 }
 
 #detail-button{
+    margin: 2px 0;
     .common-style, .common-style:visited, .common-style:focus{
         height: 26px;
         border: none;
@@ -131,7 +132,7 @@
     .common-style:disabled{
         background-color: #f4f4f4;
     }
-    .special, .special:visited, .special:focus{
+    .special, .special:visited, .special:focus, .special button{
         height: 26px;
         border: none;
         border-radius: 0;
@@ -146,7 +147,7 @@
         background-color: #c8c8c8;
         outline: 0;
     }
-    .special:disabled{
+    .special:disabled, .special button:disabled{
         background-color: #f4f4f4;
         color: #d9d9d9;
     }

test/config_test/examples/mnist-pytorch.test.yml

+authorName: nni
+experimentName: default_test
+maxExecDuration: 15m
+maxTrialNum: 4
+trialConcurrency: 2
+searchSpacePath: ./mnist_pytorch_search_space.json
+
+tuner:
+  builtinTunerName: Random
+assessor:
+  builtinAssessorName: Medianstop
+  classArgs:
+    optimize_mode: maximize
+trial:
+  codeDir: ../../../examples/trials/mnist-pytorch
+  command: python3 mnist.py  --epochs 2
+  gpuNum: 0
+
+useAnnotation: false
+multiPhase: false
+multiThread: false
+
+trainingServicePlatform: local

test/config_test/examples/mnist_pytorch_search_space.json

+{
+    "batch_size": {"_type":"choice", "_value": [16, 32, 64, 128]},
+    "hidden_size":{"_type":"choice","_value":[128, 256, 512, 1024]},
+    "lr":{"_type":"choice","_value":[0.0001, 0.001, 0.01, 0.1]},
+    "momentum":{"_type":"uniform","_value":[0, 1]}
+}

tools/nni_cmd/config_schema.py

@@ -240,7 +240,12 @@ pai_trial_schema = {
     Optional('outputDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'),\
                          error='ERROR: outputDir format error, outputDir format is hdfs://xxx.xxx.xxx.xxx:xxx'),
     Optional('virtualCluster'): setType('virtualCluster', str),
-    Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode')
+    Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'),
+    Optional('portList'): [{
+        "label": setType('label', str),
+        "beginAt": setType('beginAt', int),
+        "portNumber": setType('portNumber', int)
+    }]
     }
 }
 
@@ -310,7 +315,8 @@ kubeflow_config_schema = {
                            error='ERROR: accountName format error, accountName support using (0-9|a-z|A-Z|-)'),
             'azureShare': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,63}'),\
                           error='ERROR: azureShare format error, azureShare support using (0-9|a-z|A-Z|-)')
-        }
+        },
+        Optional('uploadRetryCount'): setNumberRange('uploadRetryCount', int, 1, 99999)
     })
 }
 
@@ -356,7 +362,8 @@ frameworkcontroller_config_schema = {
                            error='ERROR: accountName format error, accountName support using (0-9|a-z|A-Z|-)'),
             'azureShare': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,63}'),\
                           error='ERROR: azureShare format error, azureShare support using (0-9|a-z|A-Z|-)')
-        }
+        },
+        Optional('uploadRetryCount'): setNumberRange('uploadRetryCount', int, 1, 99999)
     })
 }
 

tools/nni_cmd/launcher_utils.py

@@ -198,10 +198,7 @@ def validate_common_content(experiment_config):
                     Schema({**separate_schema_dict[separate_key]['customized']}).validate(experiment_config[separate_key])
     except SchemaError as error:
         print_error('Your config file is not correct, please check your config file content!')
-        if error.__str__().__contains__('Wrong key'):
-            print_error(' '.join(error.__str__().split()[:3]))
-        else:
-            print_error(error)
+        print_error(error.code)
         exit(1)
 
     #set default value