Merge pull request #156 from Microsoft/master

merge master

src/sdk/pynni/nni/multi_phase/multi_phase_tuner.py

@@ -32,7 +32,7 @@ class MultiPhaseTuner(Recoverable):
     def generate_parameters(self, parameter_id, trial_job_id=None):
         """Returns a set of trial (hyper-)parameters, as a serializable object.
         User code must override either this function or 'generate_multiple_parameters()'.
-        parameter_id: int
+        parameter_id: identifier of the parameter (int)
         """
         raise NotImplementedError('Tuner: generate_parameters not implemented')
 
@@ -46,20 +46,30 @@ class MultiPhaseTuner(Recoverable):
 
     def receive_trial_result(self, parameter_id, parameters, value, trial_job_id):
         """Invoked when a trial reports its final result. Must override.
-        parameter_id: int
+        parameter_id: identifier of the parameter (int)
         parameters: object created by 'generate_parameters()'
         value: object reported by trial
+        trial_job_id: identifier of the trial (str)
         """
         raise NotImplementedError('Tuner: receive_trial_result not implemented')
 
     def receive_customized_trial_result(self, parameter_id, parameters, value, trial_job_id):
         """Invoked when a trial added by WebUI reports its final result. Do nothing by default.
-        parameter_id: int
+        parameter_id: identifier of the parameter (int)
         parameters: object created by user
         value: object reported by trial
+        trial_job_id: identifier of the trial (str)
         """
         _logger.info('Customized trial job %s ignored by tuner', parameter_id)
 
+    def trial_end(self, parameter_id, success, trial_job_id):
+        """Invoked when a trial is completed or terminated. Do nothing by default.
+        parameter_id: identifier of the parameter (int)
+        success: True if the trial successfully completed; False if failed or terminated
+        trial_job_id: identifier of the trial (str)
+        """
+        pass
+
     def update_search_space(self, search_space):
         """Update the search space of tuner. Must override.
         search_space: JSON object

src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py

@@ -23,6 +23,7 @@ import os
 
 
 from nni.tuner import Tuner
+from nni.utils import extract_scalar_reward
 from nni.networkmorphism_tuner.bayesian import BayesianOptimizer
 from nni.networkmorphism_tuner.nn import CnnGenerator, MlpGenerator
 from nni.networkmorphism_tuner.utils import Constant, OptimizeMode
@@ -161,7 +162,7 @@ class NetworkMorphismTuner(Tuner):
         value : dict/float
             if value is dict, it should have "default" key.
         """
-        reward = self.extract_scalar_reward(value)
+        reward = extract_scalar_reward(value)
 
         if parameter_id not in self.total_data:
             raise RuntimeError("Received parameter_id not in total_data.")

src/sdk/pynni/nni/smac_tuner/smac_tuner.py

@@ -22,6 +22,7 @@ smac_tuner.py
 """
 
 from nni.tuner import Tuner
+from nni.utils import extract_scalar_reward
 
 import sys
 import logging
@@ -166,7 +167,7 @@ class SMACTuner(Tuner):
         RuntimeError
             Received parameter id not in total_data
         """
-        reward = self.extract_scalar_reward(value)
+        reward = extract_scalar_reward(value)
         if self.optimize_mode is OptimizeMode.Maximize:
             reward = -reward
 

src/sdk/pynni/nni/tuner.py

@@ -71,6 +71,13 @@ class Tuner(Recoverable):
         """
         _logger.info('Customized trial job %s ignored by tuner', parameter_id)
 
+    def trial_end(self, parameter_id, success):
+        """Invoked when a trial is completed or terminated. Do nothing by default.
+        parameter_id: int
+        success: True if the trial successfully completed; False if failed or terminated
+        """
+        pass
+
     def update_search_space(self, search_space):
         """Update the search space of tuner. Must override.
         search_space: JSON object
@@ -96,12 +103,3 @@ class Tuner(Recoverable):
 
     def _on_error(self):
         pass
-
-    def extract_scalar_reward(self, value, scalar_key='default'):
-        if isinstance(value, float) or isinstance(value, int):
-            reward = value
-        elif isinstance(value, dict) and scalar_key in value and isinstance(value[scalar_key], (float, int)):
-            reward = value[scalar_key]
-        else:
-            raise RuntimeError('Incorrect final result: the final result for %s should be float/int, or a dict which has a key named "default" whose value is float/int.' % str(self.__class__)) 
-        return reward 
\ No newline at end of file

src/sdk/pynni/nni/utils.py

+# Copyright (c) Microsoft Corporation. All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+# associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute,
+# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# ==================================================================================================
+
+def extract_scalar_reward(value, scalar_key='default'):
+    """
+    Raises
+    ------
+    RuntimeError
+        Incorrect final result: the final result should be float/int,
+        or a dict which has a key named "default" whose value is float/int.
+    """
+    if isinstance(value, float) or isinstance(value, int):
+        reward = value
+    elif isinstance(value, dict) and scalar_key in value and isinstance(value[scalar_key], (float, int)):
+        reward = value[scalar_key]
+    else:
+        raise RuntimeError('Incorrect final result: the final result for %s should be float/int, or a dict which has a key named "default" whose value is float/int.' % str(self.__class__)) 
+    return reward
\ No newline at end of file

src/sdk/pynni/tests/test_tuner.py

@@ -23,6 +23,7 @@ import nni.protocol
 from nni.protocol import CommandType, send, receive
 from nni.tuner import Tuner
 from nni.msg_dispatcher import MsgDispatcher
+from nni.utils import extract_scalar_reward
 from io import BytesIO
 import json
 from unittest import TestCase, main
@@ -45,11 +46,11 @@ class NaiveTuner(Tuner):
         }
 
     def receive_trial_result(self, parameter_id, parameters, value):
-        reward = self.extract_scalar_reward(value)
+        reward = extract_scalar_reward(value)
         self.trial_results.append((parameter_id, parameters['param'], reward, False))
 
     def receive_customized_trial_result(self, parameter_id, parameters, value):
-        reward = self.extract_scalar_reward(value)
+        reward = extract_scalar_reward(value)
         self.trial_results.append((parameter_id, parameters['param'], reward, True))
 
     def update_search_space(self, search_space):

src/webui/package.json

@@ -6,6 +6,7 @@
     "antd": "^3.8.1",
     "axios": "^0.18.0",
     "babel-polyfill": "^6.26.0",
+    "copy-to-clipboard": "^3.0.8",
     "echarts": "^4.1.0",
     "echarts-for-react": "^2.0.14",
     "react": "^16.4.2",

src/webui/src/components/public-child/OpenRow.tsx

 import * as React from 'react';
+import * as copy from 'copy-to-clipboard';
 import PaiTrialLog from '../public-child/PaiTrialLog';
 import TrialLog from '../public-child/TrialLog';
-import JSONTree from 'react-json-tree';
 import { TableObj } from '../../static/interface';
-import { Row, Tabs } from 'antd';
+import { Row, Tabs, Button, message } from 'antd';
+import JSONTree from 'react-json-tree';
 const TabPane = Tabs.TabPane;
 
 interface OpenRowProps {
@@ -19,6 +20,15 @@ class OpenRow extends React.Component<OpenRowProps, {}> {
 
     }
 
+    copyParams = (record: TableObj) => {
+        let params = JSON.stringify(record.description.parameters);
+        if (copy(params)) {
+            message.success('Success copy parameters to clipboard in form of python dict !', 3);
+        } else {
+            message.error('Failed !', 2);
+        }
+    }
+
     render() {
         const { trainingPlatform, record, logCollection } = this.props;
 
@@ -42,12 +52,19 @@ class OpenRow extends React.Component<OpenRowProps, {}> {
                             {
                                 isHasParameters
                                     ?
+                                    <div>
                                         <JSONTree
                                             hideRoot={true}
                                             shouldExpandNode={() => true}  // default expandNode
                                             getItemString={() => (<span />)}  // remove the {} items
-                                        data={openRowDataSource}
+                                            data={openRowDataSource.parameters}
                                         />
+                                        <Button
+                                            onClick={this.copyParams.bind(this, record)}
+                                        >
+                                            Copy as Python
+                                        </Button>
+                                    </div>
                                     :
                                     <div className="logpath">
                                         <span className="logName">Error: </span>

src/webui/src/static/style/probar.scss

@@ -59,4 +59,6 @@
     width: 240px;
     font-size: 14px;
     color: #212121;
+    word-wrap: break-word;
+    word-break: normal;
 }

test/naive_test/naive_tuner.py

@@ -3,6 +3,7 @@ import logging
 import os
 
 from nni.tuner import Tuner
+from nni.utils import extract_scalar_reward
 
 _logger = logging.getLogger('NaiveTuner')
 _logger.info('start')
@@ -21,7 +22,7 @@ class NaiveTuner(Tuner):
         return { 'x': self.cur }
 
     def receive_trial_result(self, parameter_id, parameters, value):
-        reward = self.extract_scalar_reward(value)
+        reward = extract_scalar_reward(value)
         _logger.info('receive trial result: %s, %s, %s' % (parameter_id, parameters, reward))
         _result.write('%d %d\n' % (parameters['x'], reward))
         _result.flush()

tools/nni_cmd/nnictl.py

@@ -101,6 +101,10 @@ def parse_args():
     parser_trial_kill.add_argument('id', nargs='?', help='id of the trial to be killed')
     parser_trial_kill.add_argument('--experiment', '-E', required=True, dest='experiment', help='experiment id of the trial')
     parser_trial_kill.set_defaults(func=trial_kill)
+    parser_trial_export = parser_trial_subparsers.add_parser('export', help='export trial job results to csv')
+    parser_trial_export.add_argument('id', nargs='?', help='the id of experiment')
+    parser_trial_export.add_argument('--file', '-f', required=True, dest='csv_path', help='target csv file path')
+    parser_trial_export.set_defaults(func=export_trials_data)
 
     #parse experiment command
     parser_experiment = subparsers.add_parser('experiment', help='get experiment information')

tools/nni_cmd/nnictl_utils.py

@@ -18,11 +18,13 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+import csv
 import os
 import psutil
 import json
 import datetime
 import time
+
 from subprocess import call, check_output
 from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response
 from .config_utils import Config, Experiments
@@ -468,3 +470,50 @@ def monitor_experiment(args):
         except Exception as exception:
             print_error(exception)
             exit(1)
+
+
+def parse_trial_data(content):
+    """output: List[Dict]"""
+    trial_records = []
+    for trial_data in content:
+        for phase_i in range(len(trial_data['hyperParameters'])):
+            hparam = json.loads(trial_data['hyperParameters'][phase_i])['parameters']
+            hparam['id'] = trial_data['id']
+            if 'finalMetricData' in trial_data.keys() and phase_i < len(trial_data['finalMetricData']):
+                reward = json.loads(trial_data['finalMetricData'][phase_i]['data'])
+                if isinstance(reward, (float, int)):
+                    dict_tmp = {**hparam, **{'reward': reward}}
+                elif isinstance(reward, dict):
+                    dict_tmp = {**hparam, **reward}
+                else:
+                    raise ValueError("Invalid finalMetricsData format: {}/{}".format(type(reward), reward))
+            else:
+                dict_tmp = hparam
+            trial_records.append(dict_tmp)
+    return trial_records
+
+def export_trials_data(args):
+    """export experiment metadata to csv
+    """
+    nni_config = Config(get_config_filename(args))
+    rest_port = nni_config.get_config('restServerPort')
+    rest_pid = nni_config.get_config('restServerPid')
+    if not detect_process(rest_pid):
+        print_error('Experiment is not running...')
+        return
+    running, response = check_rest_server_quick(rest_port)
+    if running:
+        response = rest_get(trial_jobs_url(rest_port), 20)
+        if response is not None and check_response(response):
+            content = json.loads(response.text)
+            # dframe = pd.DataFrame.from_records([parse_trial_data(t_data) for t_data in content])
+            # dframe.to_csv(args.csv_path, sep='\t')
+            records = parse_trial_data(content)
+            with open(args.csv_path, 'w') as f_csv:
+                writer = csv.DictWriter(f_csv, set.union(*[set(r.keys()) for r in records]))
+                writer.writeheader()
+                writer.writerows(records)
+        else:
+            print_error('Export failed...')
+    else:
+        print_error('Restful server is not Running')