export trial status to csv (#905)

* export to csv core code done

* add nnictl trail export

* fix bug

* add dependency for pandas

* remove pandas from dependency

* add multi phase support

* add document

docs/en_US/NNICTLDOC.md

@@ -298,6 +298,22 @@ nnictl support commands:
       nnictl trial [trial_id] --vexperiment [experiment_id]
       ```
 
+  * __nnictl trial export__
+    * Description
+        You can use this command to export reward & hyper-parameter of trial jobs to a csv file.
+
+    * Usage
+        ```bash
+        nnictl trial export [OPTIONS]
+        ```
+    * Options
+    
+  |Name, shorthand|Required|Default|Description|
+  |------|------|------ |------|
+  |id|  False| |ID of the experiment    |
+  |--file|  True| |File path of the output csv file     |
+
+
 <a name="top"></a>
 
 ![](https://placehold.it/15/1589F0/000000?text=+) `nnictl top`

setup.py

@@ -50,6 +50,7 @@ setup(
         'hyperopt',
         'json_tricks',
         'numpy',
+        'pandas',
         'psutil',
         'pyyaml',
         'requests',

tools/nni_cmd/nnictl.py

@@ -101,6 +101,10 @@ def parse_args():
     parser_trial_kill.add_argument('id', nargs='?', help='id of the trial to be killed')
     parser_trial_kill.add_argument('--experiment', '-E', required=True, dest='experiment', help='experiment id of the trial')
     parser_trial_kill.set_defaults(func=trial_kill)
+    parser_trial_export = parser_trial_subparsers.add_parser('export', help='export trial job results to csv')
+    parser_trial_export.add_argument('id', nargs='?', help='the id of experiment')
+    parser_trial_export.add_argument('--file', '-f', required=True, dest='csv_path', help='target csv file path')
+    parser_trial_export.set_defaults(func=export_trials_data)
 
     #parse experiment command
     parser_experiment = subparsers.add_parser('experiment', help='get experiment information')

tools/nni_cmd/nnictl_utils.py

@@ -18,11 +18,13 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+import csv
 import os
 import psutil
 import json
 import datetime
 import time
+
 from subprocess import call, check_output
 from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response
 from .config_utils import Config, Experiments
@@ -468,3 +470,50 @@ def monitor_experiment(args):
         except Exception as exception:
             print_error(exception)
             exit(1)
+
+
+def parse_trial_data(content):
+    """output: List[Dict]"""
+    trial_records = []
+    for trial_data in content:
+        for phase_i in range(len(trial_data['hyperParameters'])):
+            hparam = json.loads(trial_data['hyperParameters'][phase_i])['parameters']
+            if 'finalMetricData' in trial_data.keys() and phase_i < len(trial_data['finalMetricData']):
+                reward = json.loads(trial_data['finalMetricData'][phase_i]['data'])
+                if isinstance(reward, (float, int)):
+                    dict_tmp = {**hparam, **{'reward': reward}}
+                elif isinstance(reward, dict):
+                    dict_tmp = {**hparam, **reward}
+                else:
+                    raise ValueError("Invalid finalMetricsData format: {}/{}".format(type(reward), reward))
+            else:
+                dict_tmp = hparam
+            trial_records.append(dict_tmp)
+    return trial_records
+
+def export_trials_data(args):
+    """export experiment metadata to csv
+    :param: csv file path to export
+    """
+    nni_config = Config(get_config_filename(args))
+    rest_port = nni_config.get_config('restServerPort')
+    rest_pid = nni_config.get_config('restServerPid')
+    if not detect_process(rest_pid):
+        print_error('Experiment is not running...')
+        return
+    running, response = check_rest_server_quick(rest_port)
+    if running:
+        response = rest_get(trial_jobs_url(rest_port), 20)
+        if response is not None and check_response(response):
+            content = json.loads(response.text)
+            # dframe = pd.DataFrame.from_records([parse_trial_data(t_data) for t_data in content])
+            # dframe.to_csv(args.csv_path, sep='\t')
+            records = parse_trial_data(content)
+            with open(args.csv_path, 'w') as f_csv:
+                writer = csv.DictWriter(f_csv, set.union(*[set(r.keys()) for r in records]))
+                writer.writeheader()
+                writer.writerows(records)
+        else:
+            print_error('Export failed...')
+    else:
+        print_error('Restful server is not Running')