nnictl_utils.py 22.5 KB
Newer Older
Deshui Yu's avatar
Deshui Yu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# Copyright (c) Microsoft Corporation
# All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge,
# to any person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Yan Ni's avatar
Yan Ni committed
21
import csv
Deshui Yu's avatar
Deshui Yu committed
22
23
24
import os
import psutil
import json
25
import datetime
26
import time
Yan Ni's avatar
Yan Ni committed
27

Deshui Yu's avatar
Deshui Yu committed
28
from subprocess import call, check_output
29
from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response
30
from .config_utils import Config, Experiments
Deshui Yu's avatar
Deshui Yu committed
31
from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url
SparkSnail's avatar
SparkSnail committed
32
from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \
33
     EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT
34
from .common_utils import print_normal, print_error, print_warning, detect_process
Deshui Yu's avatar
Deshui Yu committed
35

36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
def get_experiment_time(port):
    '''get the startTime and endTime of an experiment'''
    response = rest_get(experiment_url(port), REST_TIME_OUT)
    if response and check_response(response):
        content = convert_time_stamp_to_date(json.loads(response.text))
        return content.get('startTime'), content.get('endTime')
    return None, None

def get_experiment_status(port):
    '''get the status of an experiment'''
    result, response = check_rest_server_quick(port)
    if result:
        return json.loads(response.text).get('status')
    return None

def update_experiment():
SparkSnail's avatar
SparkSnail committed
52
53
54
55
56
57
58
    '''Update the experiment status in config file'''
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        return None
    for key in experiment_dict.keys():
        if isinstance(experiment_dict[key], dict):
59
            if experiment_dict[key].get('status') != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
60
61
62
                nni_config = Config(experiment_dict[key]['fileName'])
                rest_pid = nni_config.get_config('restServerPid')
                if not detect_process(rest_pid):
63
64
65
66
67
68
69
70
71
72
73
                    experiment_config.update_experiment(key, 'status', 'STOPPED')
                    continue
                rest_port = nni_config.get_config('restServerPort')
                startTime, endTime = get_experiment_time(rest_port)
                if startTime:
                    experiment_config.update_experiment(key, 'startTime', startTime)
                if endTime:
                    experiment_config.update_experiment(key, 'endTime', endTime)
                status = get_experiment_status(rest_port)
                if status:
                    experiment_config.update_experiment(key, 'status', status)
SparkSnail's avatar
SparkSnail committed
74

75
76
77
def check_experiment_id(args):
    '''check if the id is valid
    '''
78
    update_experiment()
79
80
81
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
82
        print_normal('There is no experiment running...')
chicm-ms's avatar
chicm-ms committed
83
        return None
84
    if not args.id:
85
86
        running_experiment_list = []
        for key in experiment_dict.keys():
SparkSnail's avatar
SparkSnail committed
87
            if isinstance(experiment_dict[key], dict):
88
                if experiment_dict[key].get('status') != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
89
90
91
92
                    running_experiment_list.append(key)
            elif isinstance(experiment_dict[key], list):
                # if the config file is old version, remove the configuration from file
                experiment_config.remove_experiment(key)
93
        if len(running_experiment_list) > 1:
94
            print_error('There are multiple experiments, please set the experiment id...')
95
96
97
            experiment_information = ""
            for key in running_experiment_list:
                experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
SparkSnail's avatar
SparkSnail committed
98
                experiment_dict[key]['port'], experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
99
100
101
102
            print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
            exit(1)
        elif not running_experiment_list:
            print_error('There is no experiment running!')
chicm-ms's avatar
chicm-ms committed
103
            return None
104
105
        else:
            return running_experiment_list[0]
106
107
108
109
110
    if experiment_dict.get(args.id):
        return args.id
    else:
        print_error('Id not correct!')
        return None
Deshui Yu's avatar
Deshui Yu committed
111

112
def parse_ids(args):
113
114
115
116
117
118
119
120
    '''Parse the arguments for nnictl stop
    1.If there is an id specified, return the corresponding id
    2.If there is no id specified, and there is an experiment running, return the id, or return Error
    3.If the id matches an experiment, nnictl will return the id.
    4.If the id ends with *, nnictl will match all ids matchs the regular
    5.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id
    6.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information
    '''
121
    update_experiment()
122
123
124
125
126
127
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print_normal('Experiment is not running...')
        return None
    result_list = []
128
129
    running_experiment_list = []
    for key in experiment_dict.keys():
SparkSnail's avatar
SparkSnail committed
130
        if isinstance(experiment_dict[key], dict):
131
            if experiment_dict[key].get('status') != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
132
133
134
135
                running_experiment_list.append(key)
        elif isinstance(experiment_dict[key], list):
            # if the config file is old version, remove the configuration from file
            experiment_config.remove_experiment(key)
136
    if not args.id:
137
        if len(running_experiment_list) > 1:
138
            print_error('There are multiple experiments, please set the experiment id...')
139
            experiment_information = ""
140
141
            for key in running_experiment_list:
                experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
SparkSnail's avatar
SparkSnail committed
142
                experiment_dict[key]['port'], experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
143
144
145
146
            print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
            exit(1)
        else:
            result_list = running_experiment_list
147
    elif args.id == 'all':
148
        result_list = running_experiment_list
149
    elif args.id.endswith('*'):
150
        for id in running_experiment_list:
151
152
            if id.startswith(args.id[:-1]):
                result_list.append(id)
153
    elif args.id in running_experiment_list:
154
155
        result_list.append(args.id)
    else:
156
        for id in running_experiment_list:
157
158
159
160
161
            if id.startswith(args.id):
                result_list.append(id)
        if len(result_list) > 1:
            print_error(args.id + ' is ambiguous, please choose ' + ' '.join(result_list) )
            return None
chicm-ms's avatar
chicm-ms committed
162
163
164
165
    if not result_list and args.id:
        print_error('There are no experiments matched, please set correct experiment id...')
    elif not result_list:
        print_error('There is no experiment running...')
166
167
    return result_list

168
169
170
def get_config_filename(args):
    '''get the file name of config file'''
    experiment_id = check_experiment_id(args)
chicm-ms's avatar
chicm-ms committed
171
172
173
    if experiment_id is None:
        print_error('Please set the experiment id!')
        exit(1)
174
175
176
177
178
179
180
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    return experiment_dict[experiment_id]['fileName']

def get_experiment_port(args):
    '''get the port of experiment'''
    experiment_id = check_experiment_id(args)
chicm-ms's avatar
chicm-ms committed
181
182
183
    if experiment_id is None:
        print_error('Please set the experiment id!')
        exit(1)
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    return experiment_dict[experiment_id]['port']

def convert_time_stamp_to_date(content):
    '''Convert time stamp to date time format'''
    start_time_stamp = content.get('startTime')
    end_time_stamp = content.get('endTime')
    if start_time_stamp:
        start_time = datetime.datetime.utcfromtimestamp(start_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S")
        content['startTime'] = str(start_time)
    if end_time_stamp:
        end_time = datetime.datetime.utcfromtimestamp(end_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S")
        content['endTime'] = str(end_time)
    return content

def check_rest(args):
    '''check if restful server is running'''
    nni_config = Config(get_config_filename(args))
    rest_port = nni_config.get_config('restServerPort')
    running, _ = check_rest_server_quick(rest_port)
    if not running:
        print_normal('Restful server is running...')
    else:
        print_normal('Restful server is not running...')

Deshui Yu's avatar
Deshui Yu committed
210
211
def stop_experiment(args):
    '''Stop the experiment which is running'''
212
213
214
215
216
217
    experiment_id_list = parse_ids(args)
    if experiment_id_list:
        experiment_config = Experiments()
        experiment_dict = experiment_config.get_all_experiments()
        for experiment_id in experiment_id_list:
            print_normal('Stoping experiment %s' % experiment_id)
218
            nni_config = Config(experiment_dict[experiment_id]['fileName'])
219
220
221
            rest_port = nni_config.get_config('restServerPort')
            rest_pid = nni_config.get_config('restServerPid')
            if rest_pid:
222
                stop_rest_cmds = ['kill', str(rest_pid)]
SparkSnail's avatar
SparkSnail committed
223
224
225
226
227
228
229
230
231
232
                call(stop_rest_cmds)
                tensorboard_pid_list = nni_config.get_config('tensorboardPidList')
                if tensorboard_pid_list:
                    for tensorboard_pid in tensorboard_pid_list:
                        try:
                            cmds = ['kill', '-9', str(tensorboard_pid)]
                            call(cmds)
                        except Exception as exception:
                            print_error(exception)
                    nni_config.set_config('tensorboardPidList', [])
SparkSnail's avatar
SparkSnail committed
233
            print_normal('Stop experiment success!')
234
            experiment_config.update_experiment(experiment_id, 'status', 'STOPPED')
235
236
            time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
            experiment_config.update_experiment(experiment_id, 'endTime', str(time_now))
Deshui Yu's avatar
Deshui Yu committed
237
238
239

def trial_ls(args):
    '''List trial'''
240
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
241
242
243
244
245
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
246
247
    running, response = check_rest_server_quick(rest_port)
    if running:
248
        response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT)
249
        if response and check_response(response):
250
            content = json.loads(response.text)
251
            for index, value in enumerate(content):
252
253
                content[index] = convert_time_stamp_to_date(value)
            print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
Deshui Yu's avatar
Deshui Yu committed
254
255
256
257
258
259
260
        else:
            print_error('List trial failed...')
    else:
        print_error('Restful server is not running...')

def trial_kill(args):
    '''List trial'''
261
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
262
263
264
265
266
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
267
268
    running, _ = check_rest_server_quick(rest_port)
    if running:
269
        response = rest_delete(trial_job_id_url(rest_port, args.id), REST_TIME_OUT)
270
        if response and check_response(response):
Deshui Yu's avatar
Deshui Yu committed
271
272
273
274
275
276
277
278
            print(response.text)
        else:
            print_error('Kill trial job failed...')
    else:
        print_error('Restful server is not running...')

def list_experiment(args):
    '''Get experiment information'''
279
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
280
281
282
283
284
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
285
286
    running, _ = check_rest_server_quick(rest_port)
    if running:
287
        response = rest_get(experiment_url(rest_port), REST_TIME_OUT)
288
        if response and check_response(response):
289
290
            content = convert_time_stamp_to_date(json.loads(response.text))
            print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
Deshui Yu's avatar
Deshui Yu committed
291
292
293
294
295
        else:
            print_error('List experiment failed...')
    else:
        print_error('Restful server is not running...')

296
297
def experiment_status(args):
    '''Show the status of experiment'''
298
    nni_config = Config(get_config_filename(args))
299
300
301
302
303
304
305
    rest_port = nni_config.get_config('restServerPort')
    result, response = check_rest_server_quick(rest_port)
    if not result:
        print_normal('Restful server is not running...')
    else:
        print(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))

Deshui Yu's avatar
Deshui Yu committed
306
307
308
309
310
311
312
313
314
315
def get_log_content(file_name, cmds):
    '''use cmds to read config content'''
    if os.path.exists(file_name):
        rest = check_output(cmds)
        print(rest.decode('utf-8'))
    else:
        print_normal('NULL!')

def log_internal(args, filetype):
    '''internal function to call get_log_content'''
316
    file_name = get_config_filename(args)
Deshui Yu's avatar
Deshui Yu committed
317
    if filetype == 'stdout':
318
        file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stdout')
Deshui Yu's avatar
Deshui Yu committed
319
    else:
320
        file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stderr')
Deshui Yu's avatar
Deshui Yu committed
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
    if args.head:
        get_log_content(file_full_path, ['head', '-' + str(args.head), file_full_path])
    elif args.tail:
        get_log_content(file_full_path, ['tail', '-' + str(args.tail), file_full_path])
    elif args.path:
        print_normal('The path of stdout file is: ' + file_full_path)
    else:
        get_log_content(file_full_path, ['cat', file_full_path])

def log_stdout(args):
    '''get stdout log'''
    log_internal(args, 'stdout')

def log_stderr(args):
    '''get stderr log'''
    log_internal(args, 'stderr')

338
339
340
def log_trial(args):
    ''''get trial log path'''
    trial_id_path_dict = {}
341
    nni_config = Config(get_config_filename(args))
342
343
344
345
346
347
348
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
    running, response = check_rest_server_quick(rest_port)
    if running:
349
        response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT)
350
351
352
353
354
355
        if response and check_response(response):
            content = json.loads(response.text)
            for trial in content:
                trial_id_path_dict[trial['id']] = trial['logPath']
    else:
        print_error('Restful server is not running...')
goooxu's avatar
goooxu committed
356
        exit(1)
357
358
359
360
    if args.id:
        if args.trial_id:
            if trial_id_path_dict.get(args.trial_id):
                print_normal('id:' + args.trial_id + ' path:' + trial_id_path_dict[args.trial_id])
361
362
363
            else:
                print_error('trial id is not valid!')
                exit(1)
364
        else:
365
            print_error('please specific the trial id!')
goooxu's avatar
goooxu committed
366
            exit(1)
367
    else:
368
        for key in trial_id_path_dict:
369
370
            print('id:' + key + ' path:' + trial_id_path_dict[key])

Deshui Yu's avatar
Deshui Yu committed
371
372
def get_config(args):
    '''get config info'''
373
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
374
    print(nni_config.get_all_config())
375
376
377

def webui_url(args):
    '''show the url of web ui'''
378
    nni_config = Config(get_config_filename(args))
379
380
    print_normal('{0} {1}'.format('Web UI url:', ' '.join(nni_config.get_config('webuiUrl'))))

381
382
def experiment_list(args):
    '''get the information of all experiments'''
383
384
385
386
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print('There is no experiment running...')
387
        exit(1)
388
    update_experiment()
389
390
391
392
    experiment_id_list = []
    if args.all and args.all == 'all':
        for key in experiment_dict.keys():
            experiment_id_list.append(key)
393
394
    else:
        for key in experiment_dict.keys():
395
            if experiment_dict[key]['status'] != 'STOPPED':
396
397
398
399
400
                experiment_id_list.append(key)
        if not experiment_id_list:
            print_warning('There is no experiment running...\nYou can use \'nnictl experiment list all\' to list all stopped experiments!')
    experiment_information = ""
    for key in experiment_id_list:
401
        
402
        experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], experiment_dict[key]['port'],\
SparkSnail's avatar
SparkSnail committed
403
        experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
404
    print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
SparkSnail's avatar
SparkSnail committed
405

SparkSnail's avatar
SparkSnail committed
406
407
408
409
def get_time_interval(time1, time2):
    '''get the interval of two times'''
    try:
        #convert time to timestamp
410
411
        time1 = time.mktime(time.strptime(time1, '%Y/%m/%d %H:%M:%S'))
        time2 = time.mktime(time.strptime(time2, '%Y/%m/%d %H:%M:%S'))
SparkSnail's avatar
SparkSnail committed
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
        seconds = (datetime.datetime.fromtimestamp(time2) - datetime.datetime.fromtimestamp(time1)).seconds
        #convert seconds to day:hour:minute:second
        days = seconds / 86400
        seconds %= 86400
        hours = seconds / 3600
        seconds %= 3600
        minutes = seconds / 60
        seconds %= 60
        return '%dd %dh %dm %ds' % (days, hours, minutes, seconds)
    except:
        return 'N/A'

def show_experiment_info():
    '''show experiment information in monitor'''
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print('There is no experiment running...')
        exit(1)
431
    update_experiment()
SparkSnail's avatar
SparkSnail committed
432
433
    experiment_id_list = []
    for key in experiment_dict.keys():
434
        if experiment_dict[key]['status'] != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
435
436
437
438
439
440
            experiment_id_list.append(key)
    if not experiment_id_list:
        print_warning('There is no experiment running...')
        return
    for key in experiment_id_list:
        print(EXPERIMENT_MONITOR_INFO % (key, experiment_dict[key]['status'], experiment_dict[key]['port'], \
441
             experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], get_time_interval(experiment_dict[key]['startTime'], experiment_dict[key]['endTime'])))
SparkSnail's avatar
SparkSnail committed
442
443
444
        print(TRIAL_MONITOR_HEAD)
        running, response = check_rest_server_quick(experiment_dict[key]['port'])
        if running:
445
            response = rest_get(trial_jobs_url(experiment_dict[key]['port']), REST_TIME_OUT)
SparkSnail's avatar
SparkSnail committed
446
447
            if response and check_response(response):
                content = json.loads(response.text)
448
                for index, value in enumerate(content):
SparkSnail's avatar
SparkSnail committed
449
450
451
452
453
454
455
456
457
458
459
460
                    content[index] = convert_time_stamp_to_date(value)
                    print(TRIAL_MONITOR_CONTENT % (content[index].get('id'), content[index].get('startTime'), content[index].get('endTime'), content[index].get('status')))
        print(TRIAL_MONITOR_TAIL)

def monitor_experiment(args):
    '''monitor the experiment'''
    if args.time <= 0:
        print_error('please input a positive integer as time interval, the unit is second.')
        exit(1)
    while True:
        try:
            os.system('clear')
461
            update_experiment()
SparkSnail's avatar
SparkSnail committed
462
463
464
465
466
467
468
            show_experiment_info()
            time.sleep(args.time)
        except KeyboardInterrupt:
            exit(0)
        except Exception as exception:
            print_error(exception)
            exit(1)
Yan Ni's avatar
Yan Ni committed
469
470
471
472
473
474
475
476


def parse_trial_data(content):
    """output: List[Dict]"""
    trial_records = []
    for trial_data in content:
        for phase_i in range(len(trial_data['hyperParameters'])):
            hparam = json.loads(trial_data['hyperParameters'][phase_i])['parameters']
Yan Ni's avatar
Yan Ni committed
477
            hparam['id'] = trial_data['id']
Yan Ni's avatar
Yan Ni committed
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
            if 'finalMetricData' in trial_data.keys() and phase_i < len(trial_data['finalMetricData']):
                reward = json.loads(trial_data['finalMetricData'][phase_i]['data'])
                if isinstance(reward, (float, int)):
                    dict_tmp = {**hparam, **{'reward': reward}}
                elif isinstance(reward, dict):
                    dict_tmp = {**hparam, **reward}
                else:
                    raise ValueError("Invalid finalMetricsData format: {}/{}".format(type(reward), reward))
            else:
                dict_tmp = hparam
            trial_records.append(dict_tmp)
    return trial_records

def export_trials_data(args):
    """export experiment metadata to csv
    """
    nni_config = Config(get_config_filename(args))
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
    running, response = check_rest_server_quick(rest_port)
    if running:
        response = rest_get(trial_jobs_url(rest_port), 20)
        if response is not None and check_response(response):
            content = json.loads(response.text)
            # dframe = pd.DataFrame.from_records([parse_trial_data(t_data) for t_data in content])
            # dframe.to_csv(args.csv_path, sep='\t')
            records = parse_trial_data(content)
508
509
510
511
512
513
514
515
516
517
518
519
520
            if args.type == 'json':
                json_records = []
                for trial in records:
                    value = trial.pop('reward', None)
                    trial_id =  trial.pop('id', None)
                    json_records.append({'parameter': trial, 'value': value, 'id': trial_id})
            with open(args.path, 'w') as file:
                if args.type == 'csv':
                    writer = csv.DictWriter(file, set.union(*[set(r.keys()) for r in records]))
                    writer.writeheader()
                    writer.writerows(records)
                else:
                    json.dump(json_records, file)
Yan Ni's avatar
Yan Ni committed
521
522
523
524
        else:
            print_error('Export failed...')
    else:
        print_error('Restful server is not Running')