nnictl_utils.py 22.3 KB
Newer Older
Deshui Yu's avatar
Deshui Yu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# Copyright (c) Microsoft Corporation
# All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge,
# to any person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Yan Ni's avatar
Yan Ni committed
21
import csv
Deshui Yu's avatar
Deshui Yu committed
22
23
24
import os
import psutil
import json
25
import datetime
26
import time
Yan Ni's avatar
Yan Ni committed
27

Deshui Yu's avatar
Deshui Yu committed
28
from subprocess import call, check_output
29
from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response
30
from .config_utils import Config, Experiments
Deshui Yu's avatar
Deshui Yu committed
31
from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url
SparkSnail's avatar
SparkSnail committed
32
from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \
33
     EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT
34
from .common_utils import print_normal, print_error, print_warning, detect_process
Deshui Yu's avatar
Deshui Yu committed
35

36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
def get_experiment_time(port):
    '''get the startTime and endTime of an experiment'''
    response = rest_get(experiment_url(port), REST_TIME_OUT)
    if response and check_response(response):
        content = convert_time_stamp_to_date(json.loads(response.text))
        return content.get('startTime'), content.get('endTime')
    return None, None

def get_experiment_status(port):
    '''get the status of an experiment'''
    result, response = check_rest_server_quick(port)
    if result:
        return json.loads(response.text).get('status')
    return None

def update_experiment():
SparkSnail's avatar
SparkSnail committed
52
53
54
55
56
57
58
    '''Update the experiment status in config file'''
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        return None
    for key in experiment_dict.keys():
        if isinstance(experiment_dict[key], dict):
59
            if experiment_dict[key].get('status') != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
60
61
62
                nni_config = Config(experiment_dict[key]['fileName'])
                rest_pid = nni_config.get_config('restServerPid')
                if not detect_process(rest_pid):
63
64
65
66
67
68
69
70
71
72
73
                    experiment_config.update_experiment(key, 'status', 'STOPPED')
                    continue
                rest_port = nni_config.get_config('restServerPort')
                startTime, endTime = get_experiment_time(rest_port)
                if startTime:
                    experiment_config.update_experiment(key, 'startTime', startTime)
                if endTime:
                    experiment_config.update_experiment(key, 'endTime', endTime)
                status = get_experiment_status(rest_port)
                if status:
                    experiment_config.update_experiment(key, 'status', status)
SparkSnail's avatar
SparkSnail committed
74

75
76
77
def check_experiment_id(args):
    '''check if the id is valid
    '''
78
    update_experiment()
79
80
81
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
82
        print_normal('There is no experiment running...')
chicm-ms's avatar
chicm-ms committed
83
        return None
84
    if not args.id:
85
86
        running_experiment_list = []
        for key in experiment_dict.keys():
SparkSnail's avatar
SparkSnail committed
87
            if isinstance(experiment_dict[key], dict):
88
                if experiment_dict[key].get('status') != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
89
90
91
92
                    running_experiment_list.append(key)
            elif isinstance(experiment_dict[key], list):
                # if the config file is old version, remove the configuration from file
                experiment_config.remove_experiment(key)
93
        if len(running_experiment_list) > 1:
94
            print_error('There are multiple experiments, please set the experiment id...')
95
96
97
            experiment_information = ""
            for key in running_experiment_list:
                experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
SparkSnail's avatar
SparkSnail committed
98
                experiment_dict[key]['port'], experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
99
100
101
102
            print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
            exit(1)
        elif not running_experiment_list:
            print_error('There is no experiment running!')
chicm-ms's avatar
chicm-ms committed
103
            return None
104
105
        else:
            return running_experiment_list[0]
106
107
108
109
110
111
112
113
    if hasattr(args, "experiment"):
        if experiment_dict.get(args.experiment):
            return args.experiment
    elif hasattr(args, "id"):
        if experiment_dict.get(args.id):
            return args.id
    print_error('Id not correct!')
    return None
Deshui Yu's avatar
Deshui Yu committed
114

115
def parse_ids(args):
116
117
118
119
120
121
122
123
    '''Parse the arguments for nnictl stop
    1.If there is an id specified, return the corresponding id
    2.If there is no id specified, and there is an experiment running, return the id, or return Error
    3.If the id matches an experiment, nnictl will return the id.
    4.If the id ends with *, nnictl will match all ids matchs the regular
    5.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id
    6.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information
    '''
124
    update_experiment()
125
126
127
128
129
130
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print_normal('Experiment is not running...')
        return None
    result_list = []
131
132
    running_experiment_list = []
    for key in experiment_dict.keys():
SparkSnail's avatar
SparkSnail committed
133
        if isinstance(experiment_dict[key], dict):
134
            if experiment_dict[key].get('status') != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
135
136
137
138
                running_experiment_list.append(key)
        elif isinstance(experiment_dict[key], list):
            # if the config file is old version, remove the configuration from file
            experiment_config.remove_experiment(key)
139
    if not args.id:
140
        if len(running_experiment_list) > 1:
141
            print_error('There are multiple experiments, please set the experiment id...')
142
            experiment_information = ""
143
144
            for key in running_experiment_list:
                experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
SparkSnail's avatar
SparkSnail committed
145
                experiment_dict[key]['port'], experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
146
147
148
149
            print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
            exit(1)
        else:
            result_list = running_experiment_list
150
    elif args.id == 'all':
151
        result_list = running_experiment_list
152
    elif args.id.endswith('*'):
153
        for id in running_experiment_list:
154
155
            if id.startswith(args.id[:-1]):
                result_list.append(id)
156
    elif args.id in running_experiment_list:
157
158
        result_list.append(args.id)
    else:
159
        for id in running_experiment_list:
160
161
162
163
164
            if id.startswith(args.id):
                result_list.append(id)
        if len(result_list) > 1:
            print_error(args.id + ' is ambiguous, please choose ' + ' '.join(result_list) )
            return None
chicm-ms's avatar
chicm-ms committed
165
166
167
168
    if not result_list and args.id:
        print_error('There are no experiments matched, please set correct experiment id...')
    elif not result_list:
        print_error('There is no experiment running...')
169
170
    return result_list

171
172
173
def get_config_filename(args):
    '''get the file name of config file'''
    experiment_id = check_experiment_id(args)
chicm-ms's avatar
chicm-ms committed
174
175
176
    if experiment_id is None:
        print_error('Please set the experiment id!')
        exit(1)
177
178
179
180
181
182
183
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    return experiment_dict[experiment_id]['fileName']

def get_experiment_port(args):
    '''get the port of experiment'''
    experiment_id = check_experiment_id(args)
chicm-ms's avatar
chicm-ms committed
184
185
186
    if experiment_id is None:
        print_error('Please set the experiment id!')
        exit(1)
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    return experiment_dict[experiment_id]['port']

def convert_time_stamp_to_date(content):
    '''Convert time stamp to date time format'''
    start_time_stamp = content.get('startTime')
    end_time_stamp = content.get('endTime')
    if start_time_stamp:
        start_time = datetime.datetime.utcfromtimestamp(start_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S")
        content['startTime'] = str(start_time)
    if end_time_stamp:
        end_time = datetime.datetime.utcfromtimestamp(end_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S")
        content['endTime'] = str(end_time)
    return content

def check_rest(args):
    '''check if restful server is running'''
    nni_config = Config(get_config_filename(args))
    rest_port = nni_config.get_config('restServerPort')
    running, _ = check_rest_server_quick(rest_port)
    if not running:
        print_normal('Restful server is running...')
    else:
        print_normal('Restful server is not running...')

Deshui Yu's avatar
Deshui Yu committed
213
214
def stop_experiment(args):
    '''Stop the experiment which is running'''
215
216
217
218
219
220
    experiment_id_list = parse_ids(args)
    if experiment_id_list:
        experiment_config = Experiments()
        experiment_dict = experiment_config.get_all_experiments()
        for experiment_id in experiment_id_list:
            print_normal('Stoping experiment %s' % experiment_id)
221
            nni_config = Config(experiment_dict[experiment_id]['fileName'])
222
223
224
            rest_port = nni_config.get_config('restServerPort')
            rest_pid = nni_config.get_config('restServerPid')
            if rest_pid:
225
                stop_rest_cmds = ['kill', str(rest_pid)]
SparkSnail's avatar
SparkSnail committed
226
227
228
229
230
231
232
233
234
235
                call(stop_rest_cmds)
                tensorboard_pid_list = nni_config.get_config('tensorboardPidList')
                if tensorboard_pid_list:
                    for tensorboard_pid in tensorboard_pid_list:
                        try:
                            cmds = ['kill', '-9', str(tensorboard_pid)]
                            call(cmds)
                        except Exception as exception:
                            print_error(exception)
                    nni_config.set_config('tensorboardPidList', [])
SparkSnail's avatar
SparkSnail committed
236
            print_normal('Stop experiment success!')
237
            experiment_config.update_experiment(experiment_id, 'status', 'STOPPED')
238
239
            time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
            experiment_config.update_experiment(experiment_id, 'endTime', str(time_now))
Deshui Yu's avatar
Deshui Yu committed
240
241
242

def trial_ls(args):
    '''List trial'''
243
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
244
245
246
247
248
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
249
250
    running, response = check_rest_server_quick(rest_port)
    if running:
251
        response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT)
252
        if response and check_response(response):
253
            content = json.loads(response.text)
254
            for index, value in enumerate(content):
255
256
                content[index] = convert_time_stamp_to_date(value)
            print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
Deshui Yu's avatar
Deshui Yu committed
257
258
259
260
261
262
263
        else:
            print_error('List trial failed...')
    else:
        print_error('Restful server is not running...')

def trial_kill(args):
    '''List trial'''
264
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
265
266
267
268
269
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
270
271
    running, _ = check_rest_server_quick(rest_port)
    if running:
272
        response = rest_delete(trial_job_id_url(rest_port, args.id), REST_TIME_OUT)
273
        if response and check_response(response):
Deshui Yu's avatar
Deshui Yu committed
274
275
276
277
278
279
280
281
            print(response.text)
        else:
            print_error('Kill trial job failed...')
    else:
        print_error('Restful server is not running...')

def list_experiment(args):
    '''Get experiment information'''
282
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
283
284
285
286
287
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
288
289
    running, _ = check_rest_server_quick(rest_port)
    if running:
290
        response = rest_get(experiment_url(rest_port), REST_TIME_OUT)
291
        if response and check_response(response):
292
293
            content = convert_time_stamp_to_date(json.loads(response.text))
            print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
Deshui Yu's avatar
Deshui Yu committed
294
295
296
297
298
        else:
            print_error('List experiment failed...')
    else:
        print_error('Restful server is not running...')

299
300
def experiment_status(args):
    '''Show the status of experiment'''
301
    nni_config = Config(get_config_filename(args))
302
303
304
305
306
307
308
    rest_port = nni_config.get_config('restServerPort')
    result, response = check_rest_server_quick(rest_port)
    if not result:
        print_normal('Restful server is not running...')
    else:
        print(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))

Deshui Yu's avatar
Deshui Yu committed
309
310
311
312
313
314
315
316
317
318
def get_log_content(file_name, cmds):
    '''use cmds to read config content'''
    if os.path.exists(file_name):
        rest = check_output(cmds)
        print(rest.decode('utf-8'))
    else:
        print_normal('NULL!')

def log_internal(args, filetype):
    '''internal function to call get_log_content'''
319
    file_name = get_config_filename(args)
Deshui Yu's avatar
Deshui Yu committed
320
    if filetype == 'stdout':
321
        file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stdout')
Deshui Yu's avatar
Deshui Yu committed
322
    else:
323
        file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stderr')
Deshui Yu's avatar
Deshui Yu committed
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
    if args.head:
        get_log_content(file_full_path, ['head', '-' + str(args.head), file_full_path])
    elif args.tail:
        get_log_content(file_full_path, ['tail', '-' + str(args.tail), file_full_path])
    elif args.path:
        print_normal('The path of stdout file is: ' + file_full_path)
    else:
        get_log_content(file_full_path, ['cat', file_full_path])

def log_stdout(args):
    '''get stdout log'''
    log_internal(args, 'stdout')

def log_stderr(args):
    '''get stderr log'''
    log_internal(args, 'stderr')

341
342
343
def log_trial(args):
    ''''get trial log path'''
    trial_id_path_dict = {}
344
    nni_config = Config(get_config_filename(args))
345
346
347
348
349
350
351
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
    running, response = check_rest_server_quick(rest_port)
    if running:
352
        response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT)
353
354
355
356
357
358
        if response and check_response(response):
            content = json.loads(response.text)
            for trial in content:
                trial_id_path_dict[trial['id']] = trial['logPath']
    else:
        print_error('Restful server is not running...')
goooxu's avatar
goooxu committed
359
        exit(1)
360
361
362
363
364
365
366
    if args.experiment:
        if args.id:
            if trial_id_path_dict.get(args.id):
                print('id:' + args.id + ' path:' + trial_id_path_dict[args.id])
            else:
                print_error('trial id is not valid!')
                exit(1)
367
        else:
368
369
            print_error('please specific the trial id!')
            print_error("trial id list in this experiment: " + str(list(trial_id_path_dict.keys())))
goooxu's avatar
goooxu committed
370
            exit(1)
371
    else:
372
        for key in trial_id_path_dict:
373
374
            print('id:' + key + ' path:' + trial_id_path_dict[key])

Deshui Yu's avatar
Deshui Yu committed
375
376
def get_config(args):
    '''get config info'''
377
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
378
    print(nni_config.get_all_config())
379
380
381

def webui_url(args):
    '''show the url of web ui'''
382
    nni_config = Config(get_config_filename(args))
383
384
    print_normal('{0} {1}'.format('Web UI url:', ' '.join(nni_config.get_config('webuiUrl'))))

385
386
def experiment_list(args):
    '''get the information of all experiments'''
387
388
389
390
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print('There is no experiment running...')
391
        exit(1)
392
    update_experiment()
393
394
395
396
    experiment_id_list = []
    if args.all and args.all == 'all':
        for key in experiment_dict.keys():
            experiment_id_list.append(key)
397
398
    else:
        for key in experiment_dict.keys():
399
            if experiment_dict[key]['status'] != 'STOPPED':
400
401
402
403
404
                experiment_id_list.append(key)
        if not experiment_id_list:
            print_warning('There is no experiment running...\nYou can use \'nnictl experiment list all\' to list all stopped experiments!')
    experiment_information = ""
    for key in experiment_id_list:
405
        
406
        experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], experiment_dict[key]['port'],\
SparkSnail's avatar
SparkSnail committed
407
        experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
408
    print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
SparkSnail's avatar
SparkSnail committed
409

SparkSnail's avatar
SparkSnail committed
410
411
412
413
def get_time_interval(time1, time2):
    '''get the interval of two times'''
    try:
        #convert time to timestamp
414
415
        time1 = time.mktime(time.strptime(time1, '%Y/%m/%d %H:%M:%S'))
        time2 = time.mktime(time.strptime(time2, '%Y/%m/%d %H:%M:%S'))
SparkSnail's avatar
SparkSnail committed
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
        seconds = (datetime.datetime.fromtimestamp(time2) - datetime.datetime.fromtimestamp(time1)).seconds
        #convert seconds to day:hour:minute:second
        days = seconds / 86400
        seconds %= 86400
        hours = seconds / 3600
        seconds %= 3600
        minutes = seconds / 60
        seconds %= 60
        return '%dd %dh %dm %ds' % (days, hours, minutes, seconds)
    except:
        return 'N/A'

def show_experiment_info():
    '''show experiment information in monitor'''
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print('There is no experiment running...')
        exit(1)
435
    update_experiment()
SparkSnail's avatar
SparkSnail committed
436
437
    experiment_id_list = []
    for key in experiment_dict.keys():
438
        if experiment_dict[key]['status'] != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
439
440
441
442
443
444
            experiment_id_list.append(key)
    if not experiment_id_list:
        print_warning('There is no experiment running...')
        return
    for key in experiment_id_list:
        print(EXPERIMENT_MONITOR_INFO % (key, experiment_dict[key]['status'], experiment_dict[key]['port'], \
445
             experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], get_time_interval(experiment_dict[key]['startTime'], experiment_dict[key]['endTime'])))
SparkSnail's avatar
SparkSnail committed
446
447
448
        print(TRIAL_MONITOR_HEAD)
        running, response = check_rest_server_quick(experiment_dict[key]['port'])
        if running:
449
            response = rest_get(trial_jobs_url(experiment_dict[key]['port']), REST_TIME_OUT)
SparkSnail's avatar
SparkSnail committed
450
451
            if response and check_response(response):
                content = json.loads(response.text)
452
                for index, value in enumerate(content):
SparkSnail's avatar
SparkSnail committed
453
454
455
456
457
458
459
460
461
462
463
464
                    content[index] = convert_time_stamp_to_date(value)
                    print(TRIAL_MONITOR_CONTENT % (content[index].get('id'), content[index].get('startTime'), content[index].get('endTime'), content[index].get('status')))
        print(TRIAL_MONITOR_TAIL)

def monitor_experiment(args):
    '''monitor the experiment'''
    if args.time <= 0:
        print_error('please input a positive integer as time interval, the unit is second.')
        exit(1)
    while True:
        try:
            os.system('clear')
465
            update_experiment()
SparkSnail's avatar
SparkSnail committed
466
467
468
469
470
471
472
            show_experiment_info()
            time.sleep(args.time)
        except KeyboardInterrupt:
            exit(0)
        except Exception as exception:
            print_error(exception)
            exit(1)
Yan Ni's avatar
Yan Ni committed
473
474
475
476
477
478
479
480


def parse_trial_data(content):
    """output: List[Dict]"""
    trial_records = []
    for trial_data in content:
        for phase_i in range(len(trial_data['hyperParameters'])):
            hparam = json.loads(trial_data['hyperParameters'][phase_i])['parameters']
Yan Ni's avatar
Yan Ni committed
481
            hparam['id'] = trial_data['id']
Yan Ni's avatar
Yan Ni committed
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
            if 'finalMetricData' in trial_data.keys() and phase_i < len(trial_data['finalMetricData']):
                reward = json.loads(trial_data['finalMetricData'][phase_i]['data'])
                if isinstance(reward, (float, int)):
                    dict_tmp = {**hparam, **{'reward': reward}}
                elif isinstance(reward, dict):
                    dict_tmp = {**hparam, **reward}
                else:
                    raise ValueError("Invalid finalMetricsData format: {}/{}".format(type(reward), reward))
            else:
                dict_tmp = hparam
            trial_records.append(dict_tmp)
    return trial_records

def export_trials_data(args):
    """export experiment metadata to csv
    """
    nni_config = Config(get_config_filename(args))
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
    running, response = check_rest_server_quick(rest_port)
    if running:
        response = rest_get(trial_jobs_url(rest_port), 20)
        if response is not None and check_response(response):
            content = json.loads(response.text)
            # dframe = pd.DataFrame.from_records([parse_trial_data(t_data) for t_data in content])
            # dframe.to_csv(args.csv_path, sep='\t')
            records = parse_trial_data(content)
            with open(args.csv_path, 'w') as f_csv:
                writer = csv.DictWriter(f_csv, set.union(*[set(r.keys()) for r in records]))
                writer.writeheader()
                writer.writerows(records)
        else:
            print_error('Export failed...')
    else:
        print_error('Restful server is not Running')