nnictl_utils.py 30.2 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
Deshui Yu's avatar
Deshui Yu committed
3

Yan Ni's avatar
Yan Ni committed
4
import csv
Deshui Yu's avatar
Deshui Yu committed
5
6
import os
import json
7
import time
SparkSnail's avatar
SparkSnail committed
8
9
import re
import shutil
chicm-ms's avatar
chicm-ms committed
10
11
12
from datetime import datetime, timezone
from pathlib import Path
from pyhdfs import HdfsClient
13
from nni_annotation import expand_annotations
14
from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response
15
from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url
16
from .config_utils import Config, Experiments
SparkSnail's avatar
SparkSnail committed
17
from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \
18
     EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT
SparkSnail's avatar
SparkSnail committed
19
from .common_utils import print_normal, print_error, print_warning, detect_process, get_yml_content
20
from .command_utils import check_output_command, kill_command
SparkSnail's avatar
SparkSnail committed
21
from .ssh_utils import create_ssh_sftp_client, remove_remote_directory
Deshui Yu's avatar
Deshui Yu committed
22

23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
def get_experiment_time(port):
    '''get the startTime and endTime of an experiment'''
    response = rest_get(experiment_url(port), REST_TIME_OUT)
    if response and check_response(response):
        content = convert_time_stamp_to_date(json.loads(response.text))
        return content.get('startTime'), content.get('endTime')
    return None, None

def get_experiment_status(port):
    '''get the status of an experiment'''
    result, response = check_rest_server_quick(port)
    if result:
        return json.loads(response.text).get('status')
    return None

def update_experiment():
SparkSnail's avatar
SparkSnail committed
39
40
41
42
43
44
45
    '''Update the experiment status in config file'''
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        return None
    for key in experiment_dict.keys():
        if isinstance(experiment_dict[key], dict):
46
            if experiment_dict[key].get('status') != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
47
48
49
                nni_config = Config(experiment_dict[key]['fileName'])
                rest_pid = nni_config.get_config('restServerPid')
                if not detect_process(rest_pid):
50
51
52
53
54
55
56
57
58
59
60
                    experiment_config.update_experiment(key, 'status', 'STOPPED')
                    continue
                rest_port = nni_config.get_config('restServerPort')
                startTime, endTime = get_experiment_time(rest_port)
                if startTime:
                    experiment_config.update_experiment(key, 'startTime', startTime)
                if endTime:
                    experiment_config.update_experiment(key, 'endTime', endTime)
                status = get_experiment_status(rest_port)
                if status:
                    experiment_config.update_experiment(key, 'status', status)
SparkSnail's avatar
SparkSnail committed
61

SparkSnail's avatar
SparkSnail committed
62
def check_experiment_id(args, update=True):
63
64
    '''check if the id is valid
    '''
SparkSnail's avatar
SparkSnail committed
65
66
    if update:
        update_experiment()
67
68
69
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
70
        print_normal('There is no experiment running...')
chicm-ms's avatar
chicm-ms committed
71
        return None
72
    if not args.id:
73
74
        running_experiment_list = []
        for key in experiment_dict.keys():
SparkSnail's avatar
SparkSnail committed
75
            if isinstance(experiment_dict[key], dict):
76
                if experiment_dict[key].get('status') != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
77
78
79
80
                    running_experiment_list.append(key)
            elif isinstance(experiment_dict[key], list):
                # if the config file is old version, remove the configuration from file
                experiment_config.remove_experiment(key)
81
        if len(running_experiment_list) > 1:
82
            print_error('There are multiple experiments, please set the experiment id...')
83
84
            experiment_information = ""
            for key in running_experiment_list:
85
86
87
88
89
90
91
                experiment_information += EXPERIMENT_DETAIL_FORMAT % (key,
                                                                      experiment_dict[key].get('experimentName', 'N/A'),
                                                                      experiment_dict[key]['status'],
                                                                      experiment_dict[key]['port'],
                                                                      experiment_dict[key].get('platform'),
                                                                      experiment_dict[key]['startTime'],
                                                                      experiment_dict[key]['endTime'])
92
93
94
            print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
            exit(1)
        elif not running_experiment_list:
SparkSnail's avatar
SparkSnail committed
95
            print_error('There is no experiment running.')
chicm-ms's avatar
chicm-ms committed
96
            return None
97
98
        else:
            return running_experiment_list[0]
99
100
101
    if experiment_dict.get(args.id):
        return args.id
    else:
SparkSnail's avatar
SparkSnail committed
102
        print_error('Id not correct.')
103
        return None
Deshui Yu's avatar
Deshui Yu committed
104

105
def parse_ids(args):
106
    '''Parse the arguments for nnictl stop
107
108
109
110
111
112
113
114
    1.If port is provided and id is not specified, return the id who owns the port
    2.If both port and id are provided, return the id if it owns the port, otherwise fail
    3.If there is an id specified, return the corresponding id
    4.If there is no id specified, and there is an experiment running, return the id, or return Error
    5.If the id matches an experiment, nnictl will return the id.
    6.If the id ends with *, nnictl will match all ids matchs the regular
    7.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id
    8.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information
115
    '''
116
    update_experiment()
117
118
119
120
121
122
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print_normal('Experiment is not running...')
        return None
    result_list = []
123
124
    running_experiment_list = []
    for key in experiment_dict.keys():
SparkSnail's avatar
SparkSnail committed
125
        if isinstance(experiment_dict[key], dict):
126
            if experiment_dict[key].get('status') != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
127
128
129
130
                running_experiment_list.append(key)
        elif isinstance(experiment_dict[key], list):
            # if the config file is old version, remove the configuration from file
            experiment_config.remove_experiment(key)
131
132
    if args.all:
        return running_experiment_list
133
134
135
136
137
138
139
140
    if args.port is not None:
        for key in running_experiment_list:
            if str(experiment_dict[key]['port']) == args.port:
                result_list.append(key)
        if args.id and result_list and args.id != result_list[0]:
            print_error('Experiment id and resful server port not match')
            exit(1)
    elif not args.id:
141
        if len(running_experiment_list) > 1:
142
            print_error('There are multiple experiments, please set the experiment id...')
143
            experiment_information = ""
144
            for key in running_experiment_list:
145
146
147
148
149
150
151
                experiment_information += EXPERIMENT_DETAIL_FORMAT % (key,
                                                                      experiment_dict[key].get('experimentName', 'N/A'),
                                                                      experiment_dict[key]['status'],
                                                                      experiment_dict[key]['port'],
                                                                      experiment_dict[key].get('platform'),
                                                                      experiment_dict[key]['startTime'],
                                                                      experiment_dict[key]['endTime'])
152
153
154
155
            print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
            exit(1)
        else:
            result_list = running_experiment_list
156
    elif args.id.endswith('*'):
chicm-ms's avatar
chicm-ms committed
157
158
159
        for expId in running_experiment_list:
            if expId.startswith(args.id[:-1]):
                result_list.append(expId)
160
    elif args.id in running_experiment_list:
161
162
        result_list.append(args.id)
    else:
chicm-ms's avatar
chicm-ms committed
163
164
165
        for expId in running_experiment_list:
            if expId.startswith(args.id):
                result_list.append(expId)
166
        if len(result_list) > 1:
chicm-ms's avatar
chicm-ms committed
167
            print_error(args.id + ' is ambiguous, please choose ' + ' '.join(result_list))
168
            return None
169
    if not result_list and (args.id  or args.port):
170
        print_error('There are no experiments matched, please set correct experiment id or restful server port')
SparkSnail's avatar
SparkSnail committed
171
    elif not result_list:
SparkSnail's avatar
SparkSnail committed
172
        print_error('There is no experiment running...')
173
174
    return result_list

175
176
177
def get_config_filename(args):
    '''get the file name of config file'''
    experiment_id = check_experiment_id(args)
chicm-ms's avatar
chicm-ms committed
178
    if experiment_id is None:
SparkSnail's avatar
SparkSnail committed
179
        print_error('Please set correct experiment id.')
chicm-ms's avatar
chicm-ms committed
180
        exit(1)
181
182
183
184
185
186
187
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    return experiment_dict[experiment_id]['fileName']

def get_experiment_port(args):
    '''get the port of experiment'''
    experiment_id = check_experiment_id(args)
chicm-ms's avatar
chicm-ms committed
188
    if experiment_id is None:
SparkSnail's avatar
SparkSnail committed
189
        print_error('Please set correct experiment id.')
chicm-ms's avatar
chicm-ms committed
190
        exit(1)
191
192
193
194
195
196
197
198
199
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    return experiment_dict[experiment_id]['port']

def convert_time_stamp_to_date(content):
    '''Convert time stamp to date time format'''
    start_time_stamp = content.get('startTime')
    end_time_stamp = content.get('endTime')
    if start_time_stamp:
200
        start_time = datetime.fromtimestamp(start_time_stamp // 1000, timezone.utc).astimezone().strftime("%Y/%m/%d %H:%M:%S")
201
202
        content['startTime'] = str(start_time)
    if end_time_stamp:
203
        end_time = datetime.fromtimestamp(end_time_stamp // 1000, timezone.utc).astimezone().strftime("%Y/%m/%d %H:%M:%S")
204
205
206
207
208
209
210
211
212
213
214
215
216
        content['endTime'] = str(end_time)
    return content

def check_rest(args):
    '''check if restful server is running'''
    nni_config = Config(get_config_filename(args))
    rest_port = nni_config.get_config('restServerPort')
    running, _ = check_rest_server_quick(rest_port)
    if not running:
        print_normal('Restful server is running...')
    else:
        print_normal('Restful server is not running...')

Deshui Yu's avatar
Deshui Yu committed
217
218
def stop_experiment(args):
    '''Stop the experiment which is running'''
219
220
221
    if args.id and args.id == 'all':
        print_warning('\'nnictl stop all\' is abolished, please use \'nnictl stop --all\' to stop all of experiments!')
        exit(1)
222
223
224
225
226
227
    experiment_id_list = parse_ids(args)
    if experiment_id_list:
        experiment_config = Experiments()
        experiment_dict = experiment_config.get_all_experiments()
        for experiment_id in experiment_id_list:
            print_normal('Stoping experiment %s' % experiment_id)
228
            nni_config = Config(experiment_dict[experiment_id]['fileName'])
229
230
            rest_pid = nni_config.get_config('restServerPid')
            if rest_pid:
231
                kill_command(rest_pid)
SparkSnail's avatar
SparkSnail committed
232
233
234
235
                tensorboard_pid_list = nni_config.get_config('tensorboardPidList')
                if tensorboard_pid_list:
                    for tensorboard_pid in tensorboard_pid_list:
                        try:
236
                            kill_command(tensorboard_pid)
SparkSnail's avatar
SparkSnail committed
237
238
239
                        except Exception as exception:
                            print_error(exception)
                    nni_config.set_config('tensorboardPidList', [])
SparkSnail's avatar
SparkSnail committed
240
            print_normal('Stop experiment success.')
241
            experiment_config.update_experiment(experiment_id, 'status', 'STOPPED')
chicm-ms's avatar
chicm-ms committed
242
            time_now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
243
            experiment_config.update_experiment(experiment_id, 'endTime', str(time_now))
Deshui Yu's avatar
Deshui Yu committed
244
245
246

def trial_ls(args):
    '''List trial'''
247
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
248
249
250
251
252
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
253
254
    running, response = check_rest_server_quick(rest_port)
    if running:
255
        response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT)
256
        if response and check_response(response):
257
            content = json.loads(response.text)
258
            for index, value in enumerate(content):
259
260
                content[index] = convert_time_stamp_to_date(value)
            print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
Deshui Yu's avatar
Deshui Yu committed
261
262
263
264
265
266
267
        else:
            print_error('List trial failed...')
    else:
        print_error('Restful server is not running...')

def trial_kill(args):
    '''List trial'''
268
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
269
270
271
272
273
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
274
275
    running, _ = check_rest_server_quick(rest_port)
    if running:
SparkSnail's avatar
SparkSnail committed
276
        response = rest_delete(trial_job_id_url(rest_port, args.trial_id), REST_TIME_OUT)
277
        if response and check_response(response):
Deshui Yu's avatar
Deshui Yu committed
278
279
280
281
282
283
            print(response.text)
        else:
            print_error('Kill trial job failed...')
    else:
        print_error('Restful server is not running...')

284
285
286
287
288
289
290
291
292
293
294
def trial_codegen(args):
    '''Generate code for a specific trial'''
    print_warning('Currently, this command is only for nni nas programming interface.')
    exp_id = check_experiment_id(args)
    nni_config = Config(get_config_filename(args))
    if not nni_config.get_config('experimentConfig')['useAnnotation']:
        print_error('The experiment is not using annotation')
        exit(1)
    code_dir = nni_config.get_config('experimentConfig')['trial']['codeDir']
    expand_annotations(code_dir, './exp_%s_trial_%s_code'%(exp_id, args.trial_id), exp_id, args.trial_id)

Deshui Yu's avatar
Deshui Yu committed
295
296
def list_experiment(args):
    '''Get experiment information'''
297
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
298
299
300
301
302
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
303
304
    running, _ = check_rest_server_quick(rest_port)
    if running:
305
        response = rest_get(experiment_url(rest_port), REST_TIME_OUT)
306
        if response and check_response(response):
307
308
            content = convert_time_stamp_to_date(json.loads(response.text))
            print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
Deshui Yu's avatar
Deshui Yu committed
309
310
311
312
313
        else:
            print_error('List experiment failed...')
    else:
        print_error('Restful server is not running...')

314
315
def experiment_status(args):
    '''Show the status of experiment'''
316
    nni_config = Config(get_config_filename(args))
317
318
319
320
321
322
323
    rest_port = nni_config.get_config('restServerPort')
    result, response = check_rest_server_quick(rest_port)
    if not result:
        print_normal('Restful server is not running...')
    else:
        print(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))

Deshui Yu's avatar
Deshui Yu committed
324
325
def log_internal(args, filetype):
    '''internal function to call get_log_content'''
326
    file_name = get_config_filename(args)
Deshui Yu's avatar
Deshui Yu committed
327
    if filetype == 'stdout':
328
        file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stdout')
Deshui Yu's avatar
Deshui Yu committed
329
    else:
330
        file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stderr')
331
    print(check_output_command(file_full_path, head=args.head, tail=args.tail))
332

Deshui Yu's avatar
Deshui Yu committed
333
334
335
336
337
338
339
340
def log_stdout(args):
    '''get stdout log'''
    log_internal(args, 'stdout')

def log_stderr(args):
    '''get stderr log'''
    log_internal(args, 'stderr')

341
342
343
def log_trial(args):
    ''''get trial log path'''
    trial_id_path_dict = {}
SparkSnail's avatar
SparkSnail committed
344
    trial_id_list = []
345
    nni_config = Config(get_config_filename(args))
346
347
348
349
350
351
352
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
    running, response = check_rest_server_quick(rest_port)
    if running:
353
        response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT)
354
355
356
        if response and check_response(response):
            content = json.loads(response.text)
            for trial in content:
SparkSnail's avatar
SparkSnail committed
357
358
359
                trial_id_list.append(trial.get('id'))
                if trial.get('logPath'):
                    trial_id_path_dict[trial.get('id')] = trial['logPath']
360
361
    else:
        print_error('Restful server is not running...')
goooxu's avatar
goooxu committed
362
        exit(1)
SparkSnail's avatar
SparkSnail committed
363
364
365
366
367
368
    if args.trial_id:
        if args.trial_id not in trial_id_list:
            print_error('Trial id {0} not correct, please check your command!'.format(args.trial_id))
            exit(1)
        if trial_id_path_dict.get(args.trial_id):
            print_normal('id:' + args.trial_id + ' path:' + trial_id_path_dict[args.trial_id])
369
        else:
SparkSnail's avatar
SparkSnail committed
370
            print_error('Log path is not available yet, please wait...')
goooxu's avatar
goooxu committed
371
            exit(1)
372
    else:
SparkSnail's avatar
SparkSnail committed
373
        print_normal('All of trial log info:')
374
        for key in trial_id_path_dict:
SparkSnail's avatar
SparkSnail committed
375
376
377
            print_normal('id:' + key + ' path:' + trial_id_path_dict[key])
        if not trial_id_path_dict:
            print_normal('None')
378

Deshui Yu's avatar
Deshui Yu committed
379
380
def get_config(args):
    '''get config info'''
381
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
382
    print(nni_config.get_all_config())
383
384
385

def webui_url(args):
    '''show the url of web ui'''
386
    nni_config = Config(get_config_filename(args))
387
388
    print_normal('{0} {1}'.format('Web UI url:', ' '.join(nni_config.get_config('webuiUrl'))))

SparkSnail's avatar
SparkSnail committed
389
390
391
392
393
def local_clean(directory):
    '''clean up local data'''
    print_normal('removing folder {0}'.format(directory))
    try:
        shutil.rmtree(directory)
chicm-ms's avatar
chicm-ms committed
394
    except FileNotFoundError:
SparkSnail's avatar
SparkSnail committed
395
        print_error('{0} does not exist.'.format(directory))
chicm-ms's avatar
chicm-ms committed
396

SparkSnail's avatar
SparkSnail committed
397
398
399
400
401
402
403
404
405
406
407
408
409
410
def remote_clean(machine_list, experiment_id=None):
    '''clean up remote data'''
    for machine in machine_list:
        passwd = machine.get('passwd')
        userName = machine.get('username')
        host = machine.get('ip')
        port = machine.get('port')
        if experiment_id:
            remote_dir = '/' + '/'.join(['tmp', 'nni', 'experiments', experiment_id])
        else:
            remote_dir = '/' + '/'.join(['tmp', 'nni', 'experiments'])
        sftp = create_ssh_sftp_client(host, port, userName, passwd)
        print_normal('removing folder {0}'.format(host + ':' + str(port) + remote_dir))
        remove_remote_directory(sftp, remote_dir)
chicm-ms's avatar
chicm-ms committed
411

SparkSnail's avatar
SparkSnail committed
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
def hdfs_clean(host, user_name, output_dir, experiment_id=None):
    '''clean up hdfs data'''
    hdfs_client = HdfsClient(hosts='{0}:80'.format(host), user_name=user_name, webhdfs_path='/webhdfs/api/v1', timeout=5)
    if experiment_id:
        full_path = '/' + '/'.join([user_name, 'nni', 'experiments', experiment_id])
    else:
        full_path = '/' + '/'.join([user_name, 'nni', 'experiments'])
    print_normal('removing folder {0} in hdfs'.format(full_path))
    hdfs_client.delete(full_path, recursive=True)
    if output_dir:
        pattern = re.compile('hdfs://(?P<host>([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(?P<baseDir>/.*)?')
        match_result = pattern.match(output_dir)
        if match_result:
            output_host = match_result.group('host')
            output_dir = match_result.group('baseDir')
            #check if the host is valid
            if output_host != host:
                print_warning('The host in {0} is not consistent with {1}'.format(output_dir, host))
            else:
                if experiment_id:
                    output_dir = output_dir + '/' + experiment_id
                print_normal('removing folder {0} in hdfs'.format(output_dir))
                hdfs_client.delete(output_dir, recursive=True)

def experiment_clean(args):
    '''clean up the experiment data'''
    experiment_id_list = []
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if args.all:
        experiment_id_list = list(experiment_dict.keys())
    else:
        if args.id is None:
SparkSnail's avatar
SparkSnail committed
445
            print_error('please set experiment id.')
SparkSnail's avatar
SparkSnail committed
446
447
            exit(1)
        if args.id not in experiment_dict:
SparkSnail's avatar
SparkSnail committed
448
            print_error('Cannot find experiment {0}.'.format(args.id))
SparkSnail's avatar
SparkSnail committed
449
450
451
            exit(1)
        experiment_id_list.append(args.id)
    while True:
SparkSnail's avatar
SparkSnail committed
452
        print('INFO: This action will delete experiment {0}, and it\'s not recoverable.'.format(' '.join(experiment_id_list)))
SparkSnail's avatar
SparkSnail committed
453
454
455
456
        inputs = input('INFO: do you want to continue?[y/N]:')
        if not inputs.lower() or inputs.lower() in ['n', 'no']:
            exit(0)
        elif inputs.lower() not in ['y', 'n', 'yes', 'no']:
SparkSnail's avatar
SparkSnail committed
457
            print_warning('please input Y or N.')
SparkSnail's avatar
SparkSnail committed
458
459
460
461
462
463
464
465
466
467
        else:
            break
    for experiment_id in experiment_id_list:
        nni_config = Config(experiment_dict[experiment_id]['fileName'])
        platform = nni_config.get_config('experimentConfig').get('trainingServicePlatform')
        experiment_id = nni_config.get_config('experimentId')
        if platform == 'remote':
            machine_list = nni_config.get_config('experimentConfig').get('machineList')
            remote_clean(machine_list, experiment_id)
        elif platform == 'pai':
chicm-ms's avatar
chicm-ms committed
468
            host = nni_config.get_config('experimentConfig').get('paiConfig').get('host')
SparkSnail's avatar
SparkSnail committed
469
470
471
472
473
            user_name = nni_config.get_config('experimentConfig').get('paiConfig').get('userName')
            output_dir = nni_config.get_config('experimentConfig').get('trial').get('outputDir')
            hdfs_clean(host, user_name, output_dir, experiment_id)
        elif platform != 'local':
            #TODO: support all platforms
SparkSnail's avatar
SparkSnail committed
474
            print_warning('platform {0} clean up not supported yet.'.format(platform))
SparkSnail's avatar
SparkSnail committed
475
476
477
478
479
480
481
482
483
484
            exit(0)
        #clean local data
        home = str(Path.home())
        local_dir = nni_config.get_config('experimentConfig').get('logDir')
        if not local_dir:
            local_dir = os.path.join(home, 'nni', 'experiments', experiment_id)
        local_clean(local_dir)
        experiment_config = Experiments()
        print_normal('removing metadata of experiment {0}'.format(experiment_id))
        experiment_config.remove_experiment(experiment_id)
chicm-ms's avatar
chicm-ms committed
485
        print_normal('Done.')
SparkSnail's avatar
SparkSnail committed
486
487
488
489
490
491
492
493
494
495

def get_platform_dir(config_content):
    '''get the dir list to be deleted'''
    platform = config_content.get('trainingServicePlatform')
    dir_list = []
    if platform == 'remote':
        machine_list = config_content.get('machineList')
        for machine in machine_list:
            host = machine.get('ip')
            port = machine.get('port')
SparkSnail's avatar
SparkSnail committed
496
            dir_list.append(host + ':' + str(port) + '/tmp/nni')
SparkSnail's avatar
SparkSnail committed
497
    elif platform == 'pai':
chicm-ms's avatar
chicm-ms committed
498
        host = config_content.get('paiConfig').get('host')
SparkSnail's avatar
SparkSnail committed
499
500
        user_name = config_content.get('paiConfig').get('userName')
        output_dir = config_content.get('trial').get('outputDir')
SparkSnail's avatar
SparkSnail committed
501
        dir_list.append('server: {0}, path: {1}/nni'.format(host, user_name))
SparkSnail's avatar
SparkSnail committed
502
503
504
505
506
507
508
509
        if output_dir:
            dir_list.append(output_dir)
    return dir_list

def platform_clean(args):
    '''clean up the experiment data'''
    config_path = os.path.abspath(args.config)
    if not os.path.exists(config_path):
SparkSnail's avatar
SparkSnail committed
510
        print_error('Please set correct config path.')
SparkSnail's avatar
SparkSnail committed
511
512
513
        exit(1)
    config_content = get_yml_content(config_path)
    platform = config_content.get('trainingServicePlatform')
SparkSnail's avatar
SparkSnail committed
514
515
516
    if platform == 'local':
        print_normal('it doesn’t need to clean local platform.')
        exit(0)
SparkSnail's avatar
SparkSnail committed
517
    if platform not in ['remote', 'pai']:
SparkSnail's avatar
SparkSnail committed
518
        print_normal('platform {0} not supported.'.format(platform))
SparkSnail's avatar
SparkSnail committed
519
        exit(0)
Shinai Yang's avatar
Shinai Yang committed
520
    update_experiment()
SparkSnail's avatar
SparkSnail committed
521
522
    dir_list = get_platform_dir(config_content)
    if not dir_list:
SparkSnail's avatar
SparkSnail committed
523
        print_normal('No folder of NNI caches is found.')
SparkSnail's avatar
SparkSnail committed
524
525
        exit(1)
    while True:
chicm-ms's avatar
chicm-ms committed
526
527
528
529
        print_normal('This command will remove below folders of NNI caches. If other users are using experiments' \
                     ' on below hosts, it will be broken.')
        for value in dir_list:
            print('       ' + value)
SparkSnail's avatar
SparkSnail committed
530
531
532
533
        inputs = input('INFO: do you want to continue?[y/N]:')
        if not inputs.lower() or inputs.lower() in ['n', 'no']:
            exit(0)
        elif inputs.lower() not in ['y', 'n', 'yes', 'no']:
SparkSnail's avatar
SparkSnail committed
534
            print_warning('please input Y or N.')
SparkSnail's avatar
SparkSnail committed
535
536
537
538
        else:
            break
    if platform == 'remote':
        machine_list = config_content.get('machineList')
chicm-ms's avatar
chicm-ms committed
539
        remote_clean(machine_list, None)
SparkSnail's avatar
SparkSnail committed
540
    elif platform == 'pai':
chicm-ms's avatar
chicm-ms committed
541
        host = config_content.get('paiConfig').get('host')
SparkSnail's avatar
SparkSnail committed
542
543
544
        user_name = config_content.get('paiConfig').get('userName')
        output_dir = config_content.get('trial').get('outputDir')
        hdfs_clean(host, user_name, output_dir, None)
SparkSnail's avatar
SparkSnail committed
545
    print_normal('Done.')
SparkSnail's avatar
SparkSnail committed
546

547
548
def experiment_list(args):
    '''get the information of all experiments'''
Shinai Yang's avatar
Shinai Yang committed
549
    update_experiment()
550
551
552
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
SparkSnail's avatar
SparkSnail committed
553
        print_normal('Cannot find experiments.')
554
555
        exit(1)
    experiment_id_list = []
SparkSnail's avatar
SparkSnail committed
556
    if args.all:
557
558
        for key in experiment_dict.keys():
            experiment_id_list.append(key)
559
560
    else:
        for key in experiment_dict.keys():
561
            if experiment_dict[key]['status'] != 'STOPPED':
562
563
                experiment_id_list.append(key)
        if not experiment_id_list:
564
            print_warning('There is no experiment running...\nYou can use \'nnictl experiment list --all\' to list all experiments.')
565
566
    experiment_information = ""
    for key in experiment_id_list:
567
568
569
570
571
572
573
        experiment_information += EXPERIMENT_DETAIL_FORMAT % (key,
                                                              experiment_dict[key].get('experimentName', 'N/A'),
                                                              experiment_dict[key]['status'],
                                                              experiment_dict[key]['port'],
                                                              experiment_dict[key].get('platform'),
                                                              experiment_dict[key]['startTime'],
                                                              experiment_dict[key]['endTime'])
574
    print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
SparkSnail's avatar
SparkSnail committed
575

SparkSnail's avatar
SparkSnail committed
576
577
578
579
def get_time_interval(time1, time2):
    '''get the interval of two times'''
    try:
        #convert time to timestamp
580
581
        time1 = time.mktime(time.strptime(time1, '%Y/%m/%d %H:%M:%S'))
        time2 = time.mktime(time.strptime(time2, '%Y/%m/%d %H:%M:%S'))
582
        seconds = (datetime.fromtimestamp(time2) - datetime.fromtimestamp(time1)).seconds
SparkSnail's avatar
SparkSnail committed
583
584
585
586
587
588
589
590
591
592
593
594
595
        #convert seconds to day:hour:minute:second
        days = seconds / 86400
        seconds %= 86400
        hours = seconds / 3600
        seconds %= 3600
        minutes = seconds / 60
        seconds %= 60
        return '%dd %dh %dm %ds' % (days, hours, minutes, seconds)
    except:
        return 'N/A'

def show_experiment_info():
    '''show experiment information in monitor'''
Shinai Yang's avatar
Shinai Yang committed
596
    update_experiment()
SparkSnail's avatar
SparkSnail committed
597
598
599
600
601
602
603
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print('There is no experiment running...')
        exit(1)
    experiment_id_list = []
    for key in experiment_dict.keys():
604
        if experiment_dict[key]['status'] != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
605
606
607
608
609
610
            experiment_id_list.append(key)
    if not experiment_id_list:
        print_warning('There is no experiment running...')
        return
    for key in experiment_id_list:
        print(EXPERIMENT_MONITOR_INFO % (key, experiment_dict[key]['status'], experiment_dict[key]['port'], \
chicm-ms's avatar
chicm-ms committed
611
612
             experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], \
             get_time_interval(experiment_dict[key]['startTime'], experiment_dict[key]['endTime'])))
SparkSnail's avatar
SparkSnail committed
613
614
615
        print(TRIAL_MONITOR_HEAD)
        running, response = check_rest_server_quick(experiment_dict[key]['port'])
        if running:
616
            response = rest_get(trial_jobs_url(experiment_dict[key]['port']), REST_TIME_OUT)
SparkSnail's avatar
SparkSnail committed
617
618
            if response and check_response(response):
                content = json.loads(response.text)
619
                for index, value in enumerate(content):
SparkSnail's avatar
SparkSnail committed
620
                    content[index] = convert_time_stamp_to_date(value)
chicm-ms's avatar
chicm-ms committed
621
622
                    print(TRIAL_MONITOR_CONTENT % (content[index].get('id'), content[index].get('startTime'), \
                          content[index].get('endTime'), content[index].get('status')))
SparkSnail's avatar
SparkSnail committed
623
624
625
626
627
628
629
630
631
632
        print(TRIAL_MONITOR_TAIL)

def monitor_experiment(args):
    '''monitor the experiment'''
    if args.time <= 0:
        print_error('please input a positive integer as time interval, the unit is second.')
        exit(1)
    while True:
        try:
            os.system('clear')
633
            update_experiment()
SparkSnail's avatar
SparkSnail committed
634
635
636
637
638
639
640
            show_experiment_info()
            time.sleep(args.time)
        except KeyboardInterrupt:
            exit(0)
        except Exception as exception:
            print_error(exception)
            exit(1)
Yan Ni's avatar
Yan Ni committed
641
642

def export_trials_data(args):
643
644
    '''export experiment metadata to csv
    '''
Yan Ni's avatar
Yan Ni committed
645
646
647
648
649
650
651
652
    nni_config = Config(get_config_filename(args))
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
    running, response = check_rest_server_quick(rest_port)
    if running:
653
        response = rest_get(export_data_url(rest_port), 20)
Yan Ni's avatar
Yan Ni committed
654
        if response is not None and check_response(response):
655
            if args.type == 'json':
656
657
658
659
660
661
662
663
664
665
666
667
668
                with open(args.path, 'w') as file:
                    file.write(response.text)
            elif args.type == 'csv':
                content = json.loads(response.text)
                trial_records = []
                for record in content:
                    if not isinstance(record['value'], (float, int)):
                        formated_record = {**record['parameter'], **record['value'], **{'id': record['id']}}
                    else:
                        formated_record = {**record['parameter'], **{'reward': record['value'], 'id': record['id']}}
                    trial_records.append(formated_record)
                with open(args.path, 'w') as file:
                    writer = csv.DictWriter(file, set.union(*[set(r.keys()) for r in trial_records]))
669
                    writer.writeheader()
670
671
672
673
                    writer.writerows(trial_records)
            else:
                print_error('Unknown type: %s' % args.type)
                exit(1)
Yan Ni's avatar
Yan Ni committed
674
675
676
        else:
            print_error('Export failed...')
    else:
677
        print_error('Restful server is not Running')