nnictl_utils.py 20.3 KB
Newer Older
Deshui Yu's avatar
Deshui Yu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Copyright (c) Microsoft Corporation
# All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge,
# to any person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import os
import psutil
import json
24
import datetime
25
import time
Deshui Yu's avatar
Deshui Yu committed
26
from subprocess import call, check_output
27
from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response
28
from .config_utils import Config, Experiments
Deshui Yu's avatar
Deshui Yu committed
29
from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url
SparkSnail's avatar
SparkSnail committed
30
from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \
31
     EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT
32
from .common_utils import print_normal, print_error, print_warning, detect_process
Deshui Yu's avatar
Deshui Yu committed
33

34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
def get_experiment_time(port):
    '''get the startTime and endTime of an experiment'''
    response = rest_get(experiment_url(port), REST_TIME_OUT)
    if response and check_response(response):
        content = convert_time_stamp_to_date(json.loads(response.text))
        return content.get('startTime'), content.get('endTime')
    return None, None

def get_experiment_status(port):
    '''get the status of an experiment'''
    result, response = check_rest_server_quick(port)
    if result:
        return json.loads(response.text).get('status')
    return None

def update_experiment():
SparkSnail's avatar
SparkSnail committed
50
51
52
53
54
55
56
    '''Update the experiment status in config file'''
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        return None
    for key in experiment_dict.keys():
        if isinstance(experiment_dict[key], dict):
57
            if experiment_dict[key].get('status') != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
58
59
60
                nni_config = Config(experiment_dict[key]['fileName'])
                rest_pid = nni_config.get_config('restServerPid')
                if not detect_process(rest_pid):
61
62
63
64
65
66
67
68
69
70
71
                    experiment_config.update_experiment(key, 'status', 'STOPPED')
                    continue
                rest_port = nni_config.get_config('restServerPort')
                startTime, endTime = get_experiment_time(rest_port)
                if startTime:
                    experiment_config.update_experiment(key, 'startTime', startTime)
                if endTime:
                    experiment_config.update_experiment(key, 'endTime', endTime)
                status = get_experiment_status(rest_port)
                if status:
                    experiment_config.update_experiment(key, 'status', status)
SparkSnail's avatar
SparkSnail committed
72

73
74
75
def check_experiment_id(args):
    '''check if the id is valid
    '''
76
    update_experiment()
77
78
79
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
80
        print_normal('There is no experiment running...')
chicm-ms's avatar
chicm-ms committed
81
        return None
82
    if not args.id:
83
84
        running_experiment_list = []
        for key in experiment_dict.keys():
SparkSnail's avatar
SparkSnail committed
85
            if isinstance(experiment_dict[key], dict):
86
                if experiment_dict[key].get('status') != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
87
88
89
90
                    running_experiment_list.append(key)
            elif isinstance(experiment_dict[key], list):
                # if the config file is old version, remove the configuration from file
                experiment_config.remove_experiment(key)
91
        if len(running_experiment_list) > 1:
92
            print_error('There are multiple experiments, please set the experiment id...')
93
94
95
            experiment_information = ""
            for key in running_experiment_list:
                experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
SparkSnail's avatar
SparkSnail committed
96
                experiment_dict[key]['port'], experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
97
98
99
100
            print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
            exit(1)
        elif not running_experiment_list:
            print_error('There is no experiment running!')
chicm-ms's avatar
chicm-ms committed
101
            return None
102
103
        else:
            return running_experiment_list[0]
104
105
106
107
108
109
110
111
    if hasattr(args, "experiment"):
        if experiment_dict.get(args.experiment):
            return args.experiment
    elif hasattr(args, "id"):
        if experiment_dict.get(args.id):
            return args.id
    print_error('Id not correct!')
    return None
Deshui Yu's avatar
Deshui Yu committed
112

113
def parse_ids(args):
114
115
116
117
118
119
120
121
    '''Parse the arguments for nnictl stop
    1.If there is an id specified, return the corresponding id
    2.If there is no id specified, and there is an experiment running, return the id, or return Error
    3.If the id matches an experiment, nnictl will return the id.
    4.If the id ends with *, nnictl will match all ids matchs the regular
    5.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id
    6.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information
    '''
122
    update_experiment()
123
124
125
126
127
128
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print_normal('Experiment is not running...')
        return None
    result_list = []
129
130
    running_experiment_list = []
    for key in experiment_dict.keys():
SparkSnail's avatar
SparkSnail committed
131
        if isinstance(experiment_dict[key], dict):
132
            if experiment_dict[key].get('status') != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
133
134
135
136
                running_experiment_list.append(key)
        elif isinstance(experiment_dict[key], list):
            # if the config file is old version, remove the configuration from file
            experiment_config.remove_experiment(key)
137
    if not args.id:
138
        if len(running_experiment_list) > 1:
139
            print_error('There are multiple experiments, please set the experiment id...')
140
            experiment_information = ""
141
142
            for key in running_experiment_list:
                experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
SparkSnail's avatar
SparkSnail committed
143
                experiment_dict[key]['port'], experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
144
145
146
147
            print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
            exit(1)
        else:
            result_list = running_experiment_list
148
    elif args.id == 'all':
149
        result_list = running_experiment_list
150
    elif args.id.endswith('*'):
151
        for id in running_experiment_list:
152
153
            if id.startswith(args.id[:-1]):
                result_list.append(id)
154
    elif args.id in running_experiment_list:
155
156
        result_list.append(args.id)
    else:
157
        for id in running_experiment_list:
158
159
160
161
162
            if id.startswith(args.id):
                result_list.append(id)
        if len(result_list) > 1:
            print_error(args.id + ' is ambiguous, please choose ' + ' '.join(result_list) )
            return None
chicm-ms's avatar
chicm-ms committed
163
164
165
166
    if not result_list and args.id:
        print_error('There are no experiments matched, please set correct experiment id...')
    elif not result_list:
        print_error('There is no experiment running...')
167
168
    return result_list

169
170
171
def get_config_filename(args):
    '''get the file name of config file'''
    experiment_id = check_experiment_id(args)
chicm-ms's avatar
chicm-ms committed
172
173
174
    if experiment_id is None:
        print_error('Please set the experiment id!')
        exit(1)
175
176
177
178
179
180
181
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    return experiment_dict[experiment_id]['fileName']

def get_experiment_port(args):
    '''get the port of experiment'''
    experiment_id = check_experiment_id(args)
chicm-ms's avatar
chicm-ms committed
182
183
184
    if experiment_id is None:
        print_error('Please set the experiment id!')
        exit(1)
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    return experiment_dict[experiment_id]['port']

def convert_time_stamp_to_date(content):
    '''Convert time stamp to date time format'''
    start_time_stamp = content.get('startTime')
    end_time_stamp = content.get('endTime')
    if start_time_stamp:
        start_time = datetime.datetime.utcfromtimestamp(start_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S")
        content['startTime'] = str(start_time)
    if end_time_stamp:
        end_time = datetime.datetime.utcfromtimestamp(end_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S")
        content['endTime'] = str(end_time)
    return content

def check_rest(args):
    '''check if restful server is running'''
    nni_config = Config(get_config_filename(args))
    rest_port = nni_config.get_config('restServerPort')
    running, _ = check_rest_server_quick(rest_port)
    if not running:
        print_normal('Restful server is running...')
    else:
        print_normal('Restful server is not running...')

Deshui Yu's avatar
Deshui Yu committed
211
212
def stop_experiment(args):
    '''Stop the experiment which is running'''
213
214
215
216
217
218
    experiment_id_list = parse_ids(args)
    if experiment_id_list:
        experiment_config = Experiments()
        experiment_dict = experiment_config.get_all_experiments()
        for experiment_id in experiment_id_list:
            print_normal('Stoping experiment %s' % experiment_id)
219
            nni_config = Config(experiment_dict[experiment_id]['fileName'])
220
221
222
            rest_port = nni_config.get_config('restServerPort')
            rest_pid = nni_config.get_config('restServerPid')
            if rest_pid:
223
                stop_rest_cmds = ['kill', str(rest_pid)]
SparkSnail's avatar
SparkSnail committed
224
225
226
227
228
229
230
231
232
233
                call(stop_rest_cmds)
                tensorboard_pid_list = nni_config.get_config('tensorboardPidList')
                if tensorboard_pid_list:
                    for tensorboard_pid in tensorboard_pid_list:
                        try:
                            cmds = ['kill', '-9', str(tensorboard_pid)]
                            call(cmds)
                        except Exception as exception:
                            print_error(exception)
                    nni_config.set_config('tensorboardPidList', [])
SparkSnail's avatar
SparkSnail committed
234
            print_normal('Stop experiment success!')
235
            experiment_config.update_experiment(experiment_id, 'status', 'STOPPED')
236
237
            time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
            experiment_config.update_experiment(experiment_id, 'endTime', str(time_now))
Deshui Yu's avatar
Deshui Yu committed
238
239
240

def trial_ls(args):
    '''List trial'''
241
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
242
243
244
245
246
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
247
248
    running, response = check_rest_server_quick(rest_port)
    if running:
249
        response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT)
250
        if response and check_response(response):
251
            content = json.loads(response.text)
252
            for index, value in enumerate(content):
253
254
                content[index] = convert_time_stamp_to_date(value)
            print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
Deshui Yu's avatar
Deshui Yu committed
255
256
257
258
259
260
261
        else:
            print_error('List trial failed...')
    else:
        print_error('Restful server is not running...')

def trial_kill(args):
    '''List trial'''
262
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
263
264
265
266
267
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
268
269
    running, _ = check_rest_server_quick(rest_port)
    if running:
270
        response = rest_delete(trial_job_id_url(rest_port, args.id), REST_TIME_OUT)
271
        if response and check_response(response):
Deshui Yu's avatar
Deshui Yu committed
272
273
274
275
276
277
278
279
            print(response.text)
        else:
            print_error('Kill trial job failed...')
    else:
        print_error('Restful server is not running...')

def list_experiment(args):
    '''Get experiment information'''
280
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
281
282
283
284
285
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
286
287
    running, _ = check_rest_server_quick(rest_port)
    if running:
288
        response = rest_get(experiment_url(rest_port), REST_TIME_OUT)
289
        if response and check_response(response):
290
291
            content = convert_time_stamp_to_date(json.loads(response.text))
            print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
Deshui Yu's avatar
Deshui Yu committed
292
293
294
295
296
        else:
            print_error('List experiment failed...')
    else:
        print_error('Restful server is not running...')

297
298
def experiment_status(args):
    '''Show the status of experiment'''
299
    nni_config = Config(get_config_filename(args))
300
301
302
303
304
305
306
    rest_port = nni_config.get_config('restServerPort')
    result, response = check_rest_server_quick(rest_port)
    if not result:
        print_normal('Restful server is not running...')
    else:
        print(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))

Deshui Yu's avatar
Deshui Yu committed
307
308
309
310
311
312
313
314
315
316
def get_log_content(file_name, cmds):
    '''use cmds to read config content'''
    if os.path.exists(file_name):
        rest = check_output(cmds)
        print(rest.decode('utf-8'))
    else:
        print_normal('NULL!')

def log_internal(args, filetype):
    '''internal function to call get_log_content'''
317
    file_name = get_config_filename(args)
Deshui Yu's avatar
Deshui Yu committed
318
    if filetype == 'stdout':
319
        file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stdout')
Deshui Yu's avatar
Deshui Yu committed
320
    else:
321
        file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stderr')
Deshui Yu's avatar
Deshui Yu committed
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
    if args.head:
        get_log_content(file_full_path, ['head', '-' + str(args.head), file_full_path])
    elif args.tail:
        get_log_content(file_full_path, ['tail', '-' + str(args.tail), file_full_path])
    elif args.path:
        print_normal('The path of stdout file is: ' + file_full_path)
    else:
        get_log_content(file_full_path, ['cat', file_full_path])

def log_stdout(args):
    '''get stdout log'''
    log_internal(args, 'stdout')

def log_stderr(args):
    '''get stderr log'''
    log_internal(args, 'stderr')

339
340
341
def log_trial(args):
    ''''get trial log path'''
    trial_id_path_dict = {}
342
    nni_config = Config(get_config_filename(args))
343
344
345
346
347
348
349
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
    running, response = check_rest_server_quick(rest_port)
    if running:
350
        response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT)
351
352
353
354
355
356
        if response and check_response(response):
            content = json.loads(response.text)
            for trial in content:
                trial_id_path_dict[trial['id']] = trial['logPath']
    else:
        print_error('Restful server is not running...')
goooxu's avatar
goooxu committed
357
        exit(1)
358
359
360
361
362
363
364
    if args.experiment:
        if args.id:
            if trial_id_path_dict.get(args.id):
                print('id:' + args.id + ' path:' + trial_id_path_dict[args.id])
            else:
                print_error('trial id is not valid!')
                exit(1)
365
        else:
366
367
            print_error('please specific the trial id!')
            print_error("trial id list in this experiment: " + str(list(trial_id_path_dict.keys())))
goooxu's avatar
goooxu committed
368
            exit(1)
369
    else:
370
        for key in trial_id_path_dict:
371
372
            print('id:' + key + ' path:' + trial_id_path_dict[key])

Deshui Yu's avatar
Deshui Yu committed
373
374
def get_config(args):
    '''get config info'''
375
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
376
    print(nni_config.get_all_config())
377
378
379

def webui_url(args):
    '''show the url of web ui'''
380
    nni_config = Config(get_config_filename(args))
381
382
    print_normal('{0} {1}'.format('Web UI url:', ' '.join(nni_config.get_config('webuiUrl'))))

383
384
def experiment_list(args):
    '''get the information of all experiments'''
385
386
387
388
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print('There is no experiment running...')
389
        exit(1)
390
    update_experiment()
391
392
393
394
    experiment_id_list = []
    if args.all and args.all == 'all':
        for key in experiment_dict.keys():
            experiment_id_list.append(key)
395
396
    else:
        for key in experiment_dict.keys():
397
            if experiment_dict[key]['status'] != 'STOPPED':
398
399
400
401
402
                experiment_id_list.append(key)
        if not experiment_id_list:
            print_warning('There is no experiment running...\nYou can use \'nnictl experiment list all\' to list all stopped experiments!')
    experiment_information = ""
    for key in experiment_id_list:
403
        
404
        experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], experiment_dict[key]['port'],\
SparkSnail's avatar
SparkSnail committed
405
        experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
406
    print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
SparkSnail's avatar
SparkSnail committed
407

SparkSnail's avatar
SparkSnail committed
408
409
410
411
def get_time_interval(time1, time2):
    '''get the interval of two times'''
    try:
        #convert time to timestamp
412
413
        time1 = time.mktime(time.strptime(time1, '%Y/%m/%d %H:%M:%S'))
        time2 = time.mktime(time.strptime(time2, '%Y/%m/%d %H:%M:%S'))
SparkSnail's avatar
SparkSnail committed
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
        seconds = (datetime.datetime.fromtimestamp(time2) - datetime.datetime.fromtimestamp(time1)).seconds
        #convert seconds to day:hour:minute:second
        days = seconds / 86400
        seconds %= 86400
        hours = seconds / 3600
        seconds %= 3600
        minutes = seconds / 60
        seconds %= 60
        return '%dd %dh %dm %ds' % (days, hours, minutes, seconds)
    except:
        return 'N/A'

def show_experiment_info():
    '''show experiment information in monitor'''
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print('There is no experiment running...')
        exit(1)
433
    update_experiment()
SparkSnail's avatar
SparkSnail committed
434
435
    experiment_id_list = []
    for key in experiment_dict.keys():
436
        if experiment_dict[key]['status'] != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
437
438
439
440
441
442
            experiment_id_list.append(key)
    if not experiment_id_list:
        print_warning('There is no experiment running...')
        return
    for key in experiment_id_list:
        print(EXPERIMENT_MONITOR_INFO % (key, experiment_dict[key]['status'], experiment_dict[key]['port'], \
443
             experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], get_time_interval(experiment_dict[key]['startTime'], experiment_dict[key]['endTime'])))
SparkSnail's avatar
SparkSnail committed
444
445
446
        print(TRIAL_MONITOR_HEAD)
        running, response = check_rest_server_quick(experiment_dict[key]['port'])
        if running:
447
            response = rest_get(trial_jobs_url(experiment_dict[key]['port']), REST_TIME_OUT)
SparkSnail's avatar
SparkSnail committed
448
449
            if response and check_response(response):
                content = json.loads(response.text)
450
                for index, value in enumerate(content):
SparkSnail's avatar
SparkSnail committed
451
452
453
454
455
456
457
458
459
460
461
462
                    content[index] = convert_time_stamp_to_date(value)
                    print(TRIAL_MONITOR_CONTENT % (content[index].get('id'), content[index].get('startTime'), content[index].get('endTime'), content[index].get('status')))
        print(TRIAL_MONITOR_TAIL)

def monitor_experiment(args):
    '''monitor the experiment'''
    if args.time <= 0:
        print_error('please input a positive integer as time interval, the unit is second.')
        exit(1)
    while True:
        try:
            os.system('clear')
463
            update_experiment()
SparkSnail's avatar
SparkSnail committed
464
465
466
467
468
469
470
            show_experiment_info()
            time.sleep(args.time)
        except KeyboardInterrupt:
            exit(0)
        except Exception as exception:
            print_error(exception)
            exit(1)