nnictl_utils.py 19.1 KB
Newer Older
Deshui Yu's avatar
Deshui Yu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Copyright (c) Microsoft Corporation
# All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge,
# to any person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import os
import psutil
import json
24
import datetime
25
import time
Deshui Yu's avatar
Deshui Yu committed
26
from subprocess import call, check_output
27
from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response
28
from .config_utils import Config, Experiments
Deshui Yu's avatar
Deshui Yu committed
29
from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url
SparkSnail's avatar
SparkSnail committed
30
31
from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \
     EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL
32
from .common_utils import print_normal, print_error, print_warning, detect_process
Deshui Yu's avatar
Deshui Yu committed
33

SparkSnail's avatar
SparkSnail committed
34
35
36
37
38
39
40
41
42
43
44
45
46
47
def update_experiment_status():
    '''Update the experiment status in config file'''
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        return None
    for key in experiment_dict.keys():
        if isinstance(experiment_dict[key], dict):
            if experiment_dict[key].get('status') == 'running':
                nni_config = Config(experiment_dict[key]['fileName'])
                rest_pid = nni_config.get_config('restServerPid')
                if not detect_process(rest_pid):
                    experiment_config.update_experiment(key, 'status', 'stopped')

48
49
50
def check_experiment_id(args):
    '''check if the id is valid
    '''
SparkSnail's avatar
SparkSnail committed
51
    update_experiment_status()
52
53
54
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
55
        print_normal('There is no experiment running...')
chicm-ms's avatar
chicm-ms committed
56
        return None
57
    if not args.id:
58
59
        running_experiment_list = []
        for key in experiment_dict.keys():
SparkSnail's avatar
SparkSnail committed
60
61
62
63
64
65
            if isinstance(experiment_dict[key], dict):
                if experiment_dict[key].get('status') == 'running':
                    running_experiment_list.append(key)
            elif isinstance(experiment_dict[key], list):
                # if the config file is old version, remove the configuration from file
                experiment_config.remove_experiment(key)
66
67
68
69
70
        if len(running_experiment_list) > 1:
            print_error('There are multiple experiments running, please set the experiment id...')
            experiment_information = ""
            for key in running_experiment_list:
                experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
SparkSnail's avatar
SparkSnail committed
71
                experiment_dict[key]['port'], experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
72
73
74
75
            print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
            exit(1)
        elif not running_experiment_list:
            print_error('There is no experiment running!')
chicm-ms's avatar
chicm-ms committed
76
            return None
77
78
        else:
            return running_experiment_list[0]
79
80
81
82
83
84
85
86
    if hasattr(args, "experiment"):
        if experiment_dict.get(args.experiment):
            return args.experiment
    elif hasattr(args, "id"):
        if experiment_dict.get(args.id):
            return args.id
    print_error('Id not correct!')
    return None
Deshui Yu's avatar
Deshui Yu committed
87

88
def parse_ids(args):
89
90
91
92
93
94
95
96
    '''Parse the arguments for nnictl stop
    1.If there is an id specified, return the corresponding id
    2.If there is no id specified, and there is an experiment running, return the id, or return Error
    3.If the id matches an experiment, nnictl will return the id.
    4.If the id ends with *, nnictl will match all ids matchs the regular
    5.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id
    6.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information
    '''
SparkSnail's avatar
SparkSnail committed
97
    update_experiment_status()
98
99
100
101
102
103
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print_normal('Experiment is not running...')
        return None
    result_list = []
104
105
    running_experiment_list = []
    for key in experiment_dict.keys():
SparkSnail's avatar
SparkSnail committed
106
107
108
109
110
111
        if isinstance(experiment_dict[key], dict):
            if experiment_dict[key].get('status') == 'running':
                running_experiment_list.append(key)
        elif isinstance(experiment_dict[key], list):
            # if the config file is old version, remove the configuration from file
            experiment_config.remove_experiment(key)
112
    if not args.id:
113
        if len(running_experiment_list) > 1:
114
115
            print_error('There are multiple experiments running, please set the experiment id...')
            experiment_information = ""
116
117
            for key in running_experiment_list:
                experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
SparkSnail's avatar
SparkSnail committed
118
                experiment_dict[key]['port'], experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
119
120
121
122
            print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
            exit(1)
        else:
            result_list = running_experiment_list
123
    elif args.id == 'all':
124
        result_list = running_experiment_list
125
    elif args.id.endswith('*'):
126
        for id in running_experiment_list:
127
128
            if id.startswith(args.id[:-1]):
                result_list.append(id)
129
    elif args.id in running_experiment_list:
130
131
        result_list.append(args.id)
    else:
132
        for id in running_experiment_list:
133
134
135
136
137
            if id.startswith(args.id):
                result_list.append(id)
        if len(result_list) > 1:
            print_error(args.id + ' is ambiguous, please choose ' + ' '.join(result_list) )
            return None
chicm-ms's avatar
chicm-ms committed
138
139
140
141
    if not result_list and args.id:
        print_error('There are no experiments matched, please set correct experiment id...')
    elif not result_list:
        print_error('There is no experiment running...')
142
143
    return result_list

144
145
146
def get_config_filename(args):
    '''get the file name of config file'''
    experiment_id = check_experiment_id(args)
chicm-ms's avatar
chicm-ms committed
147
148
149
    if experiment_id is None:
        print_error('Please set the experiment id!')
        exit(1)
150
151
152
153
154
155
156
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    return experiment_dict[experiment_id]['fileName']

def get_experiment_port(args):
    '''get the port of experiment'''
    experiment_id = check_experiment_id(args)
chicm-ms's avatar
chicm-ms committed
157
158
159
    if experiment_id is None:
        print_error('Please set the experiment id!')
        exit(1)
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    return experiment_dict[experiment_id]['port']

def convert_time_stamp_to_date(content):
    '''Convert time stamp to date time format'''
    start_time_stamp = content.get('startTime')
    end_time_stamp = content.get('endTime')
    if start_time_stamp:
        start_time = datetime.datetime.utcfromtimestamp(start_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S")
        content['startTime'] = str(start_time)
    if end_time_stamp:
        end_time = datetime.datetime.utcfromtimestamp(end_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S")
        content['endTime'] = str(end_time)
    return content

def check_rest(args):
    '''check if restful server is running'''
    nni_config = Config(get_config_filename(args))
    rest_port = nni_config.get_config('restServerPort')
    running, _ = check_rest_server_quick(rest_port)
    if not running:
        print_normal('Restful server is running...')
    else:
        print_normal('Restful server is not running...')

Deshui Yu's avatar
Deshui Yu committed
186
187
def stop_experiment(args):
    '''Stop the experiment which is running'''
188
189
190
191
192
193
    experiment_id_list = parse_ids(args)
    if experiment_id_list:
        experiment_config = Experiments()
        experiment_dict = experiment_config.get_all_experiments()
        for experiment_id in experiment_id_list:
            print_normal('Stoping experiment %s' % experiment_id)
194
            nni_config = Config(experiment_dict[experiment_id]['fileName'])
195
196
197
            rest_port = nni_config.get_config('restServerPort')
            rest_pid = nni_config.get_config('restServerPid')
            if rest_pid:
198
                stop_rest_cmds = ['kill', str(rest_pid)]
SparkSnail's avatar
SparkSnail committed
199
200
201
202
203
204
205
206
207
208
                call(stop_rest_cmds)
                tensorboard_pid_list = nni_config.get_config('tensorboardPidList')
                if tensorboard_pid_list:
                    for tensorboard_pid in tensorboard_pid_list:
                        try:
                            cmds = ['kill', '-9', str(tensorboard_pid)]
                            call(cmds)
                        except Exception as exception:
                            print_error(exception)
                    nni_config.set_config('tensorboardPidList', [])
SparkSnail's avatar
SparkSnail committed
209
            print_normal('Stop experiment success!')
210
211
212
            experiment_config.update_experiment(experiment_id, 'status', 'stopped')
            time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
            experiment_config.update_experiment(experiment_id, 'endTime', str(time_now))
Deshui Yu's avatar
Deshui Yu committed
213
214
215

def trial_ls(args):
    '''List trial'''
216
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
217
218
219
220
221
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
222
223
    running, response = check_rest_server_quick(rest_port)
    if running:
Deshui Yu's avatar
Deshui Yu committed
224
        response = rest_get(trial_jobs_url(rest_port), 20)
225
        if response and check_response(response):
226
            content = json.loads(response.text)
227
            for index, value in enumerate(content):
228
229
                content[index] = convert_time_stamp_to_date(value)
            print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
Deshui Yu's avatar
Deshui Yu committed
230
231
232
233
234
235
236
        else:
            print_error('List trial failed...')
    else:
        print_error('Restful server is not running...')

def trial_kill(args):
    '''List trial'''
237
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
238
239
240
241
242
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
243
244
    running, _ = check_rest_server_quick(rest_port)
    if running:
245
        response = rest_delete(trial_job_id_url(rest_port, args.id), 20)
246
        if response and check_response(response):
Deshui Yu's avatar
Deshui Yu committed
247
248
249
250
251
252
253
254
            print(response.text)
        else:
            print_error('Kill trial job failed...')
    else:
        print_error('Restful server is not running...')

def list_experiment(args):
    '''Get experiment information'''
255
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
256
257
258
259
260
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
261
262
    running, _ = check_rest_server_quick(rest_port)
    if running:
Deshui Yu's avatar
Deshui Yu committed
263
        response = rest_get(experiment_url(rest_port), 20)
264
        if response and check_response(response):
265
266
            content = convert_time_stamp_to_date(json.loads(response.text))
            print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
Deshui Yu's avatar
Deshui Yu committed
267
268
269
270
271
        else:
            print_error('List experiment failed...')
    else:
        print_error('Restful server is not running...')

272
273
def experiment_status(args):
    '''Show the status of experiment'''
274
    nni_config = Config(get_config_filename(args))
275
276
277
278
279
280
281
    rest_port = nni_config.get_config('restServerPort')
    result, response = check_rest_server_quick(rest_port)
    if not result:
        print_normal('Restful server is not running...')
    else:
        print(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))

Deshui Yu's avatar
Deshui Yu committed
282
283
284
285
286
287
288
289
290
291
def get_log_content(file_name, cmds):
    '''use cmds to read config content'''
    if os.path.exists(file_name):
        rest = check_output(cmds)
        print(rest.decode('utf-8'))
    else:
        print_normal('NULL!')

def log_internal(args, filetype):
    '''internal function to call get_log_content'''
292
    file_name = get_config_filename(args)
Deshui Yu's avatar
Deshui Yu committed
293
    if filetype == 'stdout':
294
        file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stdout')
Deshui Yu's avatar
Deshui Yu committed
295
    else:
296
        file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stderr')
Deshui Yu's avatar
Deshui Yu committed
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
    if args.head:
        get_log_content(file_full_path, ['head', '-' + str(args.head), file_full_path])
    elif args.tail:
        get_log_content(file_full_path, ['tail', '-' + str(args.tail), file_full_path])
    elif args.path:
        print_normal('The path of stdout file is: ' + file_full_path)
    else:
        get_log_content(file_full_path, ['cat', file_full_path])

def log_stdout(args):
    '''get stdout log'''
    log_internal(args, 'stdout')

def log_stderr(args):
    '''get stderr log'''
    log_internal(args, 'stderr')

314
315
316
def log_trial(args):
    ''''get trial log path'''
    trial_id_path_dict = {}
317
    nni_config = Config(get_config_filename(args))
318
319
320
321
322
323
324
325
326
327
328
329
330
331
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
    running, response = check_rest_server_quick(rest_port)
    if running:
        response = rest_get(trial_jobs_url(rest_port), 20)
        if response and check_response(response):
            content = json.loads(response.text)
            for trial in content:
                trial_id_path_dict[trial['id']] = trial['logPath']
    else:
        print_error('Restful server is not running...')
goooxu's avatar
goooxu committed
332
        exit(1)
333
334
335
336
337
338
339
    if args.experiment:
        if args.id:
            if trial_id_path_dict.get(args.id):
                print('id:' + args.id + ' path:' + trial_id_path_dict[args.id])
            else:
                print_error('trial id is not valid!')
                exit(1)
340
        else:
341
342
            print_error('please specific the trial id!')
            print_error("trial id list in this experiment: " + str(list(trial_id_path_dict.keys())))
goooxu's avatar
goooxu committed
343
            exit(1)
344
    else:
345
        for key in trial_id_path_dict:
346
347
            print('id:' + key + ' path:' + trial_id_path_dict[key])

Deshui Yu's avatar
Deshui Yu committed
348
349
def get_config(args):
    '''get config info'''
350
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
351
    print(nni_config.get_all_config())
352
353
354

def webui_url(args):
    '''show the url of web ui'''
355
    nni_config = Config(get_config_filename(args))
356
357
    print_normal('{0} {1}'.format('Web UI url:', ' '.join(nni_config.get_config('webuiUrl'))))

358
359
def experiment_list(args):
    '''get the information of all experiments'''
360
361
362
363
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print('There is no experiment running...')
364
365
366
367
368
        exit(1)
    experiment_id_list = []
    if args.all and args.all == 'all':
        for key in experiment_dict.keys():
            experiment_id_list.append(key)
369
370
    else:
        for key in experiment_dict.keys():
371
372
373
374
375
376
            if experiment_dict[key]['status'] == 'running':
                experiment_id_list.append(key)
        if not experiment_id_list:
            print_warning('There is no experiment running...\nYou can use \'nnictl experiment list all\' to list all stopped experiments!')
    experiment_information = ""
    for key in experiment_id_list:
377
        experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], experiment_dict[key]['port'],\
SparkSnail's avatar
SparkSnail committed
378
        experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
379
    print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
SparkSnail's avatar
SparkSnail committed
380

SparkSnail's avatar
SparkSnail committed
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
def get_time_interval(time1, time2):
    '''get the interval of two times'''
    try:
        #convert time to timestamp
        time1 = time.mktime(time.strptime(time1, '%Y-%m-%d %H:%M:%S'))
        time2 = time.mktime(time.strptime(time2, '%Y-%m-%d %H:%M:%S'))
        seconds = (datetime.datetime.fromtimestamp(time2) - datetime.datetime.fromtimestamp(time1)).seconds
        #convert seconds to day:hour:minute:second
        days = seconds / 86400
        seconds %= 86400
        hours = seconds / 3600
        seconds %= 3600
        minutes = seconds / 60
        seconds %= 60
        return '%dd %dh %dm %ds' % (days, hours, minutes, seconds)
    except:
        return 'N/A'

def show_experiment_info():
    '''show experiment information in monitor'''
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print('There is no experiment running...')
        exit(1)
    experiment_id_list = []
    for key in experiment_dict.keys():
        if experiment_dict[key]['status'] == 'running':
            experiment_id_list.append(key)
    if not experiment_id_list:
        print_warning('There is no experiment running...')
        return
    for key in experiment_id_list:
        current_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
        print(EXPERIMENT_MONITOR_INFO % (key, experiment_dict[key]['status'], experiment_dict[key]['port'], \
             experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], get_time_interval(experiment_dict[key]['startTime'], current_time)))
        print(TRIAL_MONITOR_HEAD)
        running, response = check_rest_server_quick(experiment_dict[key]['port'])
        if running:
            response = rest_get(trial_jobs_url(experiment_dict[key]['port']), 20)
            if response and check_response(response):
                content = json.loads(response.text)
423
                for index, value in enumerate(content):
SparkSnail's avatar
SparkSnail committed
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
                    content[index] = convert_time_stamp_to_date(value)
                    print(TRIAL_MONITOR_CONTENT % (content[index].get('id'), content[index].get('startTime'), content[index].get('endTime'), content[index].get('status')))
        print(TRIAL_MONITOR_TAIL)

def monitor_experiment(args):
    '''monitor the experiment'''
    if args.time <= 0:
        print_error('please input a positive integer as time interval, the unit is second.')
        exit(1)
    while True:
        try:
            os.system('clear')
            update_experiment_status()
            show_experiment_info()
            time.sleep(args.time)
        except KeyboardInterrupt:
            exit(0)
        except Exception as exception:
            print_error(exception)
            exit(1)