nnictl_utils.py 16 KB
Newer Older
Deshui Yu's avatar
Deshui Yu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Copyright (c) Microsoft Corporation
# All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge,
# to any person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import os
import psutil
import json
24
import datetime
25
import time
Deshui Yu's avatar
Deshui Yu committed
26
from subprocess import call, check_output
27
from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response
28
from .config_utils import Config, Experiments
Deshui Yu's avatar
Deshui Yu committed
29
from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url
30
from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT
Deshui Yu's avatar
Deshui Yu committed
31
import time
32
from .common_utils import print_normal, print_error, print_warning, detect_process
Deshui Yu's avatar
Deshui Yu committed
33

34
35
36
def check_experiment_id(args):
    '''check if the id is valid
    '''
37
38
39
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
40
        print_normal('There is no experiment running...')
chicm-ms's avatar
chicm-ms committed
41
        return None
42
    if not args.id:
43
44
        running_experiment_list = []
        for key in experiment_dict.keys():
SparkSnail's avatar
SparkSnail committed
45
46
47
48
49
50
            if isinstance(experiment_dict[key], dict):
                if experiment_dict[key].get('status') == 'running':
                    running_experiment_list.append(key)
            elif isinstance(experiment_dict[key], list):
                # if the config file is old version, remove the configuration from file
                experiment_config.remove_experiment(key)
51
52
53
54
55
        if len(running_experiment_list) > 1:
            print_error('There are multiple experiments running, please set the experiment id...')
            experiment_information = ""
            for key in running_experiment_list:
                experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
56
                experiment_dict[key]['port'], experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
57
58
59
60
            print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
            exit(1)
        elif not running_experiment_list:
            print_error('There is no experiment running!')
chicm-ms's avatar
chicm-ms committed
61
            return None
62
63
        else:
            return running_experiment_list[0]
64
    if experiment_dict.get(args.id):
65
        return args.id
66
    else:
67
        print_error('Id not correct!')
chicm-ms's avatar
chicm-ms committed
68
        return None
Deshui Yu's avatar
Deshui Yu committed
69

70
def parse_ids(args):
71
72
73
74
75
76
77
78
    '''Parse the arguments for nnictl stop
    1.If there is an id specified, return the corresponding id
    2.If there is no id specified, and there is an experiment running, return the id, or return Error
    3.If the id matches an experiment, nnictl will return the id.
    4.If the id ends with *, nnictl will match all ids matchs the regular
    5.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id
    6.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information
    '''
79
80
81
82
83
84
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print_normal('Experiment is not running...')
        return None
    result_list = []
85
86
    running_experiment_list = []
    for key in experiment_dict.keys():
SparkSnail's avatar
SparkSnail committed
87
88
89
90
91
92
        if isinstance(experiment_dict[key], dict):
            if experiment_dict[key].get('status') == 'running':
                running_experiment_list.append(key)
        elif isinstance(experiment_dict[key], list):
            # if the config file is old version, remove the configuration from file
            experiment_config.remove_experiment(key)
93
    if not args.id:
94
        if len(running_experiment_list) > 1:
95
96
            print_error('There are multiple experiments running, please set the experiment id...')
            experiment_information = ""
97
98
            for key in running_experiment_list:
                experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
99
                experiment_dict[key]['port'], experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
100
101
102
103
            print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
            exit(1)
        else:
            result_list = running_experiment_list
104
    elif args.id == 'all':
105
        result_list = running_experiment_list
106
    elif args.id.endswith('*'):
107
        for id in running_experiment_list:
108
109
            if id.startswith(args.id[:-1]):
                result_list.append(id)
110
    elif args.id in running_experiment_list:
111
112
        result_list.append(args.id)
    else:
113
        for id in running_experiment_list:
114
115
116
117
118
            if id.startswith(args.id):
                result_list.append(id)
        if len(result_list) > 1:
            print_error(args.id + ' is ambiguous, please choose ' + ' '.join(result_list) )
            return None
chicm-ms's avatar
chicm-ms committed
119
120
121
122
    if not result_list and args.id:
        print_error('There are no experiments matched, please set correct experiment id...')
    elif not result_list:
        print_error('There is no experiment running...')
123
124
    return result_list

125
126
127
def get_config_filename(args):
    '''get the file name of config file'''
    experiment_id = check_experiment_id(args)
chicm-ms's avatar
chicm-ms committed
128
129
130
    if experiment_id is None:
        print_error('Please set the experiment id!')
        exit(1)
131
132
133
134
135
136
137
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    return experiment_dict[experiment_id]['fileName']

def get_experiment_port(args):
    '''get the port of experiment'''
    experiment_id = check_experiment_id(args)
chicm-ms's avatar
chicm-ms committed
138
139
140
    if experiment_id is None:
        print_error('Please set the experiment id!')
        exit(1)
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    return experiment_dict[experiment_id]['port']

def convert_time_stamp_to_date(content):
    '''Convert time stamp to date time format'''
    start_time_stamp = content.get('startTime')
    end_time_stamp = content.get('endTime')
    if start_time_stamp:
        start_time = datetime.datetime.utcfromtimestamp(start_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S")
        content['startTime'] = str(start_time)
    if end_time_stamp:
        end_time = datetime.datetime.utcfromtimestamp(end_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S")
        content['endTime'] = str(end_time)
    return content

def check_rest(args):
    '''check if restful server is running'''
    nni_config = Config(get_config_filename(args))
    rest_port = nni_config.get_config('restServerPort')
    running, _ = check_rest_server_quick(rest_port)
    if not running:
        print_normal('Restful server is running...')
    else:
        print_normal('Restful server is not running...')

Deshui Yu's avatar
Deshui Yu committed
167
168
def stop_experiment(args):
    '''Stop the experiment which is running'''
169
170
171
172
173
174
    experiment_id_list = parse_ids(args)
    if experiment_id_list:
        experiment_config = Experiments()
        experiment_dict = experiment_config.get_all_experiments()
        for experiment_id in experiment_id_list:
            print_normal('Stoping experiment %s' % experiment_id)
175
            nni_config = Config(experiment_dict[experiment_id]['fileName'])
176
177
178
179
            rest_port = nni_config.get_config('restServerPort')
            rest_pid = nni_config.get_config('restServerPid')
            if not detect_process(rest_pid):
                print_normal('Experiment is not running...')
180
                experiment_config.update_experiment(experiment_id, 'status', 'stopped')
181
182
183
184
185
186
                return
            running, _ = check_rest_server_quick(rest_port)
            stop_rest_result = True
            if running:
                response = rest_delete(experiment_url(rest_port), 20)
                if not response or not check_response(response):
187
188
189
190
                    if response:
                        print_error(response.text)
                    else:
                        print_error('No response from restful server!')
191
192
193
194
195
                    stop_rest_result = False
            #sleep to wait rest handler done
            time.sleep(3)
            rest_pid = nni_config.get_config('restServerPid')
            if rest_pid:
196
                stop_rest_cmds = ['kill', str(rest_pid)]
SparkSnail's avatar
SparkSnail committed
197
198
199
200
201
202
203
204
205
206
                call(stop_rest_cmds)
                tensorboard_pid_list = nni_config.get_config('tensorboardPidList')
                if tensorboard_pid_list:
                    for tensorboard_pid in tensorboard_pid_list:
                        try:
                            cmds = ['kill', '-9', str(tensorboard_pid)]
                            call(cmds)
                        except Exception as exception:
                            print_error(exception)
                    nni_config.set_config('tensorboardPidList', [])
207
208
            if stop_rest_result:
                print_normal('Stop experiment success!')
209
210
211
            experiment_config.update_experiment(experiment_id, 'status', 'stopped')
            time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
            experiment_config.update_experiment(experiment_id, 'endTime', str(time_now))
Deshui Yu's avatar
Deshui Yu committed
212
213
214

def trial_ls(args):
    '''List trial'''
215
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
216
217
218
219
220
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
221
222
    running, response = check_rest_server_quick(rest_port)
    if running:
Deshui Yu's avatar
Deshui Yu committed
223
        response = rest_get(trial_jobs_url(rest_port), 20)
224
        if response and check_response(response):
225
226
227
228
            content = json.loads(response.text)
            for index, value in enumerate(content):               
                content[index] = convert_time_stamp_to_date(value)
            print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
Deshui Yu's avatar
Deshui Yu committed
229
230
231
232
233
234
235
        else:
            print_error('List trial failed...')
    else:
        print_error('Restful server is not running...')

def trial_kill(args):
    '''List trial'''
236
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
237
238
239
240
241
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
242
243
    running, _ = check_rest_server_quick(rest_port)
    if running:
Deshui Yu's avatar
Deshui Yu committed
244
        response = rest_delete(trial_job_id_url(rest_port, args.trialid), 20)
245
        if response and check_response(response):
Deshui Yu's avatar
Deshui Yu committed
246
247
248
249
250
251
252
253
            print(response.text)
        else:
            print_error('Kill trial job failed...')
    else:
        print_error('Restful server is not running...')

def list_experiment(args):
    '''Get experiment information'''
254
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
255
256
257
258
259
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
260
261
    running, _ = check_rest_server_quick(rest_port)
    if running:
Deshui Yu's avatar
Deshui Yu committed
262
        response = rest_get(experiment_url(rest_port), 20)
263
        if response and check_response(response):
264
265
            content = convert_time_stamp_to_date(json.loads(response.text))
            print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
Deshui Yu's avatar
Deshui Yu committed
266
267
268
269
270
        else:
            print_error('List experiment failed...')
    else:
        print_error('Restful server is not running...')

271
272
def experiment_status(args):
    '''Show the status of experiment'''
273
    nni_config = Config(get_config_filename(args))
274
275
276
277
278
279
280
    rest_port = nni_config.get_config('restServerPort')
    result, response = check_rest_server_quick(rest_port)
    if not result:
        print_normal('Restful server is not running...')
    else:
        print(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))

Deshui Yu's avatar
Deshui Yu committed
281
282
283
284
285
286
287
288
289
290
def get_log_content(file_name, cmds):
    '''use cmds to read config content'''
    if os.path.exists(file_name):
        rest = check_output(cmds)
        print(rest.decode('utf-8'))
    else:
        print_normal('NULL!')

def log_internal(args, filetype):
    '''internal function to call get_log_content'''
291
    file_name = get_config_filename(args)
Deshui Yu's avatar
Deshui Yu committed
292
    if filetype == 'stdout':
293
        file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stdout')
Deshui Yu's avatar
Deshui Yu committed
294
    else:
295
        file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stderr')
Deshui Yu's avatar
Deshui Yu committed
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
    if args.head:
        get_log_content(file_full_path, ['head', '-' + str(args.head), file_full_path])
    elif args.tail:
        get_log_content(file_full_path, ['tail', '-' + str(args.tail), file_full_path])
    elif args.path:
        print_normal('The path of stdout file is: ' + file_full_path)
    else:
        get_log_content(file_full_path, ['cat', file_full_path])

def log_stdout(args):
    '''get stdout log'''
    log_internal(args, 'stdout')

def log_stderr(args):
    '''get stderr log'''
    log_internal(args, 'stderr')

313
314
315
def log_trial(args):
    ''''get trial log path'''
    trial_id_path_dict = {}
316
    nni_config = Config(get_config_filename(args))
317
318
319
320
321
322
323
324
325
326
327
328
329
330
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
    running, response = check_rest_server_quick(rest_port)
    if running:
        response = rest_get(trial_jobs_url(rest_port), 20)
        if response and check_response(response):
            content = json.loads(response.text)
            for trial in content:
                trial_id_path_dict[trial['id']] = trial['logPath']
    else:
        print_error('Restful server is not running...')
goooxu's avatar
goooxu committed
331
        exit(1)
332
333
334
335
336
    if args.id:
        if trial_id_path_dict.get(args.id):
            print('id:' + args.id + ' path:' + trial_id_path_dict[args.id])
        else:
            print_error('trial id is not valid!')
goooxu's avatar
goooxu committed
337
            exit(1)
338
339
340
341
    else:
        for key in trial_id_path_dict.keys():
            print('id:' + key + ' path:' + trial_id_path_dict[key])

Deshui Yu's avatar
Deshui Yu committed
342
343
def get_config(args):
    '''get config info'''
344
    nni_config = Config(get_config_filename(args))
Deshui Yu's avatar
Deshui Yu committed
345
    print(nni_config.get_all_config())
346
347
348

def webui_url(args):
    '''show the url of web ui'''
349
    nni_config = Config(get_config_filename(args))
350
351
    print_normal('{0} {1}'.format('Web UI url:', ' '.join(nni_config.get_config('webuiUrl'))))

352
353
def experiment_list(args):
    '''get the information of all experiments'''
354
355
356
357
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    if not experiment_dict:
        print('There is no experiment running...')
358
359
360
361
362
        exit(1)
    experiment_id_list = []
    if args.all and args.all == 'all':
        for key in experiment_dict.keys():
            experiment_id_list.append(key)
363
364
    else:
        for key in experiment_dict.keys():
365
366
367
368
369
370
            if experiment_dict[key]['status'] == 'running':
                experiment_id_list.append(key)
        if not experiment_id_list:
            print_warning('There is no experiment running...\nYou can use \'nnictl experiment list all\' to list all stopped experiments!')
    experiment_information = ""
    for key in experiment_id_list:
371
        experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], experiment_dict[key]['port'],\
372
373
        experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
    print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
SparkSnail's avatar
SparkSnail committed
374