launcher.py 23.7 KB
Newer Older
Deshui Yu's avatar
Deshui Yu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Copyright (c) Microsoft Corporation
# All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge,
# to any person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


import json
import os
24
import sys
Deshui Yu's avatar
Deshui Yu committed
25
import shutil
26
import string
27
from subprocess import Popen, PIPE, call, check_output, check_call
Deshui Yu's avatar
Deshui Yu committed
28
import tempfile
29
from nni.constants import ModuleName
30
from nni_annotation import *
Deshui Yu's avatar
Deshui Yu committed
31
from .launcher_utils import validate_all_content
32
from .rest_utils import rest_put, rest_post, check_rest_server, check_rest_server_quick, check_response
SparkSnail's avatar
SparkSnail committed
33
from .url_utils import cluster_metadata_url, experiment_url, get_local_urls
34
35
from .config_utils import Config, Experiments
from .common_utils import get_yml_content, get_json_content, print_error, print_normal, print_warning, detect_process, detect_port
36
from .constants import *
37
import random
QuanluZhang's avatar
QuanluZhang committed
38
import site
39
import time
Gems Guo's avatar
Gems Guo committed
40
41
from pathlib import Path

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
def get_log_path(config_file_name):
    '''generate stdout and stderr log path'''
    stdout_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stdout')
    stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
    return stdout_full_path, stderr_full_path

def print_log_content(config_file_name):
    '''print log information'''
    stdout_full_path, stderr_full_path = get_log_path(config_file_name)
    print_normal(' Stdout:')
    stdout_cmds = ['cat', stdout_full_path]
    stdout_content = check_output(stdout_cmds)
    print(stdout_content.decode('utf-8'))
    print('\n\n')
    print_normal(' Stderr:')
    stderr_cmds = ['cat', stderr_full_path]
    stderr_content = check_output(stderr_cmds)
    print(stderr_content.decode('utf-8'))

Zejun Lin's avatar
Zejun Lin committed
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
def get_nni_installation_path():
    ''' Find nni lib from the following locations in order
    Return nni root directory if it exists
    '''
    def try_installation_path_sequentially(*sitepackages):
        '''Try different installation path sequentially util nni is found.
        Return None if nothing is found
        '''
        def _generate_installation_path(sitepackages_path):
            python_dir = str(Path(sitepackages_path).parents[2])
            entry_file = os.path.join(python_dir, 'nni', 'main.js')
            if os.path.isfile(entry_file):
                return python_dir
            return None

        for sitepackage in sitepackages:
            python_dir = _generate_installation_path(sitepackage)
            if python_dir:
                return python_dir
        return None

    if os.getenv('VIRTUAL_ENV'):
        # if 'virtualenv' package is used, `site` has not attr getsitepackages, so we will instead use VIRTUAL_ENV
        # Note that conda venv will not have VIRTUAL_ENV
        python_dir = os.getenv('VIRTUAL_ENV')
    else:
        python_sitepackage = site.getsitepackages()[0]
        # If system-wide python is used, we will give priority to using `local sitepackage`--"usersitepackages()" given that nni exists there
        if python_sitepackage.startswith('/usr') or python_sitepackage.startswith('/Library'):
            python_dir = try_installation_path_sequentially(site.getusersitepackages(), site.getsitepackages()[0])
        else:
            python_dir = try_installation_path_sequentially(site.getsitepackages()[0], site.getusersitepackages())

    if python_dir:
        entry_file = os.path.join(python_dir, 'nni', 'main.js')
        if os.path.isfile(entry_file):
            return os.path.join(python_dir, 'nni')
    print_error('Fail to find nni under python library')
    exit(1)
100

101
def start_rest_server(port, platform, mode, config_file_name, experiment_id=None, log_dir=None, log_level=None):
Deshui Yu's avatar
Deshui Yu committed
102
    '''Run nni manager process'''
103
    nni_config = Config(config_file_name)
104
    if detect_port(port):
105
106
107
108
        print_error('Port %s is used by another process, please reset the port!\n' \
        'You could use \'nnictl create --help\' to get help information' % port)
        exit(1)
    
109
    if (platform == 'pai' or platform == 'kubeflow') and detect_port(int(port) + 1):
110
111
112
        print_error('PAI mode need an additional adjacent port %d, and the port %d is used by another process!\n' \
        'You could set another port to start experiment!\n' \
        'You could use \'nnictl create --help\' to get help information' % ((int(port) + 1), (int(port) + 1)))
113
        exit(1)
Deshui Yu's avatar
Deshui Yu committed
114
115

    print_normal('Starting restful server...')
Zejun Lin's avatar
Zejun Lin committed
116
117
118
    
    entry_dir = get_nni_installation_path()
    entry_file = os.path.join(entry_dir, 'main.js')
119

QuanluZhang's avatar
QuanluZhang committed
120
    cmds = ['node', entry_file, '--port', str(port), '--mode', platform, '--start_mode', mode]
121
122
123
124
    if log_dir is not None:
        cmds += ['--log_dir', log_dir]
    if log_level is not None:
        cmds += ['--log_level', log_level]
Deshui Yu's avatar
Deshui Yu committed
125
126
    if mode == 'resume':
        cmds += ['--experiment_id', experiment_id]
127
    stdout_full_path, stderr_full_path = get_log_path(config_file_name)
goooxu's avatar
goooxu committed
128
129
    stdout_file = open(stdout_full_path, 'a+')
    stderr_file = open(stderr_full_path, 'a+')
130
131
132
133
134
    time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
    #add time information in the header of log files
    log_header = LOG_HEADER % str(time_now)
    stdout_file.write(log_header)
    stderr_file.write(log_header)
QuanluZhang's avatar
QuanluZhang committed
135
    process = Popen(cmds, cwd=entry_dir, stdout=stdout_file, stderr=stderr_file)
136
    return process, str(time_now)
Deshui Yu's avatar
Deshui Yu committed
137

138
def set_trial_config(experiment_config, port, config_file_name):
139
    '''set trial configuration'''
Deshui Yu's avatar
Deshui Yu committed
140
    request_data = dict()
141
    request_data['trial_config'] = experiment_config['trial']
Deshui Yu's avatar
Deshui Yu committed
142
    response = rest_put(cluster_metadata_url(port), json.dumps(request_data), 20)
143
144
145
    if check_response(response):
        return True
    else:
146
        print('Error message is {}'.format(response.text))
147
        _, stderr_full_path = get_log_path(config_file_name)
SparkSnail's avatar
SparkSnail committed
148
149
150
        if response:
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
151
        return False
152

153
def set_local_config(experiment_config, port, config_file_name):
154
    '''set local configuration'''
155
    return set_trial_config(experiment_config, port, config_file_name)
Deshui Yu's avatar
Deshui Yu committed
156

157
def set_remote_config(experiment_config, port, config_file_name):
Deshui Yu's avatar
Deshui Yu committed
158
159
160
161
162
    '''Call setClusterMetadata to pass trial'''
    #set machine_list
    request_data = dict()
    request_data['machine_list'] = experiment_config['machineList']
    response = rest_put(cluster_metadata_url(port), json.dumps(request_data), 20)
163
    err_message = ''
164
    if not response or not check_response(response):
165
166
        if response is not None:
            err_message = response.text
167
            _, stderr_full_path = get_log_path(config_file_name)
goooxu's avatar
goooxu committed
168
            with open(stderr_full_path, 'a+') as fout:
169
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
170
        return False, err_message
Deshui Yu's avatar
Deshui Yu committed
171
172

    #set trial_config
173
    return set_trial_config(experiment_config, port, config_file_name), err_message
Deshui Yu's avatar
Deshui Yu committed
174

175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
def setNNIManagerIp(experiment_config, port, config_file_name):
    '''set nniManagerIp'''
    if experiment_config.get('nniManagerIp') is None:
        return True, None
    ip_config_dict = dict()
    ip_config_dict['nni_manager_ip'] = { 'nniManagerIp' : experiment_config['nniManagerIp'] }
    response = rest_put(cluster_metadata_url(port), json.dumps(ip_config_dict), 20)
    err_message = None
    if not response or not response.status_code == 200:
        if response is not None:
            err_message = response.text
            _, stderr_full_path = get_log_path(config_file_name)
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
        return False, err_message
    return True, None

192
def set_pai_config(experiment_config, port, config_file_name):
193
194
195
196
    '''set pai configuration''' 
    pai_config_data = dict()
    pai_config_data['pai_config'] = experiment_config['paiConfig']
    response = rest_put(cluster_metadata_url(port), json.dumps(pai_config_data), 20)
197
    err_message = None
198
199
200
    if not response or not response.status_code == 200:
        if response is not None:
            err_message = response.text
201
            _, stderr_full_path = get_log_path(config_file_name)
202
            with open(stderr_full_path, 'a+') as fout:
chicm-ms's avatar
chicm-ms committed
203
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
204
        return False, err_message
205
206
207
    result, message = setNNIManagerIp(experiment_config, port, config_file_name)
    if not result:
        return result, message
208
    #set trial_config
209
    return set_trial_config(experiment_config, port, config_file_name), err_message
210

211
212
213
214
215
216
217
218
219
220
221
222
223
def set_kubeflow_config(experiment_config, port, config_file_name):
    '''set kubeflow configuration''' 
    kubeflow_config_data = dict()
    kubeflow_config_data['kubeflow_config'] = experiment_config['kubeflowConfig']
    response = rest_put(cluster_metadata_url(port), json.dumps(kubeflow_config_data), 20)
    err_message = None
    if not response or not response.status_code == 200:
        if response is not None:
            err_message = response.text
            _, stderr_full_path = get_log_path(config_file_name)
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
        return False, err_message
224
225
226
    result, message = setNNIManagerIp(experiment_config, port, config_file_name)
    if not result:
        return result, message
227
228
229
    #set trial_config
    return set_trial_config(experiment_config, port, config_file_name), err_message

230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
def set_frameworkcontroller_config(experiment_config, port, config_file_name):
    '''set kubeflow configuration''' 
    frameworkcontroller_config_data = dict()
    frameworkcontroller_config_data['frameworkcontroller_config'] = experiment_config['frameworkcontrollerConfig']
    response = rest_put(cluster_metadata_url(port), json.dumps(frameworkcontroller_config_data), 20)
    err_message = None
    if not response or not response.status_code == 200:
        if response is not None:
            err_message = response.text
            _, stderr_full_path = get_log_path(config_file_name)
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
        return False, err_message
    result, message = setNNIManagerIp(experiment_config, port, config_file_name)
    if not result:
        return result, message
    #set trial_config
    return set_trial_config(experiment_config, port, config_file_name), err_message

249
def set_experiment(experiment_config, mode, port, config_file_name):
Deshui Yu's avatar
Deshui Yu committed
250
251
252
253
254
255
256
    '''Call startExperiment (rest POST /experiment) with yaml file content'''
    request_data = dict()
    request_data['authorName'] = experiment_config['authorName']
    request_data['experimentName'] = experiment_config['experimentName']
    request_data['trialConcurrency'] = experiment_config['trialConcurrency']
    request_data['maxExecDuration'] = experiment_config['maxExecDuration']
    request_data['maxTrialNum'] = experiment_config['maxTrialNum']
257
    request_data['searchSpace'] = experiment_config.get('searchSpace')
258
259
260
261
    request_data['trainingServicePlatform'] = experiment_config.get('trainingServicePlatform')

    if experiment_config.get('description'):
        request_data['description'] = experiment_config['description']
chicm-ms's avatar
chicm-ms committed
262
263
    if experiment_config.get('multiPhase'):
        request_data['multiPhase'] = experiment_config.get('multiPhase')
264
265
    if experiment_config.get('multiThread'):
        request_data['multiThread'] = experiment_config.get('multiThread')
QuanluZhang's avatar
QuanluZhang committed
266
267
268
269
270
271
    if experiment_config.get('advisor'):
        request_data['advisor'] = experiment_config['advisor']
    else:
        request_data['tuner'] = experiment_config['tuner']
        if 'assessor' in experiment_config:
            request_data['assessor'] = experiment_config['assessor']
Deshui Yu's avatar
Deshui Yu committed
272
273
274
275

    request_data['clusterMetaData'] = []
    if experiment_config['trainingServicePlatform'] == 'local':
        request_data['clusterMetaData'].append(
276
            {'key':'codeDir', 'value':experiment_config['trial']['codeDir']})
Deshui Yu's avatar
Deshui Yu committed
277
        request_data['clusterMetaData'].append(
278
            {'key': 'command', 'value': experiment_config['trial']['command']})
279
    elif experiment_config['trainingServicePlatform'] == 'remote':
Deshui Yu's avatar
Deshui Yu committed
280
281
282
        request_data['clusterMetaData'].append(
            {'key': 'machine_list', 'value': experiment_config['machineList']})
        request_data['clusterMetaData'].append(
283
            {'key': 'trial_config', 'value': experiment_config['trial']})
284
285
    elif experiment_config['trainingServicePlatform'] == 'pai':
        request_data['clusterMetaData'].append(
286
            {'key': 'pai_config', 'value': experiment_config['paiConfig']})        
287
        request_data['clusterMetaData'].append(
288
289
290
291
292
293
            {'key': 'trial_config', 'value': experiment_config['trial']})
    elif experiment_config['trainingServicePlatform'] == 'kubeflow':
        request_data['clusterMetaData'].append(
            {'key': 'kubeflow_config', 'value': experiment_config['kubeflowConfig']})
        request_data['clusterMetaData'].append(
            {'key': 'trial_config', 'value': experiment_config['trial']})
294
295
296
297
298
    elif experiment_config['trainingServicePlatform'] == 'frameworkcontroller':
        request_data['clusterMetaData'].append(
            {'key': 'frameworkcontroller_config', 'value': experiment_config['frameworkcontrollerConfig']})
        request_data['clusterMetaData'].append(
            {'key': 'trial_config', 'value': experiment_config['trial']})
Deshui Yu's avatar
Deshui Yu committed
299
300

    response = rest_post(experiment_url(port), json.dumps(request_data), 20)
301
302
303
    if check_response(response):
        return response
    else:
304
        _, stderr_full_path = get_log_path(config_file_name)
SparkSnail's avatar
SparkSnail committed
305
306
307
        if response:
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
SparkSnail's avatar
SparkSnail committed
308
            print_error('Setting experiment error, error message is {}'.format(response.text))
309
        return None
Deshui Yu's avatar
Deshui Yu committed
310

311
def launch_experiment(args, experiment_config, mode, config_file_name, experiment_id=None):
Deshui Yu's avatar
Deshui Yu committed
312
    '''follow steps to start rest server and start experiment'''
313
    nni_config = Config(config_file_name)
314
315
316
317
318
319
320
321
322
323

    # check packages for tuner
    if experiment_config.get('tuner') and experiment_config['tuner'].get('builtinTunerName'):
        tuner_name = experiment_config['tuner']['builtinTunerName']
        module_name = ModuleName[tuner_name]
        try:
            check_call([sys.executable, '-c', 'import %s'%(module_name)])
        except ModuleNotFoundError as e:
            print_error('The tuner %s should be installed through nnictl'%(tuner_name))
            exit(1)
324
325
326
327
    log_dir = experiment_config['logDir'] if experiment_config.get('logDir') else None
    log_level = experiment_config['logLevel'] if experiment_config.get('logLevel') else None
    if log_level not in ['trace', 'debug'] and args.debug:
        log_level = 'debug'
Deshui Yu's avatar
Deshui Yu committed
328
    # start rest server
329
    rest_process, start_time = start_rest_server(args.port, experiment_config['trainingServicePlatform'], mode, config_file_name, experiment_id, log_dir, log_level)
Deshui Yu's avatar
Deshui Yu committed
330
331
332
    nni_config.set_config('restServerPid', rest_process.pid)
    # Deal with annotation
    if experiment_config.get('useAnnotation'):
Zejun Lin's avatar
Zejun Lin committed
333
        path = os.path.join(tempfile.gettempdir(), os.environ['USER'], 'nni', 'annotation')
QuanluZhang's avatar
QuanluZhang committed
334
335
        if not os.path.isdir(path):
            os.makedirs(path)
liuzhe-lz's avatar
liuzhe-lz committed
336
337
338
339
        path = tempfile.mkdtemp(dir=path)
        code_dir = expand_annotations(experiment_config['trial']['codeDir'], path)
        experiment_config['trial']['codeDir'] = code_dir
        search_space = generate_search_space(code_dir)
340
        experiment_config['searchSpace'] = json.dumps(search_space)
Deshui Yu's avatar
Deshui Yu committed
341
        assert search_space, ERROR_INFO % 'Generated search space is empty'
342
    elif experiment_config.get('searchSpacePath'):
Zejun Lin's avatar
Zejun Lin committed
343
344
        search_space = get_json_content(experiment_config.get('searchSpacePath'))
        experiment_config['searchSpace'] = json.dumps(search_space)
Deshui Yu's avatar
Deshui Yu committed
345
    else:
346
        experiment_config['searchSpace'] = json.dumps('')
Deshui Yu's avatar
Deshui Yu committed
347
348

    # check rest server
goooxu's avatar
goooxu committed
349
    running, _ = check_rest_server(args.port)
350
    if running:
351
        print_normal('Successfully started Restful server!')
Deshui Yu's avatar
Deshui Yu committed
352
353
    else:
        print_error('Restful server start failed!')
354
        print_log_content(config_file_name)
Deshui Yu's avatar
Deshui Yu committed
355
        try:
Gems Guo's avatar
Gems Guo committed
356
            cmds = ['kill', str(rest_process.pid)]
357
            call(cmds)
Deshui Yu's avatar
Deshui Yu committed
358
359
        except Exception:
            raise Exception(ERROR_INFO % 'Rest server stopped!')
goooxu's avatar
goooxu committed
360
        exit(1)
Deshui Yu's avatar
Deshui Yu committed
361
362
363
364

    # set remote config
    if experiment_config['trainingServicePlatform'] == 'remote':
        print_normal('Setting remote config...')
365
        config_result, err_msg = set_remote_config(experiment_config, args.port, config_file_name)
366
        if config_result:
367
            print_normal('Successfully set remote config!')
Deshui Yu's avatar
Deshui Yu committed
368
        else:
369
            print_error('Failed! Error is: {}'.format(err_msg))
Deshui Yu's avatar
Deshui Yu committed
370
            try:
Gems Guo's avatar
Gems Guo committed
371
                cmds = ['kill', str(rest_process.pid)]
372
                call(cmds)
Deshui Yu's avatar
Deshui Yu committed
373
374
            except Exception:
                raise Exception(ERROR_INFO % 'Rest server stopped!')
goooxu's avatar
goooxu committed
375
            exit(1)
Deshui Yu's avatar
Deshui Yu committed
376
377
378
379

    # set local config
    if experiment_config['trainingServicePlatform'] == 'local':
        print_normal('Setting local config...')
380
        if set_local_config(experiment_config, args.port, config_file_name):
381
            print_normal('Successfully set local config!')
Deshui Yu's avatar
Deshui Yu committed
382
        else:
383
            print_error('Set local config failed!')
Deshui Yu's avatar
Deshui Yu committed
384
            try:
Gems Guo's avatar
Gems Guo committed
385
                cmds = ['kill', str(rest_process.pid)]
386
                call(cmds)
Deshui Yu's avatar
Deshui Yu committed
387
388
            except Exception:
                raise Exception(ERROR_INFO % 'Rest server stopped!')
goooxu's avatar
goooxu committed
389
            exit(1)
390
391
392
393
    
    #set pai config
    if experiment_config['trainingServicePlatform'] == 'pai':
        print_normal('Setting pai config...')
394
        config_result, err_msg = set_pai_config(experiment_config, args.port, config_file_name)
395
        if config_result:
396
            print_normal('Successfully set pai config!')
397
        else:
398
399
            if err_msg:
                print_error('Failed! Error is: {}'.format(err_msg))
400
            try:
Gems Guo's avatar
Gems Guo committed
401
                cmds = ['kill', str(rest_process.pid)]
402
403
                call(cmds)
            except Exception:
404
                raise Exception(ERROR_INFO % 'Restful server stopped!')
goooxu's avatar
goooxu committed
405
            exit(1)
406
407
408
409
410
411
412
413
414
415
416
    
    #set kubeflow config
    if experiment_config['trainingServicePlatform'] == 'kubeflow':
        print_normal('Setting kubeflow config...')
        config_result, err_msg = set_kubeflow_config(experiment_config, args.port, config_file_name)
        if config_result:
            print_normal('Successfully set kubeflow config!')
        else:
            if err_msg:
                print_error('Failed! Error is: {}'.format(err_msg))
            try:
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
                cmds = ['pkill', str(rest_process.pid)]
                call(cmds)
            except Exception:
                raise Exception(ERROR_INFO % 'Restful server stopped!')
            exit(1)
    
        #set kubeflow config
    if experiment_config['trainingServicePlatform'] == 'frameworkcontroller':
        print_normal('Setting frameworkcontroller config...')
        config_result, err_msg = set_frameworkcontroller_config(experiment_config, args.port, config_file_name)
        if config_result:
            print_normal('Successfully set frameworkcontroller config!')
        else:
            if err_msg:
                print_error('Failed! Error is: {}'.format(err_msg))
            try:
                cmds = ['pkill', str(rest_process.pid)]
434
435
436
437
                call(cmds)
            except Exception:
                raise Exception(ERROR_INFO % 'Restful server stopped!')
            exit(1)
Deshui Yu's avatar
Deshui Yu committed
438
439
440

    # start a new experiment
    print_normal('Starting experiment...')
441
    response = set_experiment(experiment_config, mode, args.port, config_file_name)
Deshui Yu's avatar
Deshui Yu committed
442
443
444
445
446
    if response:
        if experiment_id is None:
            experiment_id = json.loads(response.text).get('experiment_id')
        nni_config.set_config('experimentId', experiment_id)
    else:
447
448
        print_error('Start experiment failed!')
        print_log_content(config_file_name)
Deshui Yu's avatar
Deshui Yu committed
449
        try:
Gems Guo's avatar
Gems Guo committed
450
            cmds = ['kill', str(rest_process.pid)]
451
            call(cmds)
Deshui Yu's avatar
Deshui Yu committed
452
        except Exception:
453
            raise Exception(ERROR_INFO % 'Restful server stopped!')
goooxu's avatar
goooxu committed
454
        exit(1)
455
456
457
458
    if experiment_config.get('nniManagerIp'):
        web_ui_url_list = ['{0}:{1}'.format(experiment_config['nniManagerIp'], str(args.port))]
    else:
        web_ui_url_list = get_local_urls(args.port)
SparkSnail's avatar
SparkSnail committed
459
    nni_config.set_config('webuiUrl', web_ui_url_list)
460
461
    
    #save experiment information
SparkSnail's avatar
SparkSnail committed
462
463
    nnictl_experiment_config = Experiments()
    nnictl_experiment_config.add_experiment(experiment_id, args.port, start_time, config_file_name, experiment_config['trainingServicePlatform'])
464
465

    print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, '   '.join(web_ui_url_list)))
Deshui Yu's avatar
Deshui Yu committed
466
467
468

def resume_experiment(args):
    '''resume an experiment'''
469
470
471
472
473
474
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    experiment_id = None
    experiment_endTime = None
    #find the latest stopped experiment
    if not args.id:
SparkSnail's avatar
SparkSnail committed
475
476
477
        print_error('Please set experiment id! \nYou could use \'nnictl resume {id}\' to resume a stopped experiment!\n' \
        'You could use \'nnictl experiment list all\' to show all of stopped experiments!')
        exit(1)
478
479
480
481
482
483
484
485
486
487
    else:
        if experiment_dict.get(args.id) is None:
            print_error('Id %s not exist!' % args.id)
            exit(1)
        if experiment_dict[args.id]['status'] == 'running':
            print_error('Experiment %s is running!' % args.id)
            exit(1)
        experiment_id = args.id
    print_normal('Resuming experiment %s...' % experiment_id)
    nni_config = Config(experiment_dict[experiment_id]['fileName'])
Deshui Yu's avatar
Deshui Yu committed
488
489
    experiment_config = nni_config.get_config('experimentConfig')
    experiment_id = nni_config.get_config('experimentId')
SparkSnail's avatar
SparkSnail committed
490
491
492
493
494
    new_config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8))
    new_nni_config = Config(new_config_file_name)
    new_nni_config.set_config('experimentConfig', experiment_config)
    launch_experiment(args, experiment_config, 'resume', new_config_file_name, experiment_id)
    new_nni_config.set_config('restServerPort', args.port)
Deshui Yu's avatar
Deshui Yu committed
495
496
497

def create_experiment(args):
    '''start a new experiment'''
498
499
    config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8))
    nni_config = Config(config_file_name)
500
    config_path = os.path.abspath(args.config)
goooxu's avatar
goooxu committed
501
502
503
    if not os.path.exists(config_path):
        print_error('Please set correct config path!')
        exit(1)
504
505
    experiment_config = get_yml_content(config_path)
    validate_all_content(experiment_config, config_path)
Deshui Yu's avatar
Deshui Yu committed
506
507

    nni_config.set_config('experimentConfig', experiment_config)
508
    launch_experiment(args, experiment_config, 'new', config_file_name)
goooxu's avatar
goooxu committed
509
    nni_config.set_config('restServerPort', args.port)