launcher.py 28.5 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
Deshui Yu's avatar
Deshui Yu committed
3
4
5

import json
import os
6
import sys
7
import string
chicm-ms's avatar
chicm-ms committed
8
9
import random
import time
Deshui Yu's avatar
Deshui Yu committed
10
import tempfile
11
from subprocess import Popen, check_call, CalledProcessError, PIPE, STDOUT
chicm-ms's avatar
chicm-ms committed
12
from nni_annotation import expand_annotations, generate_search_space
chicm-ms's avatar
chicm-ms committed
13
from nni.package_utils import get_builtin_module_class_name, get_nni_installation_path
Deshui Yu's avatar
Deshui Yu committed
14
from .launcher_utils import validate_all_content
chicm-ms's avatar
chicm-ms committed
15
from .rest_utils import rest_put, rest_post, check_rest_server, check_response
SparkSnail's avatar
SparkSnail committed
16
from .url_utils import cluster_metadata_url, experiment_url, get_local_urls
17
from .config_utils import Config, Experiments
chicm-ms's avatar
chicm-ms committed
18
from .common_utils import get_yml_content, get_json_content, print_error, print_normal, \
chicm-ms's avatar
chicm-ms committed
19
20
21
                          detect_port, get_user

from .constants import NNICTL_HOME_DIR, ERROR_INFO, REST_TIME_OUT, EXPERIMENT_SUCCESS_INFO, LOG_HEADER, INSTALLABLE_PACKAGE_META
22
from .command_utils import check_output_command, kill_command
23
from .nnictl_utils import update_experiment
Gems Guo's avatar
Gems Guo committed
24

25
26
27
28
29
30
31
32
33
34
def get_log_path(config_file_name):
    '''generate stdout and stderr log path'''
    stdout_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stdout')
    stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
    return stdout_full_path, stderr_full_path

def print_log_content(config_file_name):
    '''print log information'''
    stdout_full_path, stderr_full_path = get_log_path(config_file_name)
    print_normal(' Stdout:')
35
    print(check_output_command(stdout_full_path))
36
37
    print('\n\n')
    print_normal(' Stderr:')
38
    print(check_output_command(stderr_full_path))
39

SparkSnail's avatar
SparkSnail committed
40
def start_rest_server(port, platform, mode, config_file_name, foreground=False, experiment_id=None, log_dir=None, log_level=None):
Deshui Yu's avatar
Deshui Yu committed
41
    '''Run nni manager process'''
SparkSnail's avatar
SparkSnail committed
42
    if detect_port(port):
43
        print_error('Port %s is used by another process, please reset the port!\n' \
SparkSnail's avatar
SparkSnail committed
44
        'You could use \'nnictl create --help\' to get help information' % port)
45
        exit(1)
46

SparkSnail's avatar
SparkSnail committed
47
    if (platform != 'local') and detect_port(int(port) + 1):
48
49
        print_error('PAI mode need an additional adjacent port %d, and the port %d is used by another process!\n' \
        'You could set another port to start experiment!\n' \
SparkSnail's avatar
SparkSnail committed
50
        'You could use \'nnictl create --help\' to get help information' % ((int(port) + 1), (int(port) + 1)))
51
        exit(1)
Deshui Yu's avatar
Deshui Yu committed
52
53

    print_normal('Starting restful server...')
54

Zejun Lin's avatar
Zejun Lin committed
55
    entry_dir = get_nni_installation_path()
chicm-ms's avatar
chicm-ms committed
56
57
58
    if (not entry_dir) or (not os.path.exists(entry_dir)):
        print_error('Fail to find nni under python library')
        exit(1)
Zejun Lin's avatar
Zejun Lin committed
59
    entry_file = os.path.join(entry_dir, 'main.js')
chicm-ms's avatar
chicm-ms committed
60

demianzhang's avatar
demianzhang committed
61
62
63
    node_command = 'node'
    if sys.platform == 'win32':
        node_command = os.path.join(entry_dir[:-3], 'Scripts', 'node.exe')
64
    cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(port), '--mode', platform]
SparkSnail's avatar
SparkSnail committed
65
66
67
68
69
    if mode == 'view':
        cmds += ['--start_mode', 'resume']
        cmds += ['--readonly', 'true']
    else:
        cmds += ['--start_mode', mode]
70
71
72
73
    if log_dir is not None:
        cmds += ['--log_dir', log_dir]
    if log_level is not None:
        cmds += ['--log_level', log_level]
SparkSnail's avatar
SparkSnail committed
74
    if mode in ['resume', 'view']:
Deshui Yu's avatar
Deshui Yu committed
75
        cmds += ['--experiment_id', experiment_id]
SparkSnail's avatar
SparkSnail committed
76
    if foreground:
77
        cmds += ['--foreground', 'true']
78
    stdout_full_path, stderr_full_path = get_log_path(config_file_name)
79
    with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file:
chicm-ms's avatar
chicm-ms committed
80
        time_now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
81
82
83
84
85
86
        #add time information in the header of log files
        log_header = LOG_HEADER % str(time_now)
        stdout_file.write(log_header)
        stderr_file.write(log_header)
        if sys.platform == 'win32':
            from subprocess import CREATE_NEW_PROCESS_GROUP
SparkSnail's avatar
SparkSnail committed
87
            if foreground:
88
89
90
                process = Popen(cmds, cwd=entry_dir, stdout=PIPE, stderr=STDOUT, creationflags=CREATE_NEW_PROCESS_GROUP)
            else:
                process = Popen(cmds, cwd=entry_dir, stdout=stdout_file, stderr=stderr_file, creationflags=CREATE_NEW_PROCESS_GROUP)
91
        else:
SparkSnail's avatar
SparkSnail committed
92
            if foreground:
93
94
95
                process = Popen(cmds, cwd=entry_dir, stdout=PIPE, stderr=PIPE)
            else:
                process = Popen(cmds, cwd=entry_dir, stdout=stdout_file, stderr=stderr_file)
96
    return process, str(time_now)
Deshui Yu's avatar
Deshui Yu committed
97

98
def set_trial_config(experiment_config, port, config_file_name):
99
    '''set trial configuration'''
Deshui Yu's avatar
Deshui Yu committed
100
    request_data = dict()
101
    request_data['trial_config'] = experiment_config['trial']
102
    response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT)
103
104
105
    if check_response(response):
        return True
    else:
106
        print('Error message is {}'.format(response.text))
107
        _, stderr_full_path = get_log_path(config_file_name)
SparkSnail's avatar
SparkSnail committed
108
109
110
        if response:
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
111
        return False
112

113
def set_local_config(experiment_config, port, config_file_name):
114
    '''set local configuration'''
115
116
117
    request_data = dict()
    if experiment_config.get('localConfig'):
        request_data['local_config'] = experiment_config['localConfig']
118
119
120
121
122
123
124
        if request_data['local_config']:
            if request_data['local_config'].get('gpuIndices') and isinstance(request_data['local_config'].get('gpuIndices'), int):
                request_data['local_config']['gpuIndices'] = str(request_data['local_config'].get('gpuIndices'))
            if request_data['local_config'].get('maxTrialNumOnEachGpu'):
                request_data['local_config']['maxTrialNumOnEachGpu'] = request_data['local_config'].get('maxTrialNumOnEachGpu')
            if request_data['local_config'].get('useActiveGpu'):
                request_data['local_config']['useActiveGpu'] = request_data['local_config'].get('useActiveGpu')
125
126
127
128
129
130
131
132
133
134
        response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT)
        err_message = ''
        if not response or not check_response(response):
            if response is not None:
                err_message = response.text
                _, stderr_full_path = get_log_path(config_file_name)
                with open(stderr_full_path, 'a+') as fout:
                    fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
            return False, err_message

SparkSnail's avatar
SparkSnail committed
135
    return set_trial_config(experiment_config, port, config_file_name), None
Deshui Yu's avatar
Deshui Yu committed
136

137
def set_remote_config(experiment_config, port, config_file_name):
Deshui Yu's avatar
Deshui Yu committed
138
139
140
141
    '''Call setClusterMetadata to pass trial'''
    #set machine_list
    request_data = dict()
    request_data['machine_list'] = experiment_config['machineList']
142
143
144
145
    if request_data['machine_list']:
        for i in range(len(request_data['machine_list'])):
            if isinstance(request_data['machine_list'][i].get('gpuIndices'), int):
                request_data['machine_list'][i]['gpuIndices'] = str(request_data['machine_list'][i].get('gpuIndices'))
146
147
148
    # It needs to connect all remote machines, the time out of connection is 30 seconds.
    # So timeout of this place should be longer.
    response = rest_put(cluster_metadata_url(port), json.dumps(request_data), 60, True)
149
    err_message = ''
150
    if not response or not check_response(response):
151
152
        if response is not None:
            err_message = response.text
153
            _, stderr_full_path = get_log_path(config_file_name)
goooxu's avatar
goooxu committed
154
            with open(stderr_full_path, 'a+') as fout:
155
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
156
        return False, err_message
157
158
159
    result, message = setNNIManagerIp(experiment_config, port, config_file_name)
    if not result:
        return result, message
Deshui Yu's avatar
Deshui Yu committed
160
    #set trial_config
161
    return set_trial_config(experiment_config, port, config_file_name), err_message
Deshui Yu's avatar
Deshui Yu committed
162

163
164
165
166
167
def setNNIManagerIp(experiment_config, port, config_file_name):
    '''set nniManagerIp'''
    if experiment_config.get('nniManagerIp') is None:
        return True, None
    ip_config_dict = dict()
chicm-ms's avatar
chicm-ms committed
168
    ip_config_dict['nni_manager_ip'] = {'nniManagerIp': experiment_config['nniManagerIp']}
169
    response = rest_put(cluster_metadata_url(port), json.dumps(ip_config_dict), REST_TIME_OUT)
170
171
172
173
174
175
176
177
178
179
    err_message = None
    if not response or not response.status_code == 200:
        if response is not None:
            err_message = response.text
            _, stderr_full_path = get_log_path(config_file_name)
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
        return False, err_message
    return True, None

180
def set_pai_config(experiment_config, port, config_file_name):
181
    '''set pai configuration'''
182
183
    pai_config_data = dict()
    pai_config_data['pai_config'] = experiment_config['paiConfig']
184
    response = rest_put(cluster_metadata_url(port), json.dumps(pai_config_data), REST_TIME_OUT)
185
    err_message = None
186
187
188
    if not response or not response.status_code == 200:
        if response is not None:
            err_message = response.text
189
            _, stderr_full_path = get_log_path(config_file_name)
190
            with open(stderr_full_path, 'a+') as fout:
chicm-ms's avatar
chicm-ms committed
191
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
192
        return False, err_message
193
194
195
    result, message = setNNIManagerIp(experiment_config, port, config_file_name)
    if not result:
        return result, message
196
    #set trial_config
197
    return set_trial_config(experiment_config, port, config_file_name), err_message
198

199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
def set_pai_yarn_config(experiment_config, port, config_file_name):
    '''set paiYarn configuration'''
    pai_yarn_config_data = dict()
    pai_yarn_config_data['pai_yarn_config'] = experiment_config['paiYarnConfig']
    response = rest_put(cluster_metadata_url(port), json.dumps(pai_yarn_config_data), REST_TIME_OUT)
    err_message = None
    if not response or not response.status_code == 200:
        if response is not None:
            err_message = response.text
            _, stderr_full_path = get_log_path(config_file_name)
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
        return False, err_message
    result, message = setNNIManagerIp(experiment_config, port, config_file_name)
    if not result:
        return result, message
    #set trial_config
    return set_trial_config(experiment_config, port, config_file_name), err_message

218
def set_kubeflow_config(experiment_config, port, config_file_name):
219
    '''set kubeflow configuration'''
220
221
    kubeflow_config_data = dict()
    kubeflow_config_data['kubeflow_config'] = experiment_config['kubeflowConfig']
222
    response = rest_put(cluster_metadata_url(port), json.dumps(kubeflow_config_data), REST_TIME_OUT)
223
224
225
226
227
228
229
230
    err_message = None
    if not response or not response.status_code == 200:
        if response is not None:
            err_message = response.text
            _, stderr_full_path = get_log_path(config_file_name)
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
        return False, err_message
231
232
233
    result, message = setNNIManagerIp(experiment_config, port, config_file_name)
    if not result:
        return result, message
234
235
236
    #set trial_config
    return set_trial_config(experiment_config, port, config_file_name), err_message

237
def set_frameworkcontroller_config(experiment_config, port, config_file_name):
238
    '''set kubeflow configuration'''
239
240
    frameworkcontroller_config_data = dict()
    frameworkcontroller_config_data['frameworkcontroller_config'] = experiment_config['frameworkcontrollerConfig']
241
    response = rest_put(cluster_metadata_url(port), json.dumps(frameworkcontroller_config_data), REST_TIME_OUT)
242
243
244
245
246
247
248
249
250
251
252
253
254
255
    err_message = None
    if not response or not response.status_code == 200:
        if response is not None:
            err_message = response.text
            _, stderr_full_path = get_log_path(config_file_name)
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
        return False, err_message
    result, message = setNNIManagerIp(experiment_config, port, config_file_name)
    if not result:
        return result, message
    #set trial_config
    return set_trial_config(experiment_config, port, config_file_name), err_message

George Cheng's avatar
George Cheng committed
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
def set_dlts_config(experiment_config, port, config_file_name):
    '''set dlts configuration'''
    dlts_config_data = dict()
    dlts_config_data['dlts_config'] = experiment_config['dltsConfig']
    response = rest_put(cluster_metadata_url(port), json.dumps(dlts_config_data), REST_TIME_OUT)
    err_message = None
    if not response or not response.status_code == 200:
        if response is not None:
            err_message = response.text
            _, stderr_full_path = get_log_path(config_file_name)
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
        return False, err_message
    result, message = setNNIManagerIp(experiment_config, port, config_file_name)
    if not result:
        return result, message
    #set trial_config
    return set_trial_config(experiment_config, port, config_file_name), err_message

275
def set_experiment(experiment_config, mode, port, config_file_name):
Deshui Yu's avatar
Deshui Yu committed
276
277
278
279
280
281
282
    '''Call startExperiment (rest POST /experiment) with yaml file content'''
    request_data = dict()
    request_data['authorName'] = experiment_config['authorName']
    request_data['experimentName'] = experiment_config['experimentName']
    request_data['trialConcurrency'] = experiment_config['trialConcurrency']
    request_data['maxExecDuration'] = experiment_config['maxExecDuration']
    request_data['maxTrialNum'] = experiment_config['maxTrialNum']
283
    request_data['searchSpace'] = experiment_config.get('searchSpace')
284
285
286
287
    request_data['trainingServicePlatform'] = experiment_config.get('trainingServicePlatform')

    if experiment_config.get('description'):
        request_data['description'] = experiment_config['description']
chicm-ms's avatar
chicm-ms committed
288
289
    if experiment_config.get('multiPhase'):
        request_data['multiPhase'] = experiment_config.get('multiPhase')
290
291
    if experiment_config.get('multiThread'):
        request_data['multiThread'] = experiment_config.get('multiThread')
QuanluZhang's avatar
QuanluZhang committed
292
293
    if experiment_config.get('advisor'):
        request_data['advisor'] = experiment_config['advisor']
294
295
296
297
        if request_data['advisor'].get('gpuNum'):
            print_error('gpuNum is deprecated, please use gpuIndices instead.')
        if request_data['advisor'].get('gpuIndices') and isinstance(request_data['advisor'].get('gpuIndices'), int):
            request_data['advisor']['gpuIndices'] = str(request_data['advisor'].get('gpuIndices'))
QuanluZhang's avatar
QuanluZhang committed
298
299
    else:
        request_data['tuner'] = experiment_config['tuner']
300
301
302
303
        if request_data['tuner'].get('gpuNum'):
            print_error('gpuNum is deprecated, please use gpuIndices instead.')
        if request_data['tuner'].get('gpuIndices') and isinstance(request_data['tuner'].get('gpuIndices'), int):
            request_data['tuner']['gpuIndices'] = str(request_data['tuner'].get('gpuIndices'))
QuanluZhang's avatar
QuanluZhang committed
304
305
        if 'assessor' in experiment_config:
            request_data['assessor'] = experiment_config['assessor']
306
307
            if request_data['assessor'].get('gpuNum'):
                print_error('gpuNum is deprecated, please remove it from your config file.')
SparkSnail's avatar
SparkSnail committed
308
    #debug mode should disable version check
309
    if experiment_config.get('debug') is not None:
SparkSnail's avatar
SparkSnail committed
310
        request_data['versionCheck'] = not experiment_config.get('debug')
311
312
313
    #validate version check
    if experiment_config.get('versionCheck') is not None:
        request_data['versionCheck'] = experiment_config.get('versionCheck')
SparkSnail's avatar
SparkSnail committed
314
315
    if experiment_config.get('logCollection'):
        request_data['logCollection'] = experiment_config.get('logCollection')
Deshui Yu's avatar
Deshui Yu committed
316
317
318
319

    request_data['clusterMetaData'] = []
    if experiment_config['trainingServicePlatform'] == 'local':
        request_data['clusterMetaData'].append(
320
            {'key':'codeDir', 'value':experiment_config['trial']['codeDir']})
Deshui Yu's avatar
Deshui Yu committed
321
        request_data['clusterMetaData'].append(
322
            {'key': 'command', 'value': experiment_config['trial']['command']})
323
    elif experiment_config['trainingServicePlatform'] == 'remote':
Deshui Yu's avatar
Deshui Yu committed
324
325
326
        request_data['clusterMetaData'].append(
            {'key': 'machine_list', 'value': experiment_config['machineList']})
        request_data['clusterMetaData'].append(
327
            {'key': 'trial_config', 'value': experiment_config['trial']})
328
329
    elif experiment_config['trainingServicePlatform'] == 'pai':
        request_data['clusterMetaData'].append(
330
            {'key': 'pai_config', 'value': experiment_config['paiConfig']})
331
        request_data['clusterMetaData'].append(
332
            {'key': 'trial_config', 'value': experiment_config['trial']})
333
334
335
336
337
    elif experiment_config['trainingServicePlatform'] == 'paiYarn':
        request_data['clusterMetaData'].append(
            {'key': 'pai_yarn_config', 'value': experiment_config['paiYarnConfig']})
        request_data['clusterMetaData'].append(
            {'key': 'trial_config', 'value': experiment_config['trial']})
338
339
340
341
342
    elif experiment_config['trainingServicePlatform'] == 'kubeflow':
        request_data['clusterMetaData'].append(
            {'key': 'kubeflow_config', 'value': experiment_config['kubeflowConfig']})
        request_data['clusterMetaData'].append(
            {'key': 'trial_config', 'value': experiment_config['trial']})
343
344
345
346
347
    elif experiment_config['trainingServicePlatform'] == 'frameworkcontroller':
        request_data['clusterMetaData'].append(
            {'key': 'frameworkcontroller_config', 'value': experiment_config['frameworkcontrollerConfig']})
        request_data['clusterMetaData'].append(
            {'key': 'trial_config', 'value': experiment_config['trial']})
348
    response = rest_post(experiment_url(port), json.dumps(request_data), REST_TIME_OUT, show_error=True)
349
350
351
    if check_response(response):
        return response
    else:
352
        _, stderr_full_path = get_log_path(config_file_name)
353
        if response is not None:
SparkSnail's avatar
SparkSnail committed
354
355
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
SparkSnail's avatar
SparkSnail committed
356
            print_error('Setting experiment error, error message is {}'.format(response.text))
357
        return None
Deshui Yu's avatar
Deshui Yu committed
358

SparkSnail's avatar
SparkSnail committed
359
360
361
362
363
364
365
366
367
368
def set_platform_config(platform, experiment_config, port, config_file_name, rest_process):
    '''call set_cluster_metadata for specific platform'''
    print_normal('Setting {0} config...'.format(platform))
    config_result, err_msg = None, None
    if platform == 'local':
        config_result, err_msg = set_local_config(experiment_config, port, config_file_name)
    elif platform == 'remote':
        config_result, err_msg = set_remote_config(experiment_config, port, config_file_name)
    elif platform == 'pai':
        config_result, err_msg = set_pai_config(experiment_config, port, config_file_name)
369
370
    elif platform == 'paiYarn':
        config_result, err_msg = set_pai_yarn_config(experiment_config, port, config_file_name)
SparkSnail's avatar
SparkSnail committed
371
372
373
374
    elif platform == 'kubeflow':
        config_result, err_msg = set_kubeflow_config(experiment_config, port, config_file_name)
    elif platform == 'frameworkcontroller':
        config_result, err_msg = set_frameworkcontroller_config(experiment_config, port, config_file_name)
George Cheng's avatar
George Cheng committed
375
376
    elif platform == 'dlts':
        config_result, err_msg = set_dlts_config(experiment_config, port, config_file_name)
SparkSnail's avatar
SparkSnail committed
377
378
379
380
381
382
383
384
385
386
387
388
389
    else:
        raise Exception(ERROR_INFO % 'Unsupported platform!')
        exit(1)
    if config_result:
        print_normal('Successfully set {0} config!'.format(platform))
    else:
        print_error('Failed! Error is: {}'.format(err_msg))
        try:
            kill_command(rest_process.pid)
        except Exception:
            raise Exception(ERROR_INFO % 'Rest server stopped!')
        exit(1)

390
def launch_experiment(args, experiment_config, mode, config_file_name, experiment_id=None):
Deshui Yu's avatar
Deshui Yu committed
391
    '''follow steps to start rest server and start experiment'''
392
    nni_config = Config(config_file_name)
393
    # check packages for tuner
394
    package_name, module_name = None, None
395
    if experiment_config.get('tuner') and experiment_config['tuner'].get('builtinTunerName'):
396
        package_name = experiment_config['tuner']['builtinTunerName']
chicm-ms's avatar
chicm-ms committed
397
        module_name, _ = get_builtin_module_class_name('tuners', package_name)
398
399
    elif experiment_config.get('advisor') and experiment_config['advisor'].get('builtinAdvisorName'):
        package_name = experiment_config['advisor']['builtinAdvisorName']
chicm-ms's avatar
chicm-ms committed
400
        module_name, _ = get_builtin_module_class_name('advisors', package_name)
401
    if package_name and module_name:
402
        try:
403
404
405
            stdout_full_path, stderr_full_path = get_log_path(config_file_name)
            with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file:
                check_call([sys.executable, '-c', 'import %s'%(module_name)], stdout=stdout_file, stderr=stderr_file)
chicm-ms's avatar
chicm-ms committed
406
        except CalledProcessError:
407
408
            print_error('some errors happen when import package %s.' %(package_name))
            print_log_content(config_file_name)
chicm-ms's avatar
chicm-ms committed
409
            if package_name in INSTALLABLE_PACKAGE_META:
chicm-ms's avatar
chicm-ms committed
410
411
                print_error('If %s is not installed, it should be installed through '\
                            '\'nnictl package install --name %s\''%(package_name, package_name))
412
            exit(1)
413
414
    log_dir = experiment_config['logDir'] if experiment_config.get('logDir') else None
    log_level = experiment_config['logLevel'] if experiment_config.get('logLevel') else None
SparkSnail's avatar
SparkSnail committed
415
    #view experiment mode do not need debug function, when view an experiment, there will be no new logs created
SparkSnail's avatar
SparkSnail committed
416
    foreground = False
SparkSnail's avatar
SparkSnail committed
417
    if mode != 'view':
SparkSnail's avatar
SparkSnail committed
418
        foreground = args.foreground
SparkSnail's avatar
SparkSnail committed
419
420
        if log_level not in ['trace', 'debug'] and (args.debug or experiment_config.get('debug') is True):
            log_level = 'debug'
Deshui Yu's avatar
Deshui Yu committed
421
    # start rest server
SparkSnail's avatar
SparkSnail committed
422
423
    rest_process, start_time = start_rest_server(args.port, experiment_config['trainingServicePlatform'], \
                                                 mode, config_file_name, foreground, experiment_id, log_dir, log_level)
Deshui Yu's avatar
Deshui Yu committed
424
425
426
    nni_config.set_config('restServerPid', rest_process.pid)
    # Deal with annotation
    if experiment_config.get('useAnnotation'):
427
        path = os.path.join(tempfile.gettempdir(), get_user(), 'nni', 'annotation')
QuanluZhang's avatar
QuanluZhang committed
428
429
        if not os.path.isdir(path):
            os.makedirs(path)
liuzhe-lz's avatar
liuzhe-lz committed
430
        path = tempfile.mkdtemp(dir=path)
431
432
        nas_mode = experiment_config['trial'].get('nasMode', 'classic_mode')
        code_dir = expand_annotations(experiment_config['trial']['codeDir'], path, nas_mode=nas_mode)
liuzhe-lz's avatar
liuzhe-lz committed
433
434
        experiment_config['trial']['codeDir'] = code_dir
        search_space = generate_search_space(code_dir)
435
        experiment_config['searchSpace'] = json.dumps(search_space)
Deshui Yu's avatar
Deshui Yu committed
436
        assert search_space, ERROR_INFO % 'Generated search space is empty'
437
    elif experiment_config.get('searchSpacePath'):
Zejun Lin's avatar
Zejun Lin committed
438
439
        search_space = get_json_content(experiment_config.get('searchSpacePath'))
        experiment_config['searchSpace'] = json.dumps(search_space)
Deshui Yu's avatar
Deshui Yu committed
440
    else:
441
        experiment_config['searchSpace'] = json.dumps('')
Deshui Yu's avatar
Deshui Yu committed
442
443

    # check rest server
goooxu's avatar
goooxu committed
444
    running, _ = check_rest_server(args.port)
445
    if running:
446
        print_normal('Successfully started Restful server!')
Deshui Yu's avatar
Deshui Yu committed
447
448
    else:
        print_error('Restful server start failed!')
449
        print_log_content(config_file_name)
Deshui Yu's avatar
Deshui Yu committed
450
        try:
451
            kill_command(rest_process.pid)
Deshui Yu's avatar
Deshui Yu committed
452
453
        except Exception:
            raise Exception(ERROR_INFO % 'Rest server stopped!')
goooxu's avatar
goooxu committed
454
        exit(1)
SparkSnail's avatar
SparkSnail committed
455
456
    if mode != 'view':
        # set platform configuration
chicm-ms's avatar
chicm-ms committed
457
458
459
        set_platform_config(experiment_config['trainingServicePlatform'], experiment_config, args.port,\
                            config_file_name, rest_process)

Deshui Yu's avatar
Deshui Yu committed
460
461
    # start a new experiment
    print_normal('Starting experiment...')
462
    # set debug configuration
SparkSnail's avatar
SparkSnail committed
463
    if mode != 'view' and experiment_config.get('debug') is None:
464
        experiment_config['debug'] = args.debug
465
    response = set_experiment(experiment_config, mode, args.port, config_file_name)
Deshui Yu's avatar
Deshui Yu committed
466
467
468
469
470
    if response:
        if experiment_id is None:
            experiment_id = json.loads(response.text).get('experiment_id')
        nni_config.set_config('experimentId', experiment_id)
    else:
471
472
        print_error('Start experiment failed!')
        print_log_content(config_file_name)
Deshui Yu's avatar
Deshui Yu committed
473
        try:
474
            kill_command(rest_process.pid)
Deshui Yu's avatar
Deshui Yu committed
475
        except Exception:
476
            raise Exception(ERROR_INFO % 'Restful server stopped!')
goooxu's avatar
goooxu committed
477
        exit(1)
478
479
480
481
    if experiment_config.get('nniManagerIp'):
        web_ui_url_list = ['{0}:{1}'.format(experiment_config['nniManagerIp'], str(args.port))]
    else:
        web_ui_url_list = get_local_urls(args.port)
SparkSnail's avatar
SparkSnail committed
482
    nni_config.set_config('webuiUrl', web_ui_url_list)
483

484
    # save experiment information
SparkSnail's avatar
SparkSnail committed
485
    nnictl_experiment_config = Experiments()
486
487
488
    nnictl_experiment_config.add_experiment(experiment_id, args.port, start_time, config_file_name,
                                            experiment_config['trainingServicePlatform'],
                                            experiment_config['experimentName'])
489
490

    print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, '   '.join(web_ui_url_list)))
SparkSnail's avatar
SparkSnail committed
491
    if mode != 'view' and args.foreground:
492
493
494
495
496
497
498
        try:
            while True:
                log_content = rest_process.stdout.readline().strip().decode('utf-8')
                print(log_content)
        except KeyboardInterrupt:
            kill_command(rest_process.pid)
            print_normal('Stopping experiment...')
Deshui Yu's avatar
Deshui Yu committed
499

SparkSnail's avatar
SparkSnail committed
500
501
502
503
504
505
506
507
508
def create_experiment(args):
    '''start a new experiment'''
    config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8))
    nni_config = Config(config_file_name)
    config_path = os.path.abspath(args.config)
    if not os.path.exists(config_path):
        print_error('Please set correct config path!')
        exit(1)
    experiment_config = get_yml_content(config_path)
chicm-ms's avatar
chicm-ms committed
509
510
511
512
513
    try:
        validate_all_content(experiment_config, config_path)
    except Exception as e:
        print_error(e)
        exit(1)
SparkSnail's avatar
SparkSnail committed
514
515
516

    nni_config.set_config('experimentConfig', experiment_config)
    nni_config.set_config('restServerPort', args.port)
517
518
519
520
521
522
523
524
525
    try:
        launch_experiment(args, experiment_config, 'new', config_file_name)
    except Exception as exception:
        nni_config = Config(config_file_name)
        restServerPid = nni_config.get_config('restServerPid')
        if restServerPid:
            kill_command(restServerPid)
        print_error(exception)
        exit(1)
SparkSnail's avatar
SparkSnail committed
526
527
528

def manage_stopped_experiment(args, mode):
    '''view a stopped experiment'''
SparkSnail's avatar
SparkSnail committed
529
    update_experiment()
530
531
532
533
534
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    experiment_id = None
    #find the latest stopped experiment
    if not args.id:
535
        print_error('Please set experiment id! \nYou could use \'nnictl {0} id\' to {0} a stopped experiment!\n' \
SparkSnail's avatar
SparkSnail committed
536
        'You could use \'nnictl experiment list --all\' to show all experiments!'.format(mode))
SparkSnail's avatar
SparkSnail committed
537
        exit(1)
538
539
540
541
    else:
        if experiment_dict.get(args.id) is None:
            print_error('Id %s not exist!' % args.id)
            exit(1)
542
        if experiment_dict[args.id]['status'] != 'STOPPED':
SparkSnail's avatar
SparkSnail committed
543
            print_error('Only stopped experiments can be {0}ed!'.format(mode))
544
545
            exit(1)
        experiment_id = args.id
SparkSnail's avatar
SparkSnail committed
546
    print_normal('{0} experiment {1}...'.format(mode, experiment_id))
547
    nni_config = Config(experiment_dict[experiment_id]['fileName'])
Deshui Yu's avatar
Deshui Yu committed
548
549
    experiment_config = nni_config.get_config('experimentConfig')
    experiment_id = nni_config.get_config('experimentId')
SparkSnail's avatar
SparkSnail committed
550
551
552
553
    new_config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8))
    new_nni_config = Config(new_config_file_name)
    new_nni_config.set_config('experimentConfig', experiment_config)
    new_nni_config.set_config('restServerPort', args.port)
554
555
556
557
558
559
560
561
562
    try:
        launch_experiment(args, experiment_config, mode, new_config_file_name, experiment_id)
    except Exception as exception:
        nni_config = Config(new_config_file_name)
        restServerPid = nni_config.get_config('restServerPid')
        if restServerPid:
            kill_command(restServerPid)
        print_error(exception)
        exit(1)
Deshui Yu's avatar
Deshui Yu committed
563

SparkSnail's avatar
SparkSnail committed
564
565
566
def view_experiment(args):
    '''view a stopped experiment'''
    manage_stopped_experiment(args, 'view')
Deshui Yu's avatar
Deshui Yu committed
567

SparkSnail's avatar
SparkSnail committed
568
569
def resume_experiment(args):
    '''resume an experiment'''
liuzhe-lz's avatar
liuzhe-lz committed
570
    manage_stopped_experiment(args, 'resume')