"...research_projects/consistency_training/requirements.txt" did not exist on "a4d5b59f132126c06c1a6b1f266ee44c70440cce"
launcher.py 17.3 KB
Newer Older
Deshui Yu's avatar
Deshui Yu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# Copyright (c) Microsoft Corporation
# All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge,
# to any person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


import json
import os
import shutil
25
import string
26
from subprocess import Popen, PIPE, call, check_output
Deshui Yu's avatar
Deshui Yu committed
27
import tempfile
28
from nni_annotation import *
Deshui Yu's avatar
Deshui Yu committed
29
from .launcher_utils import validate_all_content
30
from .rest_utils import rest_put, rest_post, check_rest_server, check_rest_server_quick, check_response
SparkSnail's avatar
SparkSnail committed
31
from .url_utils import cluster_metadata_url, experiment_url, get_local_urls
32
33
from .config_utils import Config, Experiments
from .common_utils import get_yml_content, get_json_content, print_error, print_normal, print_warning, detect_process, detect_port
34
from .constants import *
35
import time
36
37
import random
import string
38
39
from os import path
import site
Deshui Yu's avatar
Deshui Yu committed
40

41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
def get_log_path(config_file_name):
    '''generate stdout and stderr log path'''
    stdout_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stdout')
    stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
    return stdout_full_path, stderr_full_path

def print_log_content(config_file_name):
    '''print log information'''
    stdout_full_path, stderr_full_path = get_log_path(config_file_name)
    print_normal(' Stdout:')
    stdout_cmds = ['cat', stdout_full_path]
    stdout_content = check_output(stdout_cmds)
    print(stdout_content.decode('utf-8'))
    print('\n\n')
    print_normal(' Stderr:')
    stderr_cmds = ['cat', stderr_full_path]
    stderr_content = check_output(stderr_cmds)
    print(stderr_content.decode('utf-8'))


61
def start_rest_server(port, platform, mode, config_file_name, experiment_id=None):
Deshui Yu's avatar
Deshui Yu committed
62
    '''Run nni manager process'''
63
    nni_config = Config(config_file_name)
64
65
66
    if detect_port(port):
        print_error('Port %s is used by another process, please reset the port!' % port)
        exit(1)
Deshui Yu's avatar
Deshui Yu committed
67
68

    print_normal('Starting restful server...')
69
    manager = os.environ.get('NNI_MANAGER', 'nnimanager')
70
    cmds = ['node', path.join(site.USER_BASE, 'nni_pkg', 'main.js'), '--port', str(port), '--mode', platform, '--start_mode', mode]
Deshui Yu's avatar
Deshui Yu committed
71
72
    if mode == 'resume':
        cmds += ['--experiment_id', experiment_id]
73
    stdout_full_path, stderr_full_path = get_log_path(config_file_name)
goooxu's avatar
goooxu committed
74
75
    stdout_file = open(stdout_full_path, 'a+')
    stderr_file = open(stderr_full_path, 'a+')
76
77
78
79
80
    time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
    #add time information in the header of log files
    log_header = LOG_HEADER % str(time_now)
    stdout_file.write(log_header)
    stderr_file.write(log_header)
81
    process = Popen(cmds, cwd=path.join(site.USER_BASE, 'nni_pkg'), stdout=stdout_file, stderr=stderr_file)
82
    return process, str(time_now)
Deshui Yu's avatar
Deshui Yu committed
83

84
def set_trial_config(experiment_config, port, config_file_name):
85
    '''set trial configuration'''
Deshui Yu's avatar
Deshui Yu committed
86
    request_data = dict()
87
    value_dict = dict()
88
89
90
    value_dict['command'] = experiment_config['trial']['command']
    value_dict['codeDir'] = experiment_config['trial']['codeDir']
    value_dict['gpuNum'] = experiment_config['trial']['gpuNum']
91
92
93
94
95
96
97
98
99
100
    if experiment_config['trial'].get('cpuNum'):
        value_dict['cpuNum'] = experiment_config['trial']['cpuNum']
    if experiment_config['trial'].get('memoryMB'):
        value_dict['memoryMB'] = experiment_config['trial']['memoryMB']
    if experiment_config['trial'].get('image'):
        value_dict['image'] = experiment_config['trial']['image']
    if experiment_config['trial'].get('dataDir'):
        value_dict['dataDir'] = experiment_config['trial']['dataDir']
    if experiment_config['trial'].get('outputDir'):
        value_dict['outputDir'] = experiment_config['trial']['outputDir']
101
    request_data['trial_config'] = value_dict
Deshui Yu's avatar
Deshui Yu committed
102
    response = rest_put(cluster_metadata_url(port), json.dumps(request_data), 20)
103
104
105
    if check_response(response):
        return True
    else:
106
        print('Error message is {}'.format(response.text))
107
        _, stderr_full_path = get_log_path(config_file_name)
goooxu's avatar
goooxu committed
108
        with open(stderr_full_path, 'a+') as fout:
109
110
            fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
        return False
111

112
def set_local_config(experiment_config, port, config_file_name):
113
    '''set local configuration'''
114
    return set_trial_config(experiment_config, port, config_file_name)
Deshui Yu's avatar
Deshui Yu committed
115

116
def set_remote_config(experiment_config, port, config_file_name):
Deshui Yu's avatar
Deshui Yu committed
117
118
119
120
121
    '''Call setClusterMetadata to pass trial'''
    #set machine_list
    request_data = dict()
    request_data['machine_list'] = experiment_config['machineList']
    response = rest_put(cluster_metadata_url(port), json.dumps(request_data), 20)
122
    err_message = ''
123
    if not response or not check_response(response):
124
125
        if response is not None:
            err_message = response.text
126
            _, stderr_full_path = get_log_path(config_file_name)
goooxu's avatar
goooxu committed
127
            with open(stderr_full_path, 'a+') as fout:
128
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
129
        return False, err_message
Deshui Yu's avatar
Deshui Yu committed
130
131

    #set trial_config
132
    return set_trial_config(experiment_config, port, config_file_name), err_message
Deshui Yu's avatar
Deshui Yu committed
133

134
def set_pai_config(experiment_config, port, config_file_name):
135
136
137
138
    '''set pai configuration''' 
    pai_config_data = dict()
    pai_config_data['pai_config'] = experiment_config['paiConfig']
    response = rest_put(cluster_metadata_url(port), json.dumps(pai_config_data), 20)
139
    err_message = None
140
141
142
    if not response or not response.status_code == 200:
        if response is not None:
            err_message = response.text
143
            _, stderr_full_path = get_log_path(config_file_name)
144
            with open(stderr_full_path, 'a+') as fout:
chicm-ms's avatar
chicm-ms committed
145
                fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
146
147
148
        return False, err_message

    #set trial_config
149
    return set_trial_config(experiment_config, port, config_file_name), err_message
150

151
def set_experiment(experiment_config, mode, port, config_file_name):
Deshui Yu's avatar
Deshui Yu committed
152
153
154
155
156
157
158
    '''Call startExperiment (rest POST /experiment) with yaml file content'''
    request_data = dict()
    request_data['authorName'] = experiment_config['authorName']
    request_data['experimentName'] = experiment_config['experimentName']
    request_data['trialConcurrency'] = experiment_config['trialConcurrency']
    request_data['maxExecDuration'] = experiment_config['maxExecDuration']
    request_data['maxTrialNum'] = experiment_config['maxTrialNum']
159
    request_data['searchSpace'] = experiment_config.get('searchSpace')
160
161
162
163
    request_data['trainingServicePlatform'] = experiment_config.get('trainingServicePlatform')

    if experiment_config.get('description'):
        request_data['description'] = experiment_config['description']
chicm-ms's avatar
chicm-ms committed
164
165
    if experiment_config.get('multiPhase'):
        request_data['multiPhase'] = experiment_config.get('multiPhase')
Deshui Yu's avatar
Deshui Yu committed
166
167
168
169
170
171
172
    request_data['tuner'] = experiment_config['tuner']
    if 'assessor' in experiment_config:
        request_data['assessor'] = experiment_config['assessor']

    request_data['clusterMetaData'] = []
    if experiment_config['trainingServicePlatform'] == 'local':
        request_data['clusterMetaData'].append(
173
            {'key':'codeDir', 'value':experiment_config['trial']['codeDir']})
Deshui Yu's avatar
Deshui Yu committed
174
        request_data['clusterMetaData'].append(
175
            {'key': 'command', 'value': experiment_config['trial']['command']})
176
    elif experiment_config['trainingServicePlatform'] == 'remote':
Deshui Yu's avatar
Deshui Yu committed
177
178
179
        request_data['clusterMetaData'].append(
            {'key': 'machine_list', 'value': experiment_config['machineList']})
        value_dict = dict()
180
181
182
        value_dict['command'] = experiment_config['trial']['command']
        value_dict['codeDir'] = experiment_config['trial']['codeDir']
        value_dict['gpuNum'] = experiment_config['trial']['gpuNum']
Deshui Yu's avatar
Deshui Yu committed
183
184
        request_data['clusterMetaData'].append(
            {'key': 'trial_config', 'value': value_dict})
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
    elif experiment_config['trainingServicePlatform'] == 'pai':
        request_data['clusterMetaData'].append(
            {'key': 'pai_config', 'value': experiment_config['paiConfig']})
        value_dict = dict()
        value_dict['command'] = experiment_config['trial']['command']
        value_dict['codeDir'] = experiment_config['trial']['codeDir']
        value_dict['gpuNum'] = experiment_config['trial']['gpuNum']
        if experiment_config['trial'].get('cpuNum'):
            value_dict['cpuNum'] = experiment_config['trial']['cpuNum']
        if experiment_config['trial'].get('memoryMB'):
            value_dict['memoryMB'] = experiment_config['trial']['memoryMB']
        if experiment_config['trial'].get('image'):
            value_dict['image'] = experiment_config['trial']['image']
        if experiment_config['trial'].get('dataDir'):
            value_dict['dataDir'] = experiment_config['trial']['dataDir']
        if experiment_config['trial'].get('outputDir'):
            value_dict['outputDir'] = experiment_config['trial']['outputDir']
        request_data['clusterMetaData'].append(
            {'key': 'trial_config', 'value': value_dict})
Deshui Yu's avatar
Deshui Yu committed
204
205

    response = rest_post(experiment_url(port), json.dumps(request_data), 20)
206
207
208
    if check_response(response):
        return response
    else:
209
        _, stderr_full_path = get_log_path(config_file_name)
goooxu's avatar
goooxu committed
210
        with open(stderr_full_path, 'a+') as fout:
211
            fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
212
        print_error('Setting experiment error, error message is {}'.format(response.text))
213
        return None
Deshui Yu's avatar
Deshui Yu committed
214

215
def launch_experiment(args, experiment_config, mode, config_file_name, experiment_id=None):
Deshui Yu's avatar
Deshui Yu committed
216
    '''follow steps to start rest server and start experiment'''
217
    nni_config = Config(config_file_name)
Deshui Yu's avatar
Deshui Yu committed
218
    # start rest server
219
    rest_process, start_time = start_rest_server(args.port, experiment_config['trainingServicePlatform'], mode, config_file_name, experiment_id)
Deshui Yu's avatar
Deshui Yu committed
220
221
222
    nni_config.set_config('restServerPid', rest_process.pid)
    # Deal with annotation
    if experiment_config.get('useAnnotation'):
liuzhe-lz's avatar
liuzhe-lz committed
223
        path = os.path.join(tempfile.gettempdir(), 'nni', 'annotation')
QuanluZhang's avatar
QuanluZhang committed
224
225
        if not os.path.isdir(path):
            os.makedirs(path)
liuzhe-lz's avatar
liuzhe-lz committed
226
227
228
229
        path = tempfile.mkdtemp(dir=path)
        code_dir = expand_annotations(experiment_config['trial']['codeDir'], path)
        experiment_config['trial']['codeDir'] = code_dir
        search_space = generate_search_space(code_dir)
230
        experiment_config['searchSpace'] = json.dumps(search_space)
Deshui Yu's avatar
Deshui Yu committed
231
        assert search_space, ERROR_INFO % 'Generated search space is empty'
232
233
234
    elif experiment_config.get('searchSpacePath'):
            search_space = get_json_content(experiment_config.get('searchSpacePath'))
            experiment_config['searchSpace'] = json.dumps(search_space)
Deshui Yu's avatar
Deshui Yu committed
235
    else:
236
        experiment_config['searchSpace'] = json.dumps('')
Deshui Yu's avatar
Deshui Yu committed
237
238

    # check rest server
goooxu's avatar
goooxu committed
239
    running, _ = check_rest_server(args.port)
240
    if running:
241
        print_normal('Successfully started Restful server!')
Deshui Yu's avatar
Deshui Yu committed
242
243
    else:
        print_error('Restful server start failed!')
244
        print_log_content(config_file_name)
Deshui Yu's avatar
Deshui Yu committed
245
        try:
246
247
            cmds = ['pkill', '-P', str(rest_process.pid)]
            call(cmds)
Deshui Yu's avatar
Deshui Yu committed
248
249
        except Exception:
            raise Exception(ERROR_INFO % 'Rest server stopped!')
goooxu's avatar
goooxu committed
250
        exit(1)
Deshui Yu's avatar
Deshui Yu committed
251
252
253
254

    # set remote config
    if experiment_config['trainingServicePlatform'] == 'remote':
        print_normal('Setting remote config...')
255
        config_result, err_msg = set_remote_config(experiment_config, args.port, config_file_name)
256
        if config_result:
257
            print_normal('Successfully set remote config!')
Deshui Yu's avatar
Deshui Yu committed
258
        else:
259
            print_error('Failed! Error is: {}'.format(err_msg))
Deshui Yu's avatar
Deshui Yu committed
260
            try:
261
262
                cmds = ['pkill', '-P', str(rest_process.pid)]
                call(cmds)
Deshui Yu's avatar
Deshui Yu committed
263
264
            except Exception:
                raise Exception(ERROR_INFO % 'Rest server stopped!')
goooxu's avatar
goooxu committed
265
            exit(1)
Deshui Yu's avatar
Deshui Yu committed
266
267
268
269

    # set local config
    if experiment_config['trainingServicePlatform'] == 'local':
        print_normal('Setting local config...')
270
        if set_local_config(experiment_config, args.port, config_file_name):
271
            print_normal('Successfully set local config!')
Deshui Yu's avatar
Deshui Yu committed
272
        else:
273
            print_error('Set local config failed!')
Deshui Yu's avatar
Deshui Yu committed
274
            try:
275
276
                cmds = ['pkill', '-P', str(rest_process.pid)]
                call(cmds)
Deshui Yu's avatar
Deshui Yu committed
277
278
            except Exception:
                raise Exception(ERROR_INFO % 'Rest server stopped!')
goooxu's avatar
goooxu committed
279
            exit(1)
280
281
282
283
    
    #set pai config
    if experiment_config['trainingServicePlatform'] == 'pai':
        print_normal('Setting pai config...')
284
        config_result, err_msg = set_pai_config(experiment_config, args.port, config_file_name)
285
        if config_result:
286
            print_normal('Successfully set pai config!')
287
        else:
288
289
            if err_msg:
                print_error('Failed! Error is: {}'.format(err_msg))
290
291
292
293
            try:
                cmds = ['pkill', '-P', str(rest_process.pid)]
                call(cmds)
            except Exception:
294
                raise Exception(ERROR_INFO % 'Restful server stopped!')
goooxu's avatar
goooxu committed
295
            exit(1)
Deshui Yu's avatar
Deshui Yu committed
296
297
298

    # start a new experiment
    print_normal('Starting experiment...')
299
    response = set_experiment(experiment_config, mode, args.port, config_file_name)
Deshui Yu's avatar
Deshui Yu committed
300
301
302
303
304
    if response:
        if experiment_id is None:
            experiment_id = json.loads(response.text).get('experiment_id')
        nni_config.set_config('experimentId', experiment_id)
    else:
305
306
        print_error('Start experiment failed!')
        print_log_content(config_file_name)
Deshui Yu's avatar
Deshui Yu committed
307
        try:
308
309
            cmds = ['pkill', '-P', str(rest_process.pid)]
            call(cmds)
Deshui Yu's avatar
Deshui Yu committed
310
        except Exception:
311
            raise Exception(ERROR_INFO % 'Restful server stopped!')
goooxu's avatar
goooxu committed
312
        exit(1)
SparkSnail's avatar
SparkSnail committed
313
314
    web_ui_url_list = get_local_urls(args.port)
    nni_config.set_config('webuiUrl', web_ui_url_list)
315
316
317
    
    #save experiment information
    experiment_config = Experiments()
318
    experiment_config.add_experiment(experiment_id, args.port, start_time, config_file_name)
319
320

    print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, '   '.join(web_ui_url_list)))
Deshui Yu's avatar
Deshui Yu committed
321

322
323
324
325
326
327
328
329
330
def cmp_time(time1, time2):
    '''compare the time'''
    try:
        time1 = time.strptime(time1,'%Y-%m-%d %H:%M:%S')
        time2 = time.strptime(time2,'%Y-%m-%d %H:%M:%S')
        return int(time1) - int(time2)
    except:
        return 0

Deshui Yu's avatar
Deshui Yu committed
331
332
def resume_experiment(args):
    '''resume an experiment'''
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
    experiment_config = Experiments()
    experiment_dict = experiment_config.get_all_experiments()
    experiment_id = None
    experiment_endTime = None
    #find the latest stopped experiment
    if not args.id:
        for key in experiment_dict.keys():
            if experiment_dict[key]['status'] == 'stopped':
                if experiment_id is None:
                    experiment_id = key
                    experiment_endTime = experiment_dict[key]['endTime']
                else:
                    if cmp_time(experiment_dict[key]['endTime'], experiment_endTime) > 0:
                        experiment_id = key
                        experiment_endTime = experiment_dict[key]['endTime']
        if experiment_id is None:
            print_error('There is no experiment stopped!')
            exit(1)
    else:
        if experiment_dict.get(args.id) is None:
            print_error('Id %s not exist!' % args.id)
            exit(1)
        if experiment_dict[args.id]['status'] == 'running':
            print_error('Experiment %s is running!' % args.id)
            exit(1)
        experiment_id = args.id
    print_normal('Resuming experiment %s...' % experiment_id)
    nni_config = Config(experiment_dict[experiment_id]['fileName'])
Deshui Yu's avatar
Deshui Yu committed
361
362
    experiment_config = nni_config.get_config('experimentConfig')
    experiment_id = nni_config.get_config('experimentId')
363
    launch_experiment(args, experiment_config, 'resume', experiment_dict[experiment_id]['fileName'], experiment_id)
Deshui Yu's avatar
Deshui Yu committed
364
365
366

def create_experiment(args):
    '''start a new experiment'''
367
368
    config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8))
    nni_config = Config(config_file_name)
369
    config_path = os.path.abspath(args.config)
goooxu's avatar
goooxu committed
370
371
372
    if not os.path.exists(config_path):
        print_error('Please set correct config path!')
        exit(1)
373
374
    experiment_config = get_yml_content(config_path)
    validate_all_content(experiment_config, config_path)
Deshui Yu's avatar
Deshui Yu committed
375
376

    nni_config.set_config('experimentConfig', experiment_config)
377
    launch_experiment(args, experiment_config, 'new', config_file_name)
goooxu's avatar
goooxu committed
378
    nni_config.set_config('restServerPort', args.port)