naive_test.py 5.15 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
Zejun Lin's avatar
Zejun Lin committed
3

chicm-ms's avatar
chicm-ms committed
4
5
import sys
import os.path as osp
chicm-ms's avatar
chicm-ms committed
6
import argparse
Zejun Lin's avatar
Zejun Lin committed
7
8
9
10
11
12
import json
import subprocess
import sys
import time
import traceback

13
from utils import is_experiment_done, get_experiment_id, get_nni_log_path, read_last_line, remove_files, setup_experiment, detect_port, wait_for_port_available
14
from utils import GREEN, RED, CLEAR, EXPERIMENT_URL
Zejun Lin's avatar
Zejun Lin committed
15

chicm-ms's avatar
chicm-ms committed
16
NNI_SOURCE_DIR = '..'
17
NAIVE_TEST_CONFIG_DIR = osp.join(NNI_SOURCE_DIR, 'test', 'training_service', 'config', 'naive_test')
chicm-ms's avatar
chicm-ms committed
18
19

def naive_test(args):
20
    '''run naive integration test'''
Zejun Lin's avatar
Zejun Lin committed
21
    to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt']
chicm-ms's avatar
chicm-ms committed
22
    to_remove = list(map(lambda file: osp.join(NAIVE_TEST_CONFIG_DIR, file), to_remove))
Zejun Lin's avatar
Zejun Lin committed
23
24
    remove_files(to_remove)

chicm-ms's avatar
chicm-ms committed
25
    proc = subprocess.run(['nnictl', 'create', '--config', args.config])
Zejun Lin's avatar
Zejun Lin committed
26
27
28
29
    assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode

    print('Spawning trials...')

30
    nnimanager_log_path = get_nni_log_path(EXPERIMENT_URL)
Zejun Lin's avatar
Zejun Lin committed
31
32
    current_trial = 0

Zejun Lin's avatar
Zejun Lin committed
33
    for _ in range(120):
Zejun Lin's avatar
Zejun Lin committed
34
35
        time.sleep(1)

chicm-ms's avatar
chicm-ms committed
36
37
        tuner_status = read_last_line(osp.join(NAIVE_TEST_CONFIG_DIR, 'tuner_result.txt'))
        assessor_status = read_last_line(osp.join(NAIVE_TEST_CONFIG_DIR, 'assessor_result.txt'))
38
        experiment_status = is_experiment_done(nnimanager_log_path)
Zejun Lin's avatar
Zejun Lin committed
39
40
41
42
43
44
45
46

        assert tuner_status != 'ERROR', 'Tuner exited with error'
        assert assessor_status != 'ERROR', 'Assessor exited with error'

        if experiment_status:
            break

        if tuner_status is not None:
chicm-ms's avatar
chicm-ms committed
47
            for line in open(osp.join(NAIVE_TEST_CONFIG_DIR, 'tuner_result.txt')):
Zejun Lin's avatar
Zejun Lin committed
48
49
50
51
52
53
54
                if line.strip() == 'ERROR':
                    break
                trial = int(line.split(' ')[0])
                if trial > current_trial:
                    current_trial = trial
                    print('Trial #%d done' % trial)

Zejun Lin's avatar
Zejun Lin committed
55
    assert experiment_status, 'Failed to finish in 2 min'
Zejun Lin's avatar
Zejun Lin committed
56

chicm-ms's avatar
chicm-ms committed
57
58
    ss1 = json.load(open(osp.join(NAIVE_TEST_CONFIG_DIR, 'search_space.json')))
    ss2 = json.load(open(osp.join(NAIVE_TEST_CONFIG_DIR, 'tuner_search_space.json')))
Zejun Lin's avatar
Zejun Lin committed
59
60
    assert ss1 == ss2, 'Tuner got wrong search space'

chicm-ms's avatar
chicm-ms committed
61
62
    tuner_result = set(open(osp.join(NAIVE_TEST_CONFIG_DIR, 'tuner_result.txt')))
    expected = set(open(osp.join(NAIVE_TEST_CONFIG_DIR, 'expected_tuner_result.txt')))
Zejun Lin's avatar
Zejun Lin committed
63
64
    # Trials may complete before NNI gets assessor's result,
    # so it is possible to have more final result than expected
chicm-ms's avatar
chicm-ms committed
65
66
    print('Tuner result:', tuner_result)
    print('Expected tuner result:', expected)
Zejun Lin's avatar
Zejun Lin committed
67
68
    assert tuner_result.issuperset(expected), 'Bad tuner result'

chicm-ms's avatar
chicm-ms committed
69
70
    assessor_result = set(open(osp.join(NAIVE_TEST_CONFIG_DIR, 'assessor_result.txt')))
    expected = set(open(osp.join(NAIVE_TEST_CONFIG_DIR, 'expected_assessor_result.txt')))
Zejun Lin's avatar
Zejun Lin committed
71
72
    assert assessor_result == expected, 'Bad assessor result'

73
    subprocess.run(['nnictl', 'stop'])
74
    wait_for_port_available(8080, 10)
75

chicm-ms's avatar
chicm-ms committed
76
77
def stop_experiment_test(args):
    config_file = args.config
78
79
    '''Test `nnictl stop` command, including `nnictl stop exp_id` and `nnictl stop all`.
    Simple `nnictl stop` is not tested here since it is used in all other test code'''
chicm-ms's avatar
chicm-ms committed
80
81
82
83
    subprocess.run(['nnictl', 'create', '--config', config_file, '--port', '8080'], check=True)
    subprocess.run(['nnictl', 'create', '--config', config_file, '--port', '8888'], check=True)
    subprocess.run(['nnictl', 'create', '--config', config_file, '--port', '8989'], check=True)
    subprocess.run(['nnictl', 'create', '--config', config_file, '--port', '8990'], check=True)
84
85
86
87
88

    # test cmd 'nnictl stop id`
    experiment_id = get_experiment_id(EXPERIMENT_URL)
    proc = subprocess.run(['nnictl', 'stop', experiment_id])
    assert proc.returncode == 0, '`nnictl stop %s` failed with code %d' % (experiment_id, proc.returncode)
89
    wait_for_port_available(8080, 10)
90
91
    assert not detect_port(8080), '`nnictl stop %s` failed to stop experiments' % experiment_id

92
93
94
    # test cmd `nnictl stop --port`
    proc = subprocess.run(['nnictl', 'stop', '--port', '8990'])
    assert proc.returncode == 0, '`nnictl stop %s` failed with code %d' % (experiment_id, proc.returncode)
95
    wait_for_port_available(8990, 10)
96
97
    assert not detect_port(8990), '`nnictl stop %s` failed to stop experiments' % experiment_id

98
99
100
    # test cmd `nnictl stop --all`
    proc = subprocess.run(['nnictl', 'stop', '--all'])
    assert proc.returncode == 0, '`nnictl stop --all` failed with code %d' % proc.returncode
101
102
    wait_for_port_available(8888, 10)
    wait_for_port_available(8989, 10)
103
    assert not detect_port(8888) and not detect_port(8989), '`nnictl stop --all` failed to stop experiments'
104
105


Zejun Lin's avatar
Zejun Lin committed
106
if __name__ == '__main__':
chicm-ms's avatar
chicm-ms committed
107
108
109
110
111
    parser = argparse.ArgumentParser()
    parser.add_argument("--config", type=str, required=True)
    parser.add_argument("--preinstall", action='store_true')
    args = parser.parse_args()
    setup_experiment(not args.preinstall)
Zejun Lin's avatar
Zejun Lin committed
112
    try:
chicm-ms's avatar
chicm-ms committed
113
114
        naive_test(args)
        stop_experiment_test(args)
Zejun Lin's avatar
Zejun Lin committed
115
116
117
118
119
120
121
        # TODO: check the output of rest server
        print(GREEN + 'PASS' + CLEAR)
    except Exception as error:
        print(RED + 'FAIL' + CLEAR)
        print('%r' % error)
        traceback.print_exc()
        sys.exit(1)