Add builtin tuner to CI (#247)

* update Makefile

* update Makefile

* add builtin-tuner test

* add builtin-tuner test

* refractor ci

* update azure.yml

* add built-in tuner test

* fix bugs

azure-pipelines.yml

@@ -12,7 +12,12 @@ steps:
       source install.sh
     displayName: 'Install dependencies'
   - script: |
-      cd test/naive
+      cd test
       export PATH=$HOME/.local/bin:$PATH
-      python3 run.py
-    displayName: 'Run tests'
+      python3 naive_test.py
+    displayName: 'Integration tests'
+  - script: |
+      cd test
+      export PATH=$HOME/.local/bin:$PATH
+      python3 sdk_tuner_test.py
+    displayName: 'Built-in tuner tests'

test/naive/run.py

-#!/usr/bin/env python3
-
-import contextlib
-import json
-import os
-import subprocess
-import requests
-import sys
-import time
-import traceback
-
-GREEN = '\33[32m'
-RED = '\33[31m'
-CLEAR = '\33[0m'
-
-class Integration_test():
-    def __init__(self):
-        self.experiment_url = 'http://localhost:8080/api/v1/nni/experiment'
-        self.experiment_id = None
-        self.experiment_done_signal = '"Experiment done"'
-
-    def read_last_line(self, file_name):
-        try:
-            *_, last_line = open(file_name)
-            return last_line.strip()
-        except (FileNotFoundError, ValueError):
-            return None
-
-    def fetch_experiment_config(self):
-        experiment_profile = requests.get(self.experiment_url)
-        self.experiment_id = json.loads(experiment_profile.text)['id']
-        self.experiment_path = os.path.join(os.environ['HOME'], 'nni/experiments', self.experiment_id)
-        self.nnimanager_log_path = os.path.join(self.experiment_path, 'log', 'nnimanager.log')
-
-    def check_experiment_status(self):
-        assert os.path.exists(self.nnimanager_log_path), 'Experiment starts failed'
-        cmds = ['cat', self.nnimanager_log_path, '|', 'grep', self.experiment_done_signal]
-        completed_process = subprocess.run(' '.join(cmds), shell = True)
-        
-        return completed_process.returncode == 0
-
-    def remove_files(self, file_list):
-        for file_path in file_list:
-            with contextlib.suppress(FileNotFoundError):
-                os.remove(file_path)
-
-    def run(self, installed = True):
-        if not installed:
-            os.environ['PATH'] = os.environ['PATH'] + ':' + os.environ['PWD']
-            sdk_path = os.path.abspath('../../src/sdk/pynni')
-            cmd_path = os.path.abspath('../../tools')
-            pypath = os.environ.get('PYTHONPATH')
-            if pypath:
-                pypath = ':'.join([pypath, sdk_path, cmd_path])
-            else:
-                pypath = ':'.join([sdk_path, cmd_path])
-            os.environ['PYTHONPATH'] = pypath
-
-        to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt']
-        self.remove_files(to_remove)
-
-        proc = subprocess.run(['nnictl', 'create', '--config', 'local.yml'])
-        assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
-
-        print('Spawning trials...')
-        time.sleep(1)
-        self.fetch_experiment_config()
-        current_trial = 0
-
-        for _ in range(100):
-            time.sleep(1)
-
-            tuner_status = self.read_last_line('tuner_result.txt')
-            assessor_status = self.read_last_line('assessor_result.txt')
-            experiment_status = self.check_experiment_status()
-
-            assert tuner_status != 'ERROR', 'Tuner exited with error'
-            assert assessor_status != 'ERROR', 'Assessor exited with error'
-
-            if experiment_status:
-                break
-
-            if tuner_status is not None:
-                for line in open('tuner_result.txt'):
-                    if line.strip() == 'ERROR':
-                        break
-                    trial = int(line.split(' ')[0])
-                    if trial > current_trial:
-                        current_trial = trial
-                        print('Trial #%d done' % trial)
-
-        assert experiment_status, 'Failed to finish in 100 sec'
-
-        ss1 = json.load(open('search_space.json'))
-        ss2 = json.load(open('tuner_search_space.json'))
-        assert ss1 == ss2, 'Tuner got wrong search space'
-
-        tuner_result = set(open('tuner_result.txt'))
-        expected = set(open('expected_tuner_result.txt'))
-        # Trials may complete before NNI gets assessor's result,
-        # so it is possible to have more final result than expected
-        assert tuner_result.issuperset(expected), 'Bad tuner result'
-
-        assessor_result = set(open('assessor_result.txt'))
-        expected = set(open('expected_assessor_result.txt'))
-        assert assessor_result == expected, 'Bad assessor result'
-
-if __name__ == '__main__':
-    installed = (sys.argv[-1] != '--preinstall')
-    ci = Integration_test()
-    try:
-        ci.run(installed)
-        # TODO: check the output of rest server
-        print(GREEN + 'PASS' + CLEAR)
-    except Exception as error:
-        print(RED + 'FAIL' + CLEAR)
-        print('%r' % error)
-        traceback.print_exc()
-        sys.exit(1)
-    finally:
-        subprocess.run(['nnictl', 'stop'])

test/naive_test.py

+#!/usr/bin/env python3
+
+import json
+import subprocess
+import sys
+import time
+import traceback
+
+from utils import check_experiment_status, fetch_experiment_config, read_last_line, remove_files, setup_experiment
+
+GREEN = '\33[32m'
+RED = '\33[31m'
+CLEAR = '\33[0m'
+
+EXPERIMENT_URL = 'http://localhost:8080/api/v1/nni/experiment'
+
+def run(installed = True):
+    
+    to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt']
+    to_remove = list(map(lambda file: 'naive_test/' + file, to_remove))
+    remove_files(to_remove)
+
+    proc = subprocess.run(['nnictl', 'create', '--config', 'naive_test/local.yml'])
+    assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
+
+    print('Spawning trials...')
+
+    nnimanager_log_path = fetch_experiment_config(EXPERIMENT_URL)
+    current_trial = 0
+
+    for _ in range(60):
+        time.sleep(1)
+
+        tuner_status = read_last_line('naive_test/tuner_result.txt')
+        assessor_status = read_last_line('naive_test/assessor_result.txt')
+        experiment_status = check_experiment_status(nnimanager_log_path)
+
+        assert tuner_status != 'ERROR', 'Tuner exited with error'
+        assert assessor_status != 'ERROR', 'Assessor exited with error'
+
+        if experiment_status:
+            break
+
+        if tuner_status is not None:
+            for line in open('naive_test/tuner_result.txt'):
+                if line.strip() == 'ERROR':
+                    break
+                trial = int(line.split(' ')[0])
+                if trial > current_trial:
+                    current_trial = trial
+                    print('Trial #%d done' % trial)
+
+    assert experiment_status, 'Failed to finish in 1 min'
+
+    ss1 = json.load(open('naive_test/search_space.json'))
+    ss2 = json.load(open('naive_test/tuner_search_space.json'))
+    assert ss1 == ss2, 'Tuner got wrong search space'
+
+    tuner_result = set(open('naive_test/tuner_result.txt'))
+    expected = set(open('naive_test/expected_tuner_result.txt'))
+    # Trials may complete before NNI gets assessor's result,
+    # so it is possible to have more final result than expected
+    assert tuner_result.issuperset(expected), 'Bad tuner result'
+
+    assessor_result = set(open('naive_test/assessor_result.txt'))
+    expected = set(open('naive_test/expected_assessor_result.txt'))
+    assert assessor_result == expected, 'Bad assessor result'
+
+if __name__ == '__main__':
+    installed = (sys.argv[-1] != '--preinstall')
+    setup_experiment(installed)
+    try:
+        run()
+        # TODO: check the output of rest server
+        print(GREEN + 'PASS' + CLEAR)
+    except Exception as error:
+        print(RED + 'FAIL' + CLEAR)
+        print('%r' % error)
+        traceback.print_exc()
+        sys.exit(1)
+    finally:
+        subprocess.run(['nnictl', 'stop'])

test/naive/README.md → test/naive_test/README.md

 ## Usage
 
 * To test before installing:
-`./run.py --preinstall`
+`python3 run.py --preinstall`
 * To test the integrity of installation:
-`./run.py`
+`python3 run.py`
 * It will print `PASS` in green eventually if everything works well.
 
 ## Details

test/sdk_tuner_test.py

+#!/usr/bin/env python3
+
+import subprocess
+import sys
+import time
+import traceback
+
+from utils import *
+
+GREEN = '\33[32m'
+RED = '\33[31m'
+CLEAR = '\33[0m'
+
+TUNER_LIST = ['TPE', 'Random', 'Anneal', 'Evolution']
+EXPERIMENT_URL = 'http://localhost:8080/api/v1/nni/experiment'
+
+
+def switch_tuner(tuner_name):
+    '''Change tuner in config.yml'''
+    config_path = 'sdk_tuner_test/local.yml'
+    experiment_config = get_yml_content(config_path)
+    experiment_config['tuner'] = {
+        'builtinTunerName': tuner_name,
+        'classArgs': {
+            'optimize_mode': 'maximize'
+        }
+    }
+    dump_yml_content(config_path, experiment_config)
+
+def test_builtin_tuner(tuner_name):
+    remove_files(['sdk_tuner_test/nni_tuner_result.txt'])
+    switch_tuner(tuner_name)
+
+    print('Testing %s...'%tuner_name)
+    proc = subprocess.run(['nnictl', 'create', '--config', 'sdk_tuner_test/local.yml'])
+    assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
+
+    nnimanager_log_path = fetch_experiment_config(EXPERIMENT_URL)
+    
+    for _ in range(10):
+        time.sleep(3)
+
+        # check if tuner exists with error
+        tuner_status = read_last_line('tuner_result.txt')
+        assert tuner_status != 'ERROR', 'Tuner exited with error'
+
+        # check if experiment is done
+        experiment_status = check_experiment_status(nnimanager_log_path)
+        if experiment_status:
+            break
+    
+    assert experiment_status, 'Failed to finish in 30 sec'
+
+def run():
+    to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt']
+    remove_files(to_remove)
+
+    for tuner_name in TUNER_LIST:
+        try:
+            test_builtin_tuner(tuner_name)
+            print(GREEN + 'Test ' +tuner_name+ ' tuner: TEST PASS' + CLEAR)
+        except Exception as error:
+            print(GREEN + 'Test ' +tuner_name+ ' tuner: TEST FAIL' + CLEAR)
+            print('%r' % error)
+            traceback.print_exc()
+            raise error
+        finally:
+            subprocess.run(['nnictl', 'stop'])
+
+
+if __name__ == '__main__':
+    installed = (sys.argv[-1] != '--preinstall')
+    setup_experiment(installed)
+
+    run()

test/sdk_tuner_test/local.yml

+authorName: nni
+experimentName: test_builtin_tuner
+maxExecDuration: 1h
+maxTrialNum: 2
+searchSpacePath: search_space.json
+trainingServicePlatform: local
+trial:
+  codeDir: .
+  command: python3 naive_trial.py
+  gpuNum: 0
+trialConcurrency: 2
+tuner:
+  builtinTunerName: Evolution
+  classArgs:
+    optimize_mode: maximize
+useAnnotation: false

test/sdk_tuner_test/naive_trial.py

+import nni
+
+params = nni.get_parameters()
+print('params:', params)
+x = params['x']
+
+nni.report_final_result(x)

test/sdk_tuner_test/search_space.json

+{
+    "x":
+    {
+        "_type" : "choice",
+        "_value" : [1, 100]
+    }
+}

test/utils.py

+import contextlib
+import json
+import os
+import subprocess
+import requests
+import traceback
+import yaml
+
+EXPERIMENT_DONE_SIGNAL = '"Experiment done"'
+
+def read_last_line(file_name):
+    try:
+        *_, last_line = open(file_name)
+        return last_line.strip()
+    except (FileNotFoundError, ValueError):
+        return None
+
+def remove_files(file_list):
+    for file_path in file_list:
+        with contextlib.suppress(FileNotFoundError):
+            os.remove(file_path)
+
+def get_yml_content(file_path):
+    '''Load yaml file content'''
+    with open(file_path, 'r') as file:
+        return yaml.load(file)
+
+def dump_yml_content(file_path, content):
+    '''Dump yaml file content'''
+    with open(file_path, 'w') as file:
+        file.write(yaml.dump(content, default_flow_style=False))
+
+def setup_experiment(installed = True):
+    if not installed:
+        os.environ['PATH'] = os.environ['PATH'] + ':' + os.environ['PWD']
+        sdk_path = os.path.abspath('../src/sdk/pynni')
+        cmd_path = os.path.abspath('../tools')
+        pypath = os.environ.get('PYTHONPATH')
+        if pypath:
+            pypath = ':'.join([pypath, sdk_path, cmd_path])
+        else:
+            pypath = ':'.join([sdk_path, cmd_path])
+        os.environ['PYTHONPATH'] = pypath
+
+def fetch_experiment_config(experiment_url):
+    experiment_profile = requests.get(experiment_url)
+    experiment_id = json.loads(experiment_profile.text)['id']
+    experiment_path = os.path.join(os.environ['HOME'], 'nni/experiments', experiment_id)
+    nnimanager_log_path = os.path.join(experiment_path, 'log', 'nnimanager.log')
+
+    return nnimanager_log_path
+
+def check_experiment_status(nnimanager_log_path):
+    assert os.path.exists(nnimanager_log_path), 'Experiment starts failed'
+    cmds = ['cat', nnimanager_log_path, '|', 'grep', EXPERIMENT_DONE_SIGNAL]
+    completed_process = subprocess.run(' '.join(cmds), shell = True)
+    
+    return completed_process.returncode == 0
\ No newline at end of file