Update ci with new built-in tuner and assessor (#359)

* fix sdk's unittest and add medianstop, batchtuner to ci

* fix sdk's unittest and add medianstop, batchtuner to ci

* remove debug info

* update azure-pipelines

* remove useless code

* add some checks

* fix pylint

* update ci test

* update ci

azure-pipelines.yml

@@ -20,7 +20,7 @@ jobs:
     displayName: 'Integration tests'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 sdk_tuner_test.py
+      PATH=$HOME/.local/bin:$PATH python3 sdk_test.py
     displayName: 'Built-in tuner tests'
 
 - job: 'Install_through_source_code'
@@ -43,5 +43,5 @@ jobs:
     displayName: 'Integration tests'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 sdk_tuner_test.py
+      PATH=$HOME/.local/bin:$PATH python3 sdk_test.py
     displayName: 'Built-in tuner tests'
\ No newline at end of file

src/sdk/pynni/tests/test_assessor.py

@@ -22,6 +22,7 @@
 import nni.protocol
 from nni.protocol import CommandType, send, receive
 from nni.assessor import Assessor, AssessResult
+from nni.msg_dispatcher import MsgDispatcher
 
 from io import BytesIO
 import json
@@ -66,14 +67,14 @@ class AssessorTestCase(TestCase):
         send(CommandType.ReportMetricData, '{"trial_job_id":"B","type":"PERIODICAL","sequence":0,"value":2}')
         send(CommandType.ReportMetricData, '{"trial_job_id":"A","type":"PERIODICAL","sequence":1,"value":3}')
         send(CommandType.TrialEnd, '{"trial_job_id":"A","event":"SYS_CANCELED"}')
-        send(CommandType.ReportMetricData, '{"trial_job_id":"B","type":"FINAL","sequence":0,"value":1}')
         send(CommandType.TrialEnd, '{"trial_job_id":"B","event":"SUCCEEDED"}')
         send(CommandType.NewTrialJob, 'null')
         _restore_io()
 
         assessor = NaiveAssessor()
+        dispatcher = MsgDispatcher(None, assessor)
         try:
-            assessor.run()
+            dispatcher.run()
         except Exception as e:
             self.assertIs(type(e), AssertionError)
             self.assertEqual(e.args[0], 'Unsupported command: CommandType.NewTrialJob')

src/sdk/pynni/tests/test_tuner.py

@@ -22,6 +22,7 @@
 import nni.protocol
 from nni.protocol import CommandType, send, receive
 from nni.tuner import Tuner
+from nni.msg_dispatcher import MsgDispatcher
 
 from io import BytesIO
 import json
@@ -87,8 +88,9 @@ class TunerTestCase(TestCase):
         _restore_io()
 
         tuner = NaiveTuner()
+        dispatcher = MsgDispatcher(tuner)
         try:
-            tuner.run()
+            dispatcher.run()
         except Exception as e:
             self.assertIs(type(e), AssertionError)
             self.assertEqual(e.args[0], 'Unsupported command: CommandType.KillTrialJob')

test/naive_test.py

-#!/usr/bin/env python3
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 import json
 import subprocess
@@ -6,7 +24,7 @@ import sys
 import time
 import traceback
 
-from utils import check_experiment_status, fetch_experiment_config, read_last_line, remove_files, setup_experiment
+from utils import check_experiment_status, fetch_nni_log_path, read_last_line, remove_files, setup_experiment
 
 GREEN = '\33[32m'
 RED = '\33[31m'
@@ -14,8 +32,8 @@ CLEAR = '\33[0m'
 
 EXPERIMENT_URL = 'http://localhost:8080/api/v1/nni/experiment'
 
-def run(installed = True):
-    
+def run():
+    '''run naive integration test'''
     to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt']
     to_remove = list(map(lambda file: 'naive_test/' + file, to_remove))
     remove_files(to_remove)
@@ -25,7 +43,7 @@ def run(installed = True):
 
     print('Spawning trials...')
 
-    nnimanager_log_path = fetch_experiment_config(EXPERIMENT_URL)
+    nnimanager_log_path = fetch_nni_log_path(EXPERIMENT_URL)
     current_trial = 0
 
     for _ in range(60):

test/sdk_tuner_test.py → test/sdk_test.py

-#!/usr/bin/env python3
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 import subprocess
 import sys
 import time
 import traceback
 
-from utils import *
+from utils import get_yml_content, dump_yml_content, setup_experiment, fetch_nni_log_path, check_experiment_status
 
 GREEN = '\33[32m'
 RED = '\33[31m'
 CLEAR = '\33[0m'
 
-TUNER_LIST = ['TPE', 'Random', 'Anneal', 'Evolution']
+TUNER_LIST = ['BatchTuner', 'TPE', 'Random', 'Anneal', 'Evolution']
+ASSESSOR_LIST = ['Medianstop']
 EXPERIMENT_URL = 'http://localhost:8080/api/v1/nni/experiment'
 
 
-def switch_tuner(tuner_name):
-    '''Change tuner in config.yml'''
-    config_path = 'sdk_tuner_test/local.yml'
+def switch(dispatch_type, dispatch_name):
+    '''Change dispatch in config.yml'''
+    config_path = 'sdk_test/local.yml'
     experiment_config = get_yml_content(config_path)
-    experiment_config['tuner'] = {
-        'builtinTunerName': tuner_name,
+    experiment_config[dispatch_type.lower()] = {
+        'builtin' + dispatch_type + 'Name': dispatch_name,
         'classArgs': {
             'optimize_mode': 'maximize'
         }
     }
     dump_yml_content(config_path, experiment_config)
 
-def test_builtin_tuner(tuner_name):
-    remove_files(['sdk_tuner_test/nni_tuner_result.txt'])
-    switch_tuner(tuner_name)
+def test_builtin_dispatcher(dispatch_type, dispatch_name):
+    '''test a dispatcher whose type is dispatch_type and name is dispatch_name'''
+    switch(dispatch_type, dispatch_name)
 
-    print('Testing %s...'%tuner_name)
-    proc = subprocess.run(['nnictl', 'create', '--config', 'sdk_tuner_test/local.yml'])
+    print('Testing %s...' % dispatch_name)
+    proc = subprocess.run(['nnictl', 'create', '--config', 'sdk_test/local.yml'])
     assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
 
-    nnimanager_log_path = fetch_experiment_config(EXPERIMENT_URL)
+    nnimanager_log_path = fetch_nni_log_path(EXPERIMENT_URL)
 
     for _ in range(10):
         time.sleep(3)
-
-        # check if tuner exists with error
-        tuner_status = read_last_line('tuner_result.txt')
-        assert tuner_status != 'ERROR', 'Tuner exited with error'
-
         # check if experiment is done
         experiment_status = check_experiment_status(nnimanager_log_path)
         if experiment_status:
@@ -51,16 +65,16 @@ def test_builtin_tuner(tuner_name):
 
     assert experiment_status, 'Failed to finish in 30 sec'
 
-def run():
-    to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt']
-    remove_files(to_remove)
-
-    for tuner_name in TUNER_LIST:
+def run(dispatch_type):
+    '''test all dispatchers whose type is dispatch_type'''
+    assert dispatch_type in ['Tuner', 'Assessor'], 'Unsupported dispatcher type: %s' % (dispatch_type)
+    dipsatcher_list = TUNER_LIST if dispatch_type == 'Tuner' else ASSESSOR_LIST
+    for dispatcher_name in dipsatcher_list:
         try:
-            test_builtin_tuner(tuner_name)
-            print(GREEN + 'Test ' +tuner_name+ ' tuner: TEST PASS' + CLEAR)
+            test_builtin_dispatcher(dispatch_type, dispatcher_name)
+            print(GREEN + 'Test %s %s: TEST PASS' % (dispatcher_name, dispatch_type) + CLEAR)
         except Exception as error:
-            print(GREEN + 'Test ' +tuner_name+ ' tuner: TEST FAIL' + CLEAR)
+            print(RED + 'Test %s %s: TEST FAIL' % (dispatcher_name, dispatch_type) + CLEAR)
             print('%r' % error)
             traceback.print_exc()
             raise error
@@ -72,4 +86,5 @@ if __name__ == '__main__':
     installed = (sys.argv[-1] != '--preinstall')
     setup_experiment(installed)
 
-    run()
+    run('Tuner')
+    run('Assessor')

test/sdk_tuner_test/local.yml → test/sdk_test/local.yml

+assessor:
+  builtinAssessorName: Medianstop
+  classArgs:
+    optimize_mode: maximize
 authorName: nni
-experimentName: test_builtin_tuner
+experimentName: test_sdk
 maxExecDuration: 1h
 maxTrialNum: 2
 searchSpacePath: search_space.json

test/utils.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
 import contextlib
 import json
 import os
 import subprocess
 import requests
-import traceback
 import yaml
 
 EXPERIMENT_DONE_SIGNAL = '"Experiment done"'
 
 def read_last_line(file_name):
+    '''read last line of a file and return None if file not found'''
     try:
         *_, last_line = open(file_name)
         return last_line.strip()
@@ -16,6 +36,7 @@ def read_last_line(file_name):
         return None
 
 def remove_files(file_list):
+    '''remove a list of files'''
     for file_path in file_list:
         with contextlib.suppress(FileNotFoundError):
             os.remove(file_path)
@@ -30,7 +51,8 @@ def dump_yml_content(file_path, content):
     with open(file_path, 'w') as file:
         file.write(yaml.dump(content, default_flow_style=False))
 
-def setup_experiment(installed = True):
+def setup_experiment(installed=True):
+    '''setup the experiment if nni is not installed'''
     if not installed:
         os.environ['PATH'] = os.environ['PATH'] + ':' + os.environ['PWD']
         sdk_path = os.path.abspath('../src/sdk/pynni')
@@ -42,7 +64,8 @@ def setup_experiment(installed = True):
             pypath = ':'.join([sdk_path, cmd_path])
         os.environ['PYTHONPATH'] = pypath
 
-def fetch_experiment_config(experiment_url):
+def fetch_nni_log_path(experiment_url):
+    '''get nni's log path from nni's experiment url'''
     experiment_profile = requests.get(experiment_url)
     experiment_id = json.loads(experiment_profile.text)['id']
     experiment_path = os.path.join(os.environ['HOME'], 'nni/experiments', experiment_id)
@@ -51,8 +74,9 @@ def fetch_experiment_config(experiment_url):
     return nnimanager_log_path
 
 def check_experiment_status(nnimanager_log_path):
+    '''check if the experiment is done successfully'''
     assert os.path.exists(nnimanager_log_path), 'Experiment starts failed'
     cmds = ['cat', nnimanager_log_path, '|', 'grep', EXPERIMENT_DONE_SIGNAL]
-    completed_process = subprocess.run(' '.join(cmds), shell = True)
+    completed_process = subprocess.run(' '.join(cmds), shell=True)
 
     return completed_process.returncode == 0