Refactor nnictl, show error message gracefully (#986)

Advisor and assessor could not be used simultaneously.
Refactor config file error message.
Support scientific notation, like 1e-2

setup.py

@@ -51,7 +51,7 @@ setup(
         'json_tricks',
         'numpy',
         'psutil',
-        'pyyaml',
+        'ruamel.yaml',
         'requests',
         'scipy',
         'schema',

test/utils.py

@@ -24,7 +24,7 @@ import json
 import os
 import subprocess
 import requests
-import yaml
+import ruamel.yaml as yaml
 
 EXPERIMENT_DONE_SIGNAL = '"Experiment done"'
 
@@ -55,7 +55,7 @@ def remove_files(file_list):
 def get_yml_content(file_path):
     '''Load yaml file content'''
     with open(file_path, 'r') as file:
-        return yaml.load(file)
+        return yaml.load(file, Loader=yaml.Loader)
 
 def dump_yml_content(file_path, content):
     '''Dump yaml file content'''

tools/nni_cmd/common_utils.py

@@ -19,7 +19,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 import json
-import yaml
+import ruamel.yaml as yaml
 import psutil
 import socket
 from .constants import ERROR_INFO, NORMAL_INFO, WARNING_INFO, COLOR_RED_FORMAT, COLOR_YELLOW_FORMAT
@@ -28,7 +28,7 @@ def get_yml_content(file_path):
     '''Load yaml file content'''
     try:
         with open(file_path, 'r') as file:
-            return yaml.load(file)
+            return yaml.load(file, Loader=yaml.Loader)
     except TypeError as err:
         print('Error: ', err)
         return None

tools/nni_cmd/constants.py

@@ -85,3 +85,9 @@ COLOR_RED_FORMAT = '\033[1;31;31m%s\033[0m'
 COLOR_GREEN_FORMAT = '\033[1;32;32m%s\033[0m'
 
 COLOR_YELLOW_FORMAT = '\033[1;33;33m%s\033[0m'
+
+SCHEMA_TYPE_ERROR = '%s should be %s type!'
+
+SCHEMA_RANGE_ERROR = '%s should be in range of %s!'
+
+SCHEMA_PATH_ERROR = '%s path not exist!'

tools/nni_cmd/launcher_utils.py

@@ -20,8 +20,11 @@
 
 import os
 import json
-from .config_schema import LOCAL_CONFIG_SCHEMA, REMOTE_CONFIG_SCHEMA, PAI_CONFIG_SCHEMA, KUBEFLOW_CONFIG_SCHEMA, FRAMEWORKCONTROLLER_CONFIG_SCHEMA
+from .config_schema import LOCAL_CONFIG_SCHEMA, REMOTE_CONFIG_SCHEMA, PAI_CONFIG_SCHEMA, KUBEFLOW_CONFIG_SCHEMA, FRAMEWORKCONTROLLER_CONFIG_SCHEMA, \
+tuner_schema_dict, advisor_schema_dict, assessor_schema_dict               
+from schema import SchemaMissingKeyError, SchemaForbiddenKeyError, SchemaUnexpectedTypeError, SchemaWrongKeyError, SchemaError
 from .common_utils import get_json_content, print_error, print_warning, print_normal
+from schema import Schema, And, Use, Optional, Regex, Or
 
 def expand_path(experiment_config, key):
     '''Change '~' to user home directory'''
@@ -134,21 +137,49 @@ def validate_common_content(experiment_config):
             'kubeflow': KUBEFLOW_CONFIG_SCHEMA,
             'frameworkcontroller': FRAMEWORKCONTROLLER_CONFIG_SCHEMA
         }
+    separate_schema_dict = {
+        'tuner': tuner_schema_dict,
+        'advisor': advisor_schema_dict,
+        'assessor': assessor_schema_dict
+    }
+    separate_builtInName_dict = {
+        'tuner': 'builtinTunerName',
+        'advisor': 'builtinAdvisorName',
+        'assessor': 'builtinAssessorName'
+    }
     try:
         schema_dict.get(experiment_config['trainingServicePlatform']).validate(experiment_config)
-        #set default value
-        if experiment_config.get('maxExecDuration') is None:
-            experiment_config['maxExecDuration'] = '999d'
-        if experiment_config.get('maxTrialNum') is None:
-            experiment_config['maxTrialNum'] = 99999
-        if experiment_config['trainingServicePlatform'] == 'remote':
-            for index in range(len(experiment_config['machineList'])):
-                if experiment_config['machineList'][index].get('port') is None:
-                    experiment_config['machineList'][index]['port'] = 22
-                
-    except Exception as exception:
-        print_error('Your config file is not correct, please check your config file content!\n%s' % exception)
+        for separate_key in separate_schema_dict.keys():
+            if experiment_config.get(separate_key):
+                if experiment_config[separate_key].get(separate_builtInName_dict[separate_key]):
+                    validate = False
+                    for key in separate_schema_dict[separate_key].keys():
+                        if key.__contains__(experiment_config[separate_key][separate_builtInName_dict[separate_key]]):
+                            Schema({**separate_schema_dict[separate_key][key]}).validate(experiment_config[separate_key])
+                            validate = True
+                            break
+                    if not validate:
+                        print_error('%s %s error!' % (separate_key, separate_builtInName_dict[separate_key]))
+                        exit(1)
+                else:
+                    Schema({**separate_schema_dict[separate_key]['customized']}).validate(experiment_config[separate_key])
+    except SchemaError as error:
+        print_error('Your config file is not correct, please check your config file content!')
+        if error.__str__().__contains__('Wrong key'):
+            print_error(' '.join(error.__str__().split()[:3]))
+        else:
+            print_error(error)
         exit(1)
+    
+    #set default value
+    if experiment_config.get('maxExecDuration') is None:
+        experiment_config['maxExecDuration'] = '999d'
+    if experiment_config.get('maxTrialNum') is None:
+        experiment_config['maxTrialNum'] = 99999
+    if experiment_config['trainingServicePlatform'] == 'remote':
+        for index in range(len(experiment_config['machineList'])):
+            if experiment_config['machineList'][index].get('port') is None:
+                experiment_config['machineList'][index]['port'] = 22
 
 def validate_customized_file(experiment_config, spec_key):
     '''
@@ -230,6 +261,9 @@ def validate_all_content(experiment_config, config_path):
     validate_pai_trial_conifg(experiment_config)
     experiment_config['maxExecDuration'] = parse_time(experiment_config['maxExecDuration'])
     if experiment_config.get('advisor'):
+        if experiment_config.get('assessor') or experiment_config.get('tuner'):
+            print_error('advisor could not be set with assessor or tuner simultaneously!')
+            exit(1)
         parse_advisor_content(experiment_config)
         validate_annotation_content(experiment_config, 'advisor', 'builtinAdvisorName')
     else:

tools/setup.py

@@ -8,7 +8,7 @@ setuptools.setup(
     python_requires = '>=3.5',
     install_requires = [
         'requests',
-        'pyyaml',
+        'ruamel.yaml',
         'psutil',
         'astor',
         'schema',