Dev hyperband (#405)

* support hyperband

* add example for hyperband

* register Hyperband in tuner

* after debug

* update doc

* trivial change

* update spec validation of yaml config

* modify nnictl launcher

* modify nnimanager and util to support advisor

* Quick fix nnictl config logic (#289)

* fix nnictl bug

* fix install.sh

* add desc for Dockerfile.build.base

* update document for Dockerfile

* update

* refactor port detect

* update

* refactor NNICTLDOC.md

* add document for pai and nnictl

* add default value for port

* add exception handling in trial_keeper.py

* fix port bug

* fix resume

* fix nnictl resume and fix nnictl stop

* fix document

* update

* refactor nnictl

* update

* update doc

* update

* update nnictl

* fix comment

* revert dockerfile

* update

* update

* update

* fix nnictl error hit

* fix comments

* fix bash-completion

* fix paramiko install

* quick fix resume logic

...

tools/nni_cmd/config_schema.py

@@ -34,7 +34,22 @@ Optional('multiPhase'): bool,
 Optional('multiThread'): bool,
 Optional('nniManagerIp'): str,
 'useAnnotation': bool,
-'tuner': Or({
+Optional('advisor'): Or({
+    'builtinAdvisorName': Or('Hyperband'),
+    'classArgs': {
+        'optimize_mode': Or('maximize', 'minimize'),
+        Optional('R'): int,
+        Optional('eta'): int
+    },
+    Optional('gpuNum'): And(int, lambda x: 0 <= x <= 99999),
+},{
+    'codeDir': os.path.exists,
+    'classFileName': str,
+    'className': str,
+    Optional('classArgs'): dict,
+    Optional('gpuNum'): And(int, lambda x: 0 <= x <= 99999),
+}),
+Optional('tuner'): Or({
     'builtinTunerName': Or('TPE', 'Random', 'Anneal', 'SMAC', 'Evolution'),
     Optional('classArgs'): {
         'optimize_mode': Or('maximize', 'minimize')

tools/nni_cmd/launcher.py

@@ -203,9 +203,12 @@ def set_experiment(experiment_config, mode, port, config_file_name):
         request_data['description'] = experiment_config['description']
     if experiment_config.get('multiPhase'):
         request_data['multiPhase'] = experiment_config.get('multiPhase')
-    request_data['tuner'] = experiment_config['tuner']
-    if 'assessor' in experiment_config:
-        request_data['assessor'] = experiment_config['assessor']
+    if experiment_config.get('advisor'):
+        request_data['advisor'] = experiment_config['advisor']
+    else:
+        request_data['tuner'] = experiment_config['tuner']
+        if 'assessor' in experiment_config:
+            request_data['assessor'] = experiment_config['assessor']
 
     request_data['clusterMetaData'] = []
     if experiment_config['trainingServicePlatform'] == 'local':

tools/nni_cmd/launcher_utils.py

@@ -55,6 +55,8 @@ def parse_path(experiment_config, config_path):
         expand_path(experiment_config['tuner'], 'codeDir')
     if experiment_config.get('assessor'):
         expand_path(experiment_config['assessor'], 'codeDir')
+    if experiment_config.get('advisor'):
+        expand_path(experiment_config['advisor'], 'codeDir')
     
     #if users use relative path, convert it to absolute path
     root_path = os.path.dirname(config_path)
@@ -66,6 +68,8 @@ def parse_path(experiment_config, config_path):
         parse_relative_path(root_path, experiment_config['tuner'], 'codeDir')
     if experiment_config.get('assessor'):
         parse_relative_path(root_path, experiment_config['assessor'], 'codeDir')
+    if experiment_config.get('advisor'):
+        parse_relative_path(root_path, experiment_config['advisor'], 'codeDir')
 
 def validate_search_space_content(experiment_config):
     '''Validate searchspace content, 
@@ -108,47 +112,58 @@ def validate_common_content(experiment_config):
         print_error('Your config file is not correct, please check your config file content!\n%s' % exception)
         exit(1)
 
+def validate_customized_file(experiment_config, spec_key):
+    '''
+    check whether the file of customized tuner/assessor/advisor exists
+    spec_key: 'tuner', 'assessor', 'advisor'
+    '''
+    if experiment_config[spec_key].get('codeDir') and \
+        experiment_config[spec_key].get('classFileName') and \
+        experiment_config[spec_key].get('className'):
+        if not os.path.exists(os.path.join(
+                experiment_config[spec_key]['codeDir'],
+                experiment_config[spec_key]['classFileName'])):
+            print_error('%s file directory is not valid!'%(spec_key))
+            exit(1)
+    else:
+        print_error('%s file directory is not valid!'%(spec_key))
+        exit(1)
+
 def parse_tuner_content(experiment_config):
     '''Validate whether tuner in experiment_config is valid'''
     if experiment_config['tuner'].get('builtinTunerName'):
         experiment_config['tuner']['className'] = experiment_config['tuner']['builtinTunerName']
-    elif experiment_config['tuner'].get('codeDir') and \
-        experiment_config['tuner'].get('classFileName') and \
-        experiment_config['tuner'].get('className'):
-        if not os.path.exists(os.path.join(
-                experiment_config['tuner']['codeDir'],
-                experiment_config['tuner']['classFileName'])):
-            print_error('Tuner file directory is not valid!')
-            exit(1)
     else:
-        raise ValueError('Tuner format is not valid!')
+        validate_customized_file(experiment_config, 'tuner')
 
 def parse_assessor_content(experiment_config):
     '''Validate whether assessor in experiment_config is valid'''
     if experiment_config.get('assessor'):
         if experiment_config['assessor'].get('builtinAssessorName'):
             experiment_config['assessor']['className'] = experiment_config['assessor']['builtinAssessorName']
-        elif experiment_config['assessor'].get('codeDir') and \
-            experiment_config['assessor'].get('classFileName') and \
-            experiment_config['assessor'].get('className'):
-            if not os.path.exists(os.path.join(
-                    experiment_config['assessor']['codeDir'],
-                    experiment_config['assessor']['classFileName'])):
-                print_error('Assessor file directory is not valid!')
-                exit(1)
         else:
-            print_error('Assessor format is not valid!')
-            exit(1)
+            validate_customized_file(experiment_config, 'assessor')
 
-def validate_annotation_content(experiment_config):
-    '''Valid whether useAnnotation and searchSpacePath is coexist'''
+def parse_advisor_content(experiment_config):
+    '''Validate whether advisor in experiment_config is valid'''
+    if experiment_config['advisor'].get('builtinAdvisorName'):
+        experiment_config['advisor']['className'] = experiment_config['advisor']['builtinAdvisorName']
+    else:
+        validate_customized_file(experiment_config, 'advisor')
+
+def validate_annotation_content(experiment_config, spec_key, builtin_name):
+    '''
+    Valid whether useAnnotation and searchSpacePath is coexist
+    spec_key: 'advisor' or 'tuner'
+    builtin_name: 'builtinAdvisorName' or 'builtinTunerName'
+    '''
     if experiment_config.get('useAnnotation'):
         if experiment_config.get('searchSpacePath'):
             print_error('If you set useAnnotation=true, please leave searchSpacePath empty')
             exit(1)
     else:
         # validate searchSpaceFile
-        if experiment_config['tuner'].get('tunerName') and experiment_config['tuner'].get('optimizationMode'):
+        if experiment_config[spec_key].get(builtin_name):
             if experiment_config.get('searchSpacePath') is None:
                 print_error('Please set searchSpace!')
                 exit(1)
@@ -165,6 +180,12 @@ def validate_all_content(experiment_config, config_path):
     parse_path(experiment_config, config_path)
     validate_common_content(experiment_config)
     parse_time(experiment_config)
-    parse_tuner_content(experiment_config)
-    parse_assessor_content(experiment_config)
-    validate_annotation_content(experiment_config)
+    if experiment_config.get('advisor'):
+        parse_advisor_content(experiment_config)
+        validate_annotation_content(experiment_config, 'advisor', 'builtinAdvisorName')
+    else:
+        if not experiment_config.get('tuner'):
+            raise Exception('Please provide tuner spec!')
+        parse_tuner_content(experiment_config)
+        parse_assessor_content(experiment_config)
+        validate_annotation_content(experiment_config, 'tuner', 'builtinTunerName')