"ts/webui/src/git@developer.sourcefind.cn:OpenDAS/nni.git" did not exist on "be652aa4375b84549dd0a462aee23b80556fe532"
Unverified Commit 8e732f2c authored by QuanluZhang's avatar QuanluZhang Committed by GitHub
Browse files

fix bug in smac that configspace cannot deal with complex searchspace (#716)

fix bug in smac that configspace cannot deal with complex searchspace. That is converting categorical
parent 60223b2d
...@@ -41,6 +41,7 @@ def generate_pcs(nni_search_space_content): ...@@ -41,6 +41,7 @@ def generate_pcs(nni_search_space_content):
# parameter_name real [min_value, max_value] [default value] log # parameter_name real [min_value, max_value] [default value] log
# https://automl.github.io/SMAC3/stable/options.html # https://automl.github.io/SMAC3/stable/options.html
''' '''
categorical_dict = {}
search_space = nni_search_space_content search_space = nni_search_space_content
with open('param_config_space.pcs', 'w') as pcs_fd: with open('param_config_space.pcs', 'w') as pcs_fd:
if isinstance(search_space, dict): if isinstance(search_space, dict):
...@@ -48,10 +49,14 @@ def generate_pcs(nni_search_space_content): ...@@ -48,10 +49,14 @@ def generate_pcs(nni_search_space_content):
if isinstance(search_space[key], dict): if isinstance(search_space[key], dict):
try: try:
if search_space[key]['_type'] == 'choice': if search_space[key]['_type'] == 'choice':
choice_len = len(search_space[key]['_value'])
pcs_fd.write('%s categorical {%s} [%s]\n' % ( pcs_fd.write('%s categorical {%s} [%s]\n' % (
key, key,
json.dumps(search_space[key]['_value'])[1:-1], json.dumps(list(range(choice_len)))[1:-1],
json.dumps(search_space[key]['_value'][0]))) json.dumps(0)))
if key in categorical_dict:
raise RuntimeError('%s has already existed, please make sure search space has no duplicate key.' % key)
categorical_dict[key] = search_space[key]['_value']
elif search_space[key]['_type'] == 'randint': elif search_space[key]['_type'] == 'randint':
# TODO: support lower bound in randint # TODO: support lower bound in randint
pcs_fd.write('%s integer [0, %d] [%d]\n' % ( pcs_fd.write('%s integer [0, %d] [%d]\n' % (
...@@ -83,6 +88,8 @@ def generate_pcs(nni_search_space_content): ...@@ -83,6 +88,8 @@ def generate_pcs(nni_search_space_content):
raise RuntimeError('_type or _value error.') raise RuntimeError('_type or _value error.')
else: else:
raise RuntimeError('incorrect search space.') raise RuntimeError('incorrect search space.')
return categorical_dict
return None
def generate_scenario(ss_content): def generate_scenario(ss_content):
''' '''
...@@ -119,7 +126,7 @@ def generate_scenario(ss_content): ...@@ -119,7 +126,7 @@ def generate_scenario(ss_content):
sce_fd.write('paramfile = param_config_space.pcs\n') sce_fd.write('paramfile = param_config_space.pcs\n')
sce_fd.write('run_obj = quality\n') sce_fd.write('run_obj = quality\n')
generate_pcs(ss_content) return generate_pcs(ss_content)
if __name__ == '__main__': if __name__ == '__main__':
generate_scenario('search_space.json') generate_scenario('search_space.json')
...@@ -58,6 +58,7 @@ class SMACTuner(Tuner): ...@@ -58,6 +58,7 @@ class SMACTuner(Tuner):
self.first_one = True self.first_one = True
self.update_ss_done = False self.update_ss_done = False
self.loguniform_key = set() self.loguniform_key = set()
self.categorical_dict = {}
def _main_cli(self): def _main_cli(self):
''' '''
...@@ -128,7 +129,9 @@ class SMACTuner(Tuner): ...@@ -128,7 +129,9 @@ class SMACTuner(Tuner):
NOTE: updating search space is not supported. NOTE: updating search space is not supported.
''' '''
if not self.update_ss_done: if not self.update_ss_done:
generate_scenario(search_space) self.categorical_dict = generate_scenario(search_space)
if self.categorical_dict is None:
raise RuntimeError('categorical dict is not correctly returned after parsing search space.')
self.optimizer = self._main_cli() self.optimizer = self._main_cli()
self.smbo_solver = self.optimizer.solver self.smbo_solver = self.optimizer.solver
self.loguniform_key = {key for key in search_space.keys() if search_space[key]['_type'] == 'loguniform'} self.loguniform_key = {key for key in search_space.keys() if search_space[key]['_type'] == 'loguniform'}
...@@ -152,13 +155,21 @@ class SMACTuner(Tuner): ...@@ -152,13 +155,21 @@ class SMACTuner(Tuner):
else: else:
self.smbo_solver.nni_smac_receive_runs(self.total_data[parameter_id], reward) self.smbo_solver.nni_smac_receive_runs(self.total_data[parameter_id], reward)
def convert_loguniform(self, challenger_dict): def convert_loguniform_categorical(self, challenger_dict):
''' '''
convert the values of type `loguniform` back to their initial range Convert the values of type `loguniform` back to their initial range
Also, we convert categorical:
categorical values in search space are changed to list of numbers before,
those original values will be changed back in this function
''' '''
for key, value in challenger_dict.items(): for key, value in challenger_dict.items():
# convert to loguniform
if key in self.loguniform_key: if key in self.loguniform_key:
challenger_dict[key] = np.exp(challenger_dict[key]) challenger_dict[key] = np.exp(challenger_dict[key])
# convert categorical back to original value
if key in self.categorical_dict:
idx = challenger_dict[key]
challenger_dict[key] = self.categorical_dict[key][idx]
return challenger_dict return challenger_dict
def generate_parameters(self, parameter_id): def generate_parameters(self, parameter_id):
...@@ -169,13 +180,13 @@ class SMACTuner(Tuner): ...@@ -169,13 +180,13 @@ class SMACTuner(Tuner):
init_challenger = self.smbo_solver.nni_smac_start() init_challenger = self.smbo_solver.nni_smac_start()
self.total_data[parameter_id] = init_challenger self.total_data[parameter_id] = init_challenger
json_tricks.dumps(init_challenger.get_dictionary()) json_tricks.dumps(init_challenger.get_dictionary())
return self.convert_loguniform(init_challenger.get_dictionary()) return self.convert_loguniform_categorical(init_challenger.get_dictionary())
else: else:
challengers = self.smbo_solver.nni_smac_request_challengers() challengers = self.smbo_solver.nni_smac_request_challengers()
for challenger in challengers: for challenger in challengers:
self.total_data[parameter_id] = challenger self.total_data[parameter_id] = challenger
json_tricks.dumps(challenger.get_dictionary()) json_tricks.dumps(challenger.get_dictionary())
return self.convert_loguniform(challenger.get_dictionary()) return self.convert_loguniform_categorical(challenger.get_dictionary())
def generate_multiple_parameters(self, parameter_id_list): def generate_multiple_parameters(self, parameter_id_list):
''' '''
...@@ -189,7 +200,7 @@ class SMACTuner(Tuner): ...@@ -189,7 +200,7 @@ class SMACTuner(Tuner):
init_challenger = self.smbo_solver.nni_smac_start() init_challenger = self.smbo_solver.nni_smac_start()
self.total_data[one_id] = init_challenger self.total_data[one_id] = init_challenger
json_tricks.dumps(init_challenger.get_dictionary()) json_tricks.dumps(init_challenger.get_dictionary())
params.append(self.convert_loguniform(init_challenger.get_dictionary())) params.append(self.convert_loguniform_categorical(init_challenger.get_dictionary()))
else: else:
challengers = self.smbo_solver.nni_smac_request_challengers() challengers = self.smbo_solver.nni_smac_request_challengers()
cnt = 0 cnt = 0
...@@ -199,6 +210,6 @@ class SMACTuner(Tuner): ...@@ -199,6 +210,6 @@ class SMACTuner(Tuner):
break break
self.total_data[parameter_id_list[cnt]] = challenger self.total_data[parameter_id_list[cnt]] = challenger
json_tricks.dumps(challenger.get_dictionary()) json_tricks.dumps(challenger.get_dictionary())
params.append(self.convert_loguniform(challenger.get_dictionary())) params.append(self.convert_loguniform_categorical(challenger.get_dictionary()))
cnt += 1 cnt += 1
return params return params
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment