Skip to content

GitLab

Unverified Commit 22316800 authored by Yuge Zhang's avatar Yuge Zhang Committed by GitHub
Browse files

SMAC tuner bug fixes (#1644)

parent 041c3f0e
Hide whitespace changes
Inline Side-by-side
Showing with 64 additions and 75 deletions
src/sdk/pynni/nni/smac_tuner/__init__.py
View file @ 22316800
from .smac_tuner import SMACTuner
\ No newline at end of file
src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py
View file @ 22316800
......@@ -18,18 +18,17 @@
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import os
import json
import numpy as np
def get_json_content(file_path):
"""Load json file content
Parameters
----------
file_path:
path to the file
Raises
------
TypeError
......@@ -42,10 +41,10 @@ def get_json_content(file_path):
print('Error: ', err)
return None
def generate_pcs(nni_search_space_content):
"""Generate the Parameter Configuration Space (PCS) which defines the
legal ranges of the parameters to be optimized and their default values.
Generally, the format is:
# parameter_name categorical {value_1, ..., value_N} [default value]
# parameter_name ordinal {value_1, ..., value_N} [default value]
......@@ -53,19 +52,15 @@ def generate_pcs(nni_search_space_content):
# parameter_name integer [min_value, max_value] [default value] log
# parameter_name real [min_value, max_value] [default value]
# parameter_name real [min_value, max_value] [default value] log
Reference: https://automl.github.io/SMAC3/stable/options.html
Parameters
----------
nni_search_space_content: search_space
The search space in this experiment in nni
Returns
-------
Parameter Configuration Space (PCS)
the legal ranges of the parameters to be optimized and their default values
Raises
------
RuntimeError
......@@ -73,41 +68,45 @@ def generate_pcs(nni_search_space_content):
"""
categorical_dict = {}
search_space = nni_search_space_content
def dump_categorical(fd, key, categories):
choice_len = len(categories)
if key in categorical_dict:
raise RuntimeError(
'%s has already existed, please make sure search space has no duplicate key.' % key)
categorical_dict[key] = search_space[key]['_value']
fd.write('%s categorical {%s} [0]\n' % (key, ','.join(map(str, range(choice_len)))))
with open('param_config_space.pcs', 'w') as pcs_fd:
if isinstance(search_space, dict):
for key in search_space.keys():
if isinstance(search_space[key], dict):
try:
if search_space[key]['_type'] == 'choice':
choice_len = len(search_space[key]['_value'])
pcs_fd.write('%s categorical {%s} [%s]\n' % (
key,
json.dumps(list(range(choice_len)))[1:-1],
json.dumps(0)))
if key in categorical_dict:
raise RuntimeError('%s has already existed, please make sure search space has no duplicate key.' % key)
categorical_dict[key] = search_space[key]['_value']
dump_categorical(pcs_fd, key, search_space[key]['_value'])
elif search_space[key]['_type'] == 'randint':
pcs_fd.write('%s integer [%d, %d] [%d]\n' % (
key,
search_space[key]['_value'][0],
search_space[key]['_value'][1] - 1,
search_space[key]['_value'][0]))
lower, upper = search_space[key]['_value']
if lower + 1 == upper:
dump_categorical(pcs_fd, key, [lower])
else:
pcs_fd.write('%s integer [%d, %d] [%d]\n' % (key, lower, upper - 1, lower))
elif search_space[key]['_type'] == 'uniform':
pcs_fd.write('%s real %s [%s]\n' % (
key,
json.dumps(search_space[key]['_value']),
json.dumps(search_space[key]['_value'][0])))
low, high = search_space[key]['_value']
if low == high:
dump_categorical(pcs_fd, key, [low])
else:
pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low))
elif search_space[key]['_type'] == 'loguniform':
# use np.round here to ensure that the rounded defaut value is in the range, which will be rounded in configure_space package
search_space[key]['_value'] = list(np.round(np.log(search_space[key]['_value']), 10))
pcs_fd.write('%s real %s [%s]\n' % (
key,
json.dumps(search_space[key]['_value']),
json.dumps(search_space[key]['_value'][0])))
# use np.round here to ensure that the rounded default value is in the range,
# which will be rounded in configure_space package
low, high = list(np.round(np.log(search_space[key]['_value']), 10))
if low == high:
dump_categorical(pcs_fd, key, [search_space[key]['_value'][0]])
else:
pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low))
elif search_space[key]['_type'] == 'quniform':
low, high, q = search_space[key]['_value'][0:3]
vals = np.clip(np.arange(np.round(low/q), np.round(high/q)+1) * q, low, high).tolist()
vals = np.clip(np.arange(np.round(low / q), np.round(high / q) + 1) * q, low, high).tolist()
pcs_fd.write('%s ordinal {%s} [%s]\n' % (
key,
json.dumps(vals)[1:-1],
......@@ -121,17 +120,15 @@ def generate_pcs(nni_search_space_content):
return categorical_dict
return None
def generate_scenario(ss_content):
"""Generate the scenario. The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and
can be constructed either by providing an actual scenario-object, or by specifing the options in a scenario file.
Reference: https://automl.github.io/SMAC3/stable/options.html
The format of the scenario file is one option per line:
OPTION1 = VALUE1
OPTION2 = VALUE2
...
Parameters
----------
abort_on_first_run_crash: bool
......@@ -194,7 +191,6 @@ def generate_scenario(ss_content):
wallclock_limit: int
Maximum amount of wallclock-time used for optimization. Default: inf.
Use default because this is controlled by nni
Returns
-------
Scenario:
......@@ -203,11 +199,12 @@ def generate_scenario(ss_content):
"""
with open('scenario.txt', 'w') as sce_fd:
sce_fd.write('deterministic = 0\n')
#sce_fd.write('output_dir = \n')
# sce_fd.write('output_dir = \n')
sce_fd.write('paramfile = param_config_space.pcs\n')
sce_fd.write('run_obj = quality\n')
return generate_pcs(ss_content)
if __name__ == '__main__':
generate_scenario('search_space.json')
src/sdk/pynni/nni/smac_tuner/smac_tuner.py
View file @ 22316800
......@@ -21,34 +21,30 @@
smac_tuner.py
"""
import sys
import logging
import numpy as np
import sys
import numpy as np
from ConfigSpaceNNI import Configuration
from nni.tuner import Tuner
from nni.utils import OptimizeMode, extract_scalar_reward
from smac.utils.io.cmd_reader import CMDReader
from smac.scenario.scenario import Scenario
from smac.facade.smac_facade import SMAC
from smac.facade.roar_facade import ROAR
from smac.facade.epils_facade import EPILS
from ConfigSpaceNNI import Configuration
from smac.facade.roar_facade import ROAR
from smac.facade.smac_facade import SMAC
from smac.scenario.scenario import Scenario
from smac.utils.io.cmd_reader import CMDReader
from .convert_ss_to_scenario import generate_scenario
from nni.tuner import Tuner
from nni.utils import OptimizeMode, extract_scalar_reward
class SMACTuner(Tuner):
"""
Parameters
----------
optimize_mode: str
optimize mode, 'maximize' or 'minimize'
optimize mode, 'maximize' or 'minimize', by default 'maximize'
"""
def __init__(self, optimize_mode):
def __init__(self, optimize_mode="maximize"):
"""Constructor"""
self.logger = logging.getLogger(
self.__module__ + "." + self.__class__.__name__)
......@@ -64,7 +60,6 @@ class SMACTuner(Tuner):
def _main_cli(self):
"""Main function of SMAC for CLI interface
Returns
-------
instance
......@@ -130,15 +125,17 @@ class SMACTuner(Tuner):
return optimizer
def update_search_space(self, search_space):
"""TODO: this is urgly, we put all the initialization work in this method, because initialization relies
on search space, also because update_search_space is called at the beginning.
"""
NOTE: updating search space is not supported.
Parameters
----------
search_space:
search_space: dict
search space
"""
# TODO: this is ugly, we put all the initialization work in this method, because initialization relies
# on search space, also because update_search_space is called at the beginning.
if not self.update_ss_done:
self.categorical_dict = generate_scenario(search_space)
if self.categorical_dict is None:
......@@ -152,7 +149,6 @@ class SMACTuner(Tuner):
def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
"""receive_trial_result
Parameters
----------
parameter_id: int
......@@ -161,7 +157,6 @@ class SMACTuner(Tuner):
parameters
value:
value
Raises
------
RuntimeError
......@@ -179,17 +174,16 @@ class SMACTuner(Tuner):
else:
self.smbo_solver.nni_smac_receive_runs(self.total_data[parameter_id], reward)
def convert_loguniform_categorical(self, challenger_dict):
"""Convert the values of type `loguniform` back to their initial range
Also, we convert categorical:
categorical values in search space are changed to list of numbers before,
those original values will be changed back in this function
def param_postprocess(self, challenger_dict):
"""
Postprocessing for a set of parameter includes:
1. Convert the values of type `loguniform` back to their initial range.
2. Convert categorical: categorical values in search space are changed to list of numbers before,
those original values will be changed back in this function.
Parameters
----------
challenger_dict: dict
challenger dict
Returns
-------
dict
......@@ -210,12 +204,10 @@ class SMACTuner(Tuner):
def generate_parameters(self, parameter_id, **kwargs):
"""generate one instance of hyperparameters
Parameters
----------
parameter_id: int
parameter id
Returns
-------
list
......@@ -224,21 +216,19 @@ class SMACTuner(Tuner):
if self.first_one:
init_challenger = self.smbo_solver.nni_smac_start()
self.total_data[parameter_id] = init_challenger
return self.convert_loguniform_categorical(init_challenger.get_dictionary())
return self.param_postprocess(init_challenger.get_dictionary())
else:
challengers = self.smbo_solver.nni_smac_request_challengers()
for challenger in challengers:
self.total_data[parameter_id] = challenger
return self.convert_loguniform_categorical(challenger.get_dictionary())
return self.param_postprocess(challenger.get_dictionary())
def generate_multiple_parameters(self, parameter_id_list, **kwargs):
"""generate mutiple instances of hyperparameters
Parameters
----------
parameter_id_list: list
list of parameter id
Returns
-------
list
......@@ -249,7 +239,7 @@ class SMACTuner(Tuner):
for one_id in parameter_id_list:
init_challenger = self.smbo_solver.nni_smac_start()
self.total_data[one_id] = init_challenger
params.append(self.convert_loguniform_categorical(init_challenger.get_dictionary()))
params.append(self.param_postprocess(init_challenger.get_dictionary()))
else:
challengers = self.smbo_solver.nni_smac_request_challengers()
cnt = 0
......@@ -258,16 +248,17 @@ class SMACTuner(Tuner):
if cnt >= len(parameter_id_list):
break
self.total_data[parameter_id_list[cnt]] = challenger
params.append(self.convert_loguniform_categorical(challenger.get_dictionary()))
params.append(self.param_postprocess(challenger.get_dictionary()))
cnt += 1
return params
def import_data(self, data):
"""Import additional data for tuning
"""
Import additional data for tuning
Parameters
----------
data:
a list of dictionarys, each of which has at least two keys, 'parameter' and 'value'
data: list of dict
Each of which has at least two keys, `parameter` and `value`.
"""
_completed_num = 0
for trial_info in data:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment