Skip to content

GitLab

"git@developer.sourcefind.cn:gaoqiong/migraphx.git" did not exist on "274c772b385003c4bef3bbe90b29de3dd2287dd7"
Unverified Commit b0f34da1 authored by cruiseliu's avatar cruiseliu Committed by GitHub
Browse files

Refactor Hyperopt Tuners (Stage 2) - util update and test (#4238)

parent 30643638
Show whitespace changes
Inline Side-by-side
Showing with 286 additions and 195 deletions
nni/algorithms/hpo/random_tuner.py
View file @ b0f34da1
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
"""
Naive random tuner for hyper-parameter optimization.
You can specify an integer seed to determine random result.
"""
__all__ = ['RandomTuner', 'suggest', 'suggest_parameter']
import numpy as np
import schema
......@@ -29,25 +37,15 @@ class RandomClassArgsValidator(ClassArgsValidator):
def suggest(rng, space):
params = {}
for spec in space.values():
if not spec.is_activated(params):
continue
for key, spec in space.items():
if spec.is_activated_in(params):
params[key] = suggest_parameter(rng, spec)
return params
def suggest_parameter(rng, spec):
if spec.categorical:
params[spec.key] = rng.integers(spec.size)
continue
return rng.integers(spec.size)
if spec.normal_distributed:
if spec.log_distributed:
x = rng.lognormal(spec.mu, spec.sigma)
else:
x = rng.normal(spec.mu, spec.sigma)
return rng.normal(spec.mu, spec.sigma)
else:
if spec.log_distributed:
x = np.exp(rng.uniform(np.log(spec.low), np.log(spec.high)))
else:
x = rng.uniform(spec.low, spec.high)
if spec.q is not None:
x = np.round(x / spec.q) * spec.q
params[spec.key] = x
return params
return rng.uniform(spec.low, spec.high)
nni/common/hpo_utils/__init__.py
View file @ b0f34da1
......@@ -3,3 +3,4 @@
from .validation import validate_search_space
from .formatting import *
from .optimize_mode import OptimizeMode
nni/common/hpo_utils/formatting.py
View file @ b0f34da1
......@@ -6,6 +6,9 @@ This script provides a more program-friendly representation of HPO search space.
The format is considered internal helper and is not visible to end users.
You will find this useful when you want to support nested search space.
The random tuner is an intuitive example for this utility.
You should check its code before reading docstrings in this file.
"""
__all__ = [
......@@ -15,11 +18,14 @@ __all__ = [
]
import math
from types import SimpleNamespace
from typing import Any, List, NamedTuple, Optional, Tuple
class ParameterSpec(NamedTuple):
"""
Specification (aka space / range) of one single parameter.
Specification (aka space / range / domain) of one single parameter.
NOTE: For `loguniform` (and `qloguniform`), the fields `low` and `high` are logarithm of original values.
"""
name: str # The object key in JSON
......@@ -27,126 +33,137 @@ class ParameterSpec(NamedTuple):
values: List[Any] # "_value" in JSON
key: Tuple[str] # The "path" of this parameter
parent_index: Optional[int] # If the parameter is in a nested choice, this is its parent's index;
# if the parameter is at top level, this is `None`.
categorical: bool # Whether this paramter is categorical (unordered) or numerical (ordered)
size: int = None # If it's categorical, how many canidiates it has
size: int = None # If it's categorical, how many candidates it has
# uniform distributed
low: float = None # Lower bound of uniform parameter
high: float = None # Upper bound of uniform parameter
normal_distributed: bool = None # Whether this parameter is uniform or normal distrubuted
mu: float = None # Mean of normal parameter
sigma: float = None # Scale of normal parameter
mu: float = None # µ of normal parameter
sigma: float = None # σ of normal parameter
q: Optional[float] = None # If not `None`, the parameter value should be an integer multiple of this
clip: Optional[Tuple[float, float]] = None
# For q(log)uniform, this equals to "values[:2]"; for others this is None
q: Optional[float] = None # If not `None`, the value should be an integer multiple of this
log_distributed: bool = None # Whether this parameter is log distributed
# When true, low/high/mu/sigma describes log of parameter value (like np.lognormal)
def is_activated(self, partial_parameters):
def is_activated_in(self, partial_parameters):
"""
For nested search space, check whether this parameter should be skipped for current set of paremters.
This function works because the return value of `format_search_space()` is sorted in a way that
parents always appear before children.
This function must be used in a pattern similar to random tuner. Otherwise it will misbehave.
"""
return self.parent_index is None or partial_parameters.get(self.key[:-1]) == self.parent_index
if len(self.key) < 2 or isinstance(self.key[-2], str):
return True
return partial_parameters[self.key[:-2]] == self.key[-2]
def format_search_space(search_space, ordered_randint=False):
formatted = _format_search_space(tuple(), None, search_space)
if ordered_randint:
for i, spec in enumerate(formatted):
if spec.type == 'randint':
formatted[i] = _format_ordered_randint(spec.key, spec.parent_index, spec.values)
def format_search_space(search_space):
"""
Convert user provided search space into a dict of ParameterSpec.
The dict key is dict value's `ParameterSpec.key`.
"""
formatted = _format_search_space(tuple(), search_space)
# In CPython 3.6, dicts preserve order by internal implementation.
# In Python 3.7+, dicts preserve order by language spec.
# Python 3.6 is crappy enough. Don't bother to do extra work for it.
# Remove these comments when we drop 3.6 support.
return {spec.key: spec for spec in formatted}
def deformat_parameters(parameters, formatted_search_space):
"""
`paramters` is a dict whose key is `ParamterSpec.key`, and value is integer index if the parameter is categorical.
Convert it to the format expected by end users.
Convert internal format parameters to users' expected format.
"test/ut/sdk/test_hpo_formatting.py" provides examples of how this works.
The function do following jobs:
1. For "choice" and "randint", convert index (integer) to corresponding value.
2. For "*log*", convert x to `exp(x)`.
3. For "q*", convert x to `round(x / q) * q`, then clip into range.
4. For nested choices, convert flatten key-value pairs into nested structure.
"""
ret = {}
for key, x in parameters.items():
spec = formatted_search_space[key]
if not spec.categorical:
_assign(ret, key, x)
elif spec.type == 'randint':
if spec.categorical:
if spec.type == 'randint':
lower = min(math.ceil(float(x)) for x in spec.values)
_assign(ret, key, lower + x)
elif _is_nested_choices(spec.values):
_assign(ret, tuple([*key, '_name']), spec.values[x]['_name'])
else:
_assign(ret, key, spec.values[x])
else:
if spec.log_distributed:
x = math.exp(x)
if spec.q is not None:
x = round(x / spec.q) * spec.q
if spec.clip:
x = max(x, spec.clip[0])
x = min(x, spec.clip[1])
_assign(ret, key, x)
return ret
def _format_search_space(parent_key, parent_index, space):
def _format_search_space(parent_key, space):
formatted = []
for name, spec in space.items():
if name == '_name':
continue
key = tuple([*parent_key, name])
formatted.append(_format_parameter(key, parent_index, spec['_type'], spec['_value']))
formatted.append(_format_parameter(key, spec['_type'], spec['_value']))
if spec['_type'] == 'choice' and _is_nested_choices(spec['_value']):
for index, sub_space in enumerate(spec['_value']):
formatted += _format_search_space(key, index, sub_space)
key = tuple([*parent_key, name, index])
formatted += _format_search_space(key, sub_space)
return formatted
def _format_parameter(key, parent_index, type_, values):
spec = {}
spec['name'] = key[-1]
spec['type'] = type_
spec['values'] = values
spec['key'] = key
spec['parent_index'] = parent_index
def _format_parameter(key, type_, values):
spec = SimpleNamespace(
name = key[-1],
type = type_,
values = values,
key = key,
categorical = type_ in ['choice', 'randint'],
)
if type_ in ['choice', 'randint']:
spec['categorical'] = True
if spec.categorical:
if type_ == 'choice':
spec['size'] = len(values)
spec.size = len(values)
else:
lower, upper = sorted(math.ceil(float(x)) for x in values)
spec['size'] = upper - lower
lower = math.ceil(float(values[0]))
upper = math.ceil(float(values[1]))
spec.size = upper - lower
else:
spec['categorical'] = False
if type_.startswith('q'):
spec['q'] = float(values[2])
spec['log_distributed'] = ('log' in type_)
spec.q = float(values[2])
else:
spec.q = None
spec.log_distributed = ('log' in type_)
if 'normal' in type_:
spec['normal_distributed'] = True
spec['mu'] = float(values[0])
spec['sigma'] = float(values[1])
spec.normal_distributed = True
spec.mu = float(values[0])
spec.sigma = float(values[1])
else:
spec['normal_distributed'] = False
spec['low'], spec['high'] = sorted(float(x) for x in values[:2])
if 'q' in spec:
spec['low'] = math.ceil(spec['low'] / spec['q']) * spec['q']
spec['high'] = math.floor(spec['high'] / spec['q']) * spec['q']
return ParameterSpec(**spec)
def _format_ordered_randint(key, parent_index, values):
lower, upper = sorted(math.ceil(float(x)) for x in values)
return ParameterSpec(
name = key[-1],
type = 'randint',
values = values,
key = key,
parent_index = parent_index,
categorical = False,
low = float(lower),
high = float(upper - 1),
normal_distributed = False,
q = 1.0,
log_distributed = False,
)
spec.normal_distributed = False
spec.low = float(values[0])
spec.high = float(values[1])
if spec.q is not None:
spec.clip = (spec.low, spec.high)
if spec.log_distributed:
# make it align with mu
spec.low = math.log(spec.low)
spec.high = math.log(spec.high)
return ParameterSpec(**spec.__dict__)
def _is_nested_choices(values):
if not values:
return False
assert values # choices should not be empty
for value in values:
if not isinstance(value, dict):
return False
......@@ -157,6 +174,8 @@ def _is_nested_choices(values):
def _assign(params, key, x):
if len(key) == 1:
params[key[0]] = x
elif isinstance(key[0], int):
_assign(params, key[1:], x)
else:
if key[0] not in params:
params[key[0]] = {}
......
nni/common/hpo_utils/optimize_mode.py 0 → 100644
View file @ b0f34da1
from enum import Enum
class OptimizeMode(Enum):
Minimize = 'minimize'
Maximize = 'maximize'
test/ut/sdk/test_hpo_format_space.py deleted 100644 → 0
View file @ 30643638
from nni.common.hpo_utils import format_search_space, deformat_parameters
user_space = {
'dropout_rate': { '_type': 'uniform', '_value': [0.5, 0.9] },
'conv_size': { '_type': 'choice', '_value': [2, 3, 5, 7] },
'hidden_size': { '_type': 'qloguniform', '_value': [128, 1024, 1] },
'batch_size': { '_type': 'randint', '_value': [16, 32] },
'learning_rate': { '_type': 'loguniform', '_value': [0.0001, 0.1] },
'nested': {
'_type': 'choice',
'_value': [
{
'_name': 'empty',
},
{
'_name': 'double_nested',
'xy': {
'_type': 'choice',
'_value': [
{
'_name': 'x',
'x': { '_type': 'normal', '_value': [0, 1.0] },
},
{
'_name': 'y',
'y': { '_type': 'qnormal', '_value': [0, 1, 0.5] },
},
],
},
'z': { '_type': 'quniform', '_value': [-0.5, 0.5, 0.1] },
},
{
'_name': 'common',
'x': { '_type': 'lognormal', '_value': [1, 0.1] },
'y': { '_type': 'qlognormal', '_value': [-1, 1, 0.1] },
},
],
},
}
internal_space_simple = [ # the full internal space is too long, omit None and False values here
{'name':'dropout_rate', 'type':'uniform', 'values':[0.5,0.9], 'key':('dropout_rate',), 'low':0.5, 'high':0.9},
{'name':'conv_size', 'type':'choice', 'values':[2,3,5,7], 'key':('conv_size',), 'categorical':True, 'size':4},
{'name':'hidden_size', 'type':'qloguniform', 'values':[128,1024,1], 'key':('hidden_size',), 'low':128.0, 'high':1024.0, 'q':1.0, 'log_distributed':True},
{'name':'batch_size', 'type':'randint', 'values':[16,32], 'key':('batch_size',), 'categorical':True, 'size':16},
{'name':'learning_rate', 'type':'loguniform', 'values':[0.0001,0.1], 'key':('learning_rate',), 'low':0.0001, 'high':0.1, 'log_distributed':True},
{'name':'nested', 'type':'choice', '_value_names':['empty','double_nested','common'], 'key':('nested',), 'categorical':True, 'size':3, 'nested_choice':True},
{'name':'xy', 'type':'choice', '_value_names':['x','y'], 'key':('nested','xy'), 'parent_index':1, 'categorical':True, 'size':2, 'nested_choice':True},
{'name':'x', 'type':'normal', 'values':[0,1.0], 'key':('nested','xy','x'), 'parent_index':0, 'normal_distributed':True, 'mu':0.0, 'sigma':1.0},
{'name':'y', 'type':'qnormal', 'values':[0,1,0.5], 'key':('nested','xy','y'), 'parent_index':1, 'normal_distributed':True, 'mu':0.0, 'sigma':1.0, 'q':0.5},
{'name':'z', 'type':'quniform', 'values':[-0.5,0.5,0.1], 'key':('nested','z'), 'parent_index':1, 'low':-0.5, 'high':0.5, 'q':0.1},
{'name':'x', 'type':'lognormal', 'values':[1,0.1], 'key':('nested','x'), 'parent_index':2, 'normal_distributed':True, 'mu':1.0, 'sigma':0.1, 'log_distributed':True},
{'name':'y', 'type':'qlognormal', 'values':[-1,1,0.1], 'key':('nested','y'), 'parent_index':2, 'normal_distributed':True, 'mu':-1.0, 'sigma':1.0, 'q':0.1, 'log_distributed':True},
]
def test_format_search_space():
formatted = format_search_space(user_space)
for spec, expected in zip(formatted.values(), internal_space_simple):
for key, value in spec._asdict().items():
if key == 'values' and '_value_names' in expected:
assert [v['_name'] for v in value] == expected['_value_names']
elif key in expected:
assert value == expected[key]
else:
assert value is None or value == False
internal_parameters = {
('dropout_rate',): 0.7,
('conv_size',): 2,
('hidden_size',): 200.0,
('batch_size',): 3,
('learning_rate',): 0.0345,
('nested',): 1,
('nested', 'xy'): 0,
('nested', 'xy', 'x'): 0.123,
}
user_parameters = {
'dropout_rate': 0.7,
'conv_size': 5,
'hidden_size': 200.0,
'batch_size': 19,
'learning_rate': 0.0345,
'nested': {
'_name': 'double_nested',
'xy': {
'_name': 'x',
'x': 0.123,
},
},
}
def test_deformat_parameters():
space = format_search_space(user_space)
generated = deformat_parameters(internal_parameters, space)
assert generated == user_parameters
if __name__ == '__main__':
test_format_search_space()
test_deformat_parameters()
test/ut/sdk/test_hpo_formatting.py 0 → 100644
View file @ b0f34da1
from math import exp, log
from nni.common.hpo_utils import deformat_parameters, format_search_space
user_space = {
'pool': { '_type': 'choice', '_value': ['max', 'min', 'avg'] },
'kernel': { '_type': 'randint', '_value': [2, 8] },
'D': { # distribution
'_type': 'choice',
'_value': [
{
'_name': 'UNIFORM',
'dropout': { '_type': 'uniform', '_value': [0.5, 0.9] },
'hidden': { '_type': 'quniform', '_value': [100, 1000, 3] },
'U_lr': { '_type': 'loguniform', '_value': [0.0001, 0.1] },
'U_batch': { '_type': 'qloguniform', '_value': [16.0, 128.0, 0.725] },
},
{
'_name': 'NORMAL',
'dropout': { '_type': 'normal', '_value': [0.7, 0.2] },
'hidden': { '_type': 'qnormal', '_value': [500, 200, 3] },
'N_lr': { '_type': 'lognormal', '_value': [-6, 3] },
'N_batch': { '_type': 'qlognormal', '_value': [3.5, 1.2, 0.725] },
},
{
'_name': 'EMPTY',
},
]
},
'not_nested': {
'_type': 'choice',
'_value': [
{'x': 0, 'y': 0},
{'x': 1, 'y': 2},
],
},
}
spec_names = ['pool', 'kernel', 'D', 'dropout', 'hidden', 'U_lr', 'U_batch', 'dropout', 'hidden', 'N_lr', 'N_batch', 'not_nested']
spec_types = ['choice', 'randint', 'choice', 'uniform', 'quniform', 'loguniform', 'qloguniform', 'normal', 'qnormal', 'lognormal', 'qlognormal', 'choice']
spec_values = [['max','min','avg'], [2,8], user_space['D']['_value'], [0.5,0.9], [100.0,1000.0,3.0], [0.0001,0.1], [16.0,128.0,0.725], [0.7,0.2], [500.0,200.0,3.0], [-6.0,3.0], [3.5,1.2,0.725], [{'x':0,'y':0},{'x':1,'y':2}]]
spec_keys = [('pool',), ('kernel',), ('D',), ('D',0,'dropout'), ('D',0,'hidden'), ('D',0,'U_lr'), ('D',0,'U_batch'), ('D',1,'dropout'), ('D',1,'hidden'), ('D',1,'N_lr'), ('D',1,'N_batch'), ('not_nested',)]
spec_categoricals = [True, True, True, False, False, False, False, False, False, False, False, True]
spec_sizes = [3, 6, 3, None, None, None, None, None, None, None, None, 2]
spec_lows = [None, None, None, 0.5, 100.0, log(0.0001), log(16.0), None, None, None, None, None]
spec_highs = [None, None, None, 0.9, 1000.0, log(0.1), log(128.0), None, None, None, None, None]
spec_normals = [None, None, None, False, False, False, False, True, True, True, True, None]
spec_mus = [None, None, None, None, None, None, None, 0.7, 500.0, -6.0, 3.5, None]
spec_sigmas = [None, None, None, None, None, None, None, 0.2, 200.0, 3.0, 1.2, None]
spec_qs = [None, None, None, None, 3.0, None, 0.725, None, 3.0, None, 0.725, None]
spec_clips = [None, None, None, None, (100.0,1000.0), None, (16.0,128.0), None, None, None, None, None]
spec_logs = [None, None, None, False, False, True, True, False, False, True, True, None]
def test_formatting():
internal_space = format_search_space(user_space)
assert all(key == value.key for key, value in internal_space.items())
specs = list(internal_space.values())
assert spec_names == [spec.name for spec in specs]
assert spec_types == [spec.type for spec in specs]
assert spec_values == [spec.values for spec in specs]
assert spec_keys == [spec.key for spec in specs]
assert spec_categoricals == [spec.categorical for spec in specs]
assert spec_sizes == [spec.size for spec in specs]
assert spec_lows == [spec.low for spec in specs]
assert spec_highs == [spec.high for spec in specs]
assert spec_normals == [spec.normal_distributed for spec in specs]
assert spec_mus == [spec.mu for spec in specs]
assert spec_sigmas == [spec.sigma for spec in specs]
assert spec_qs == [spec.q for spec in specs]
assert spec_clips == [spec.clip for spec in specs]
assert spec_logs == [spec.log_distributed for spec in specs]
internal_params_1 = {
('pool',): 0,
('kernel',): 5,
('D',): 0,
('D',0,'dropout'): 0.7,
('D',0,'hidden'): 100.1, # round to 99.0, then clip to 100.0
('D',0,'U_lr'): -4.6,
('D',0,'U_batch'): 4.0,
('not_nested',): 0,
}
user_params_1 = {
'pool': 'max',
'kernel': 7,
'D': {
'_name': 'UNIFORM',
'dropout': 0.7,
'hidden': 100.0,
'U_lr': exp(-4.6),
'U_batch': 54.375,
},
'not_nested': {'x': 0, 'y': 0},
}
internal_params_2 = {
('pool',): 2,
('kernel',): 0,
('D',): 1,
('D',1,'dropout'): 0.7,
('D',1,'hidden'): 100.1,
('D',1,'N_lr'): -4.6,
('D',1,'N_batch'): 4.0,
('not_nested',): 1,
}
user_params_2 = {
'pool': 'avg',
'kernel': 2,
'D': {
'_name': 'NORMAL',
'dropout': 0.7,
'hidden': 99.0,
'N_lr': exp(-4.6),
'N_batch': 54.375,
},
'not_nested': {'x': 1, 'y': 2},
}
internal_params_3 = {
('pool',): 1,
('kernel',): 1,
('D',): 2,
('not_nested',): 1,
}
user_params_3 = {
'pool': 'min',
'kernel': 3,
'D': {
'_name': 'EMPTY',
},
'not_nested': {'x': 1, 'y': 2},
}
def test_deformatting():
internal_space = format_search_space(user_space)
assert deformat_parameters(internal_params_1, internal_space) == user_params_1
assert deformat_parameters(internal_params_2, internal_space) == user_params_2
assert deformat_parameters(internal_params_3, internal_space) == user_params_3
def test_activate():
internal_space = format_search_space(user_space)
assert internal_space[('pool',)].is_activated_in({})
partial = { ('pool',): 1, ('kernel',): 1, ('D',): 0 }
assert internal_space[('D', 0, 'dropout')].is_activated_in(partial)
assert internal_space[('D', 0, 'U_lr')].is_activated_in(partial)
assert not internal_space[('D', 1, 'dropout')].is_activated_in(partial)
assert not internal_space[('D', 1, 'N_lr')].is_activated_in(partial)
partial = { ('pool',): 1, ('kernel',): 1, ('D',): 2 }
assert not internal_space[('D', 0, 'dropout')].is_activated_in(partial)
assert not internal_space[('D', 0, 'U_lr')].is_activated_in(partial)
assert not internal_space[('D', 1, 'dropout')].is_activated_in(partial)
assert not internal_space[('D', 1, 'N_lr')].is_activated_in(partial)
assert internal_space[('not_nested',)].is_activated_in(partial)
if __name__ == '__main__':
test_formatting()
test_deformatting()
test_activate()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment