Skip to content

GitLab

Unverified Commit 751445d3 authored by Guoxin's avatar Guoxin Committed by GitHub
Browse files

docstr/pylint of GP Tuner & CurveFitting Assessor & MedianStop Assessor (#1692) 

# docstr/pylint of GP Tuner & CurveFitting Assessor & MedianStop Assessor
parent e6df29cf
Show whitespace changes
Inline Side-by-side
Showing with 409 additions and 226 deletions
docs/en_US/sdk_reference.rst
View file @ 751445d3
...@@ -39,6 +39,9 @@ Tuner ...@@ -39,6 +39,9 @@ Tuner
.. autoclass:: nni.batch_tuner.batch_tuner.BatchTuner .. autoclass:: nni.batch_tuner.batch_tuner.BatchTuner
:members: :members:
.. autoclass:: nni.gp_tuner.gp_tuner.GPTuner
:members:
Assessor Assessor
------------------------ ------------------------
.. autoclass:: nni.assessor.Assessor .. autoclass:: nni.assessor.Assessor
......
src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py
View file @ 751445d3
...@@ -29,13 +29,13 @@ class CurvefittingAssessor(Assessor): ...@@ -29,13 +29,13 @@ class CurvefittingAssessor(Assessor):
Parameters Parameters
---------- ----------
epoch_num: int epoch_num : int
The total number of epoch The total number of epoch
optimize_mode: str optimize_mode : str
optimize mode, 'maximize' or 'minimize' optimize mode, 'maximize' or 'minimize'
start_step: int start_step : int
only after receiving start_step number of reported intermediate results only after receiving start_step number of reported intermediate results
threshold: float threshold : float
The threshold that we decide to early stop the worse performance curve. The threshold that we decide to early stop the worse performance curve.
""" """
def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95, gap=1): def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95, gap=1):
...@@ -70,9 +70,9 @@ class CurvefittingAssessor(Assessor): ...@@ -70,9 +70,9 @@ class CurvefittingAssessor(Assessor):
Parameters Parameters
---------- ----------
trial_job_id: int trial_job_id : int
trial job id trial job id
success: bool success : bool
True if succssfully finish the experiment, False otherwise True if succssfully finish the experiment, False otherwise
""" """
if success: if success:
...@@ -90,9 +90,9 @@ class CurvefittingAssessor(Assessor): ...@@ -90,9 +90,9 @@ class CurvefittingAssessor(Assessor):
Parameters Parameters
---------- ----------
trial_job_id: int trial_job_id : int
trial job id trial job id
trial_history: list trial_history : list
The history performance matrix of each trial The history performance matrix of each trial
Returns Returns
...@@ -105,7 +105,6 @@ class CurvefittingAssessor(Assessor): ...@@ -105,7 +105,6 @@ class CurvefittingAssessor(Assessor):
Exception Exception
unrecognize exception in curvefitting_assessor unrecognize exception in curvefitting_assessor
""" """
trial_job_id = trial_job_id
self.trial_history = trial_history self.trial_history = trial_history
if not self.set_best_performance: if not self.set_best_performance:
return AssessResult.Good return AssessResult.Good
......
src/sdk/pynni/nni/curvefitting_assessor/curvefunctions.py
View file @ 751445d3
...@@ -14,7 +14,9 @@ ...@@ -14,7 +14,9 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
A family of functions used by CurvefittingAssessor
"""
import numpy as np import numpy as np
all_models = {} all_models = {}
...@@ -29,10 +31,10 @@ def vap(x, a, b, c): ...@@ -29,10 +31,10 @@ def vap(x, a, b, c):
Parameters Parameters
---------- ----------
x: int x : int
a: float a : float
b: float b : float
c: float c : float
Returns Returns
------- -------
...@@ -50,10 +52,10 @@ def pow3(x, c, a, alpha): ...@@ -50,10 +52,10 @@ def pow3(x, c, a, alpha):
Parameters Parameters
---------- ----------
x: int x : int
c: float c : float
a: float a : float
alpha: float alpha : float
Returns Returns
------- -------
...@@ -71,9 +73,9 @@ def linear(x, a, b): ...@@ -71,9 +73,9 @@ def linear(x, a, b):
Parameters Parameters
---------- ----------
x: int x : int
a: float a : float
b: float b : float
Returns Returns
------- -------
...@@ -91,9 +93,9 @@ def logx_linear(x, a, b): ...@@ -91,9 +93,9 @@ def logx_linear(x, a, b):
Parameters Parameters
---------- ----------
x: int x : int
a: float a : float
b: float b : float
Returns Returns
------- -------
...@@ -112,10 +114,10 @@ def dr_hill_zero_background(x, theta, eta, kappa): ...@@ -112,10 +114,10 @@ def dr_hill_zero_background(x, theta, eta, kappa):
Parameters Parameters
---------- ----------
x: int x : int
theta: float theta : float
eta: float eta : float
kappa: float kappa : float
Returns Returns
------- -------
...@@ -133,10 +135,10 @@ def log_power(x, a, b, c): ...@@ -133,10 +135,10 @@ def log_power(x, a, b, c):
Parameters Parameters
---------- ----------
x: int x : int
a: float a : float
b: float b : float
c: float c : float
Returns Returns
------- -------
...@@ -154,11 +156,11 @@ def pow4(x, alpha, a, b, c): ...@@ -154,11 +156,11 @@ def pow4(x, alpha, a, b, c):
Parameters Parameters
---------- ----------
x: int x : int
alpha: float alpha : float
a: float a : float
b: float b : float
c: float c : float
Returns Returns
------- -------
...@@ -177,11 +179,11 @@ def mmf(x, alpha, beta, kappa, delta): ...@@ -177,11 +179,11 @@ def mmf(x, alpha, beta, kappa, delta):
Parameters Parameters
---------- ----------
x: int x : int
alpha: float alpha : float
beta: float beta : float
kappa: float kappa : float
delta: float delta : float
Returns Returns
------- -------
...@@ -199,11 +201,11 @@ def exp4(x, c, a, b, alpha): ...@@ -199,11 +201,11 @@ def exp4(x, c, a, b, alpha):
Parameters Parameters
---------- ----------
x: int x : int
c: float c : float
a: float a : float
b: float b : float
alpha: float alpha : float
Returns Returns
------- -------
...@@ -221,9 +223,9 @@ def ilog2(x, c, a): ...@@ -221,9 +223,9 @@ def ilog2(x, c, a):
Parameters Parameters
---------- ----------
x: int x : int
c: float c : float
a: float a : float
Returns Returns
------- -------
...@@ -242,11 +244,11 @@ def weibull(x, alpha, beta, kappa, delta): ...@@ -242,11 +244,11 @@ def weibull(x, alpha, beta, kappa, delta):
Parameters Parameters
---------- ----------
x: int x : int
alpha: float alpha : float
beta: float beta : float
kappa: float kappa : float
delta: float delta : float
Returns Returns
------- -------
...@@ -264,11 +266,11 @@ def janoschek(x, a, beta, k, delta): ...@@ -264,11 +266,11 @@ def janoschek(x, a, beta, k, delta):
Parameters Parameters
---------- ----------
x: int x : int
a: float a : float
beta: float beta : float
k: float k : float
delta: float delta : float
Returns Returns
------- -------
......
src/sdk/pynni/nni/curvefitting_assessor/model_factory.py
View file @ 751445d3
...@@ -40,7 +40,7 @@ class CurveModel: ...@@ -40,7 +40,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
target_pos: int target_pos : int
The point we need to predict The point we need to predict
""" """
def __init__(self, target_pos): def __init__(self, target_pos):
...@@ -120,14 +120,14 @@ class CurveModel: ...@@ -120,14 +120,14 @@ class CurveModel:
Parameters Parameters
---------- ----------
model: string model : string
name of the curve function model name of the curve function model
pos: int pos : int
the epoch number of the position you want to predict the epoch number of the position you want to predict
Returns Returns
------- -------
int: int
The expected matrix at pos The expected matrix at pos
""" """
if model_para_num[model] == 2: if model_para_num[model] == 2:
...@@ -143,9 +143,9 @@ class CurveModel: ...@@ -143,9 +143,9 @@ class CurveModel:
Parameters Parameters
---------- ----------
pos: int pos : int
the epoch number of the position you want to predict the epoch number of the position you want to predict
sample: list sample : list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns Returns
...@@ -165,7 +165,7 @@ class CurveModel: ...@@ -165,7 +165,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
samples: list samples : list
a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix,
representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}}
...@@ -187,7 +187,7 @@ class CurveModel: ...@@ -187,7 +187,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
sample: list sample : list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns Returns
...@@ -206,9 +206,9 @@ class CurveModel: ...@@ -206,9 +206,9 @@ class CurveModel:
Parameters Parameters
---------- ----------
pos: int pos : int
the epoch number of the position you want to predict the epoch number of the position you want to predict
sample: list sample : list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns Returns
...@@ -225,7 +225,7 @@ class CurveModel: ...@@ -225,7 +225,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
sample: list sample : list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns Returns
...@@ -244,7 +244,7 @@ class CurveModel: ...@@ -244,7 +244,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
samples: list samples : list
a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix,
representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}}
...@@ -267,7 +267,7 @@ class CurveModel: ...@@ -267,7 +267,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
samples: list samples : list
a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix,
representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}}
...@@ -322,7 +322,7 @@ class CurveModel: ...@@ -322,7 +322,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
trial_history: list trial_history : list
The history performance matrix of each trial. The history performance matrix of each trial.
Returns Returns
......
src/sdk/pynni/nni/gp_tuner/gp_tuner.py
View file @ 751445d3
...@@ -17,9 +17,11 @@ ...@@ -17,9 +17,11 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
''' """
gp_tuner.py GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions.
'''
See :class:`GPTuner` for details.
"""
import warnings import warnings
import logging import logging
...@@ -38,18 +40,40 @@ logger = logging.getLogger("GP_Tuner_AutoML") ...@@ -38,18 +40,40 @@ logger = logging.getLogger("GP_Tuner_AutoML")
class GPTuner(Tuner): class GPTuner(Tuner):
''' """
GPTuner GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions.
'''
Parameters
----------
optimize_mode : str
optimize mode, 'maximize' or 'minimize', by default 'maximize'
utility : str
utility function (also called 'acquisition funcition') to use, which can be 'ei', 'ucb' or 'poi'. By default 'ei'.
kappa : float
value used by utility function 'ucb'. The bigger kappa is, the more the tuner will be exploratory. By default 5.
xi : float
used by utility function 'ei' and 'poi'. The bigger xi is, the more the tuner will be exploratory. By default 0.
nu : float
used to specify Matern kernel. The smaller nu, the less smooth the approximated function is. By default 2.5.
alpha : float
Used to specify Gaussian Process Regressor. Larger values correspond to increased noise level in the observations.
By default 1e-6.
cold_start_num : int
Number of random exploration to perform before Gaussian Process. By default 10.
selection_num_warm_up : int
Number of random points to evaluate for getting the point which maximizes the acquisition function. By default 100000
selection_num_starting_points : int
Number of times to run L-BFGS-B from a random starting point after the warmup. By default 250.
"""
def __init__(self, optimize_mode="maximize", utility='ei', kappa=5, xi=0, nu=2.5, alpha=1e-6, cold_start_num=10, def __init__(self, optimize_mode="maximize", utility='ei', kappa=5, xi=0, nu=2.5, alpha=1e-6, cold_start_num=10,
selection_num_warm_up=100000, selection_num_starting_points=250): selection_num_warm_up=100000, selection_num_starting_points=250):
self.optimize_mode = OptimizeMode(optimize_mode) self._optimize_mode = OptimizeMode(optimize_mode)
# utility function related # utility function related
self.utility = utility self._utility = utility
self.kappa = kappa self._kappa = kappa
self.xi = xi self._xi = xi
# target space # target space
self._space = None self._space = None
...@@ -72,30 +96,23 @@ class GPTuner(Tuner): ...@@ -72,30 +96,23 @@ class GPTuner(Tuner):
self._selection_num_starting_points = selection_num_starting_points self._selection_num_starting_points = selection_num_starting_points
# num of imported data # num of imported data
self.supplement_data_num = 0 self._supplement_data_num = 0
def update_search_space(self, search_space): def update_search_space(self, search_space):
"""Update the self.bounds and self.types by the search_space.json """
Update the self.bounds and self.types by the search_space.json file.
Parameters Override of the abstract method in :class:`~nni.tuner.Tuner`.
----------
search_space : dict
""" """
self._space = TargetSpace(search_space, self._random_state) self._space = TargetSpace(search_space, self._random_state)
def generate_parameters(self, parameter_id, **kwargs): def generate_parameters(self, parameter_id, **kwargs):
"""Generate next parameter for trial """
If the number of trial result is lower than cold start number, Method which provides one set of hyper-parameters.
gp will first randomly generate some parameters. If the number of trial result is lower than cold_start_number, GPTuner will first randomly generate some parameters.
Otherwise, choose the parameters by the Gussian Process Model Otherwise, choose the parameters by the Gussian Process Model.
Parameters
----------
parameter_id : int
Returns Override of the abstract method in :class:`~nni.tuner.Tuner`.
-------
result : dict
""" """
if self._space.len() < self._cold_start_num: if self._space.len() < self._cold_start_num:
results = self._space.random_sample() results = self._space.random_sample()
...@@ -107,7 +124,7 @@ class GPTuner(Tuner): ...@@ -107,7 +124,7 @@ class GPTuner(Tuner):
self._gp.fit(self._space.params, self._space.target) self._gp.fit(self._space.params, self._space.target)
util = UtilityFunction( util = UtilityFunction(
kind=self.utility, kappa=self.kappa, xi=self.xi) kind=self._utility, kappa=self._kappa, xi=self._xi)
results = acq_max( results = acq_max(
f_acq=util.utility, f_acq=util.utility,
...@@ -124,17 +141,13 @@ class GPTuner(Tuner): ...@@ -124,17 +141,13 @@ class GPTuner(Tuner):
return results return results
def receive_trial_result(self, parameter_id, parameters, value, **kwargs): def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
"""Tuner receive result from trial. """
Method invoked when a trial reports its final result.
Parameters Override of the abstract method in :class:`~nni.tuner.Tuner`.
----------
parameter_id : int
parameters : dict
value : dict/float
if value is dict, it should have "default" key.
""" """
value = extract_scalar_reward(value) value = extract_scalar_reward(value)
if self.optimize_mode == OptimizeMode.Minimize: if self._optimize_mode == OptimizeMode.Minimize:
value = -value value = -value
logger.info("Received trial result.") logger.info("Received trial result.")
...@@ -143,26 +156,27 @@ class GPTuner(Tuner): ...@@ -143,26 +156,27 @@ class GPTuner(Tuner):
self._space.register(parameters, value) self._space.register(parameters, value)
def import_data(self, data): def import_data(self, data):
"""Import additional data for tuning """
Parameters Import additional data for tuning.
----------
data: Override of the abstract method in :class:`~nni.tuner.Tuner`.
a list of dictionarys, each of which has at least two keys, 'parameter' and 'value'
""" """
_completed_num = 0 _completed_num = 0
for trial_info in data: for trial_info in data:
logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) logger.info(
"Importing data, current processing progress %s / %s", _completed_num, len(data))
_completed_num += 1 _completed_num += 1
assert "parameter" in trial_info assert "parameter" in trial_info
_params = trial_info["parameter"] _params = trial_info["parameter"]
assert "value" in trial_info assert "value" in trial_info
_value = trial_info['value'] _value = trial_info['value']
if not _value: if not _value:
logger.info("Useless trial data, value is %s, skip this trial data.", _value) logger.info(
"Useless trial data, value is %s, skip this trial data.", _value)
continue continue
self.supplement_data_num += 1 self._supplement_data_num += 1
_parameter_id = '_'.join( _parameter_id = '_'.join(
["ImportData", str(self.supplement_data_num)]) ["ImportData", str(self._supplement_data_num)])
self.receive_trial_result( self.receive_trial_result(
parameter_id=_parameter_id, parameters=_params, value=_value) parameter_id=_parameter_id, parameters=_params, value=_value)
logger.info("Successfully import data to GP tuner.") logger.info("Successfully import data to GP tuner.")
src/sdk/pynni/nni/gp_tuner/target_space.py
View file @ 751445d3
...@@ -17,39 +17,51 @@ ...@@ -17,39 +17,51 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
''' """
target_space.py Tool class to hold the param-space coordinates (X) and target values (Y).
''' """
import numpy as np import numpy as np
import nni.parameter_expressions as parameter_expressions import nni.parameter_expressions as parameter_expressions
def _hashable(params): def _hashable(params):
""" ensure that an point is hashable by a python dict """ """
Transform list params to tuple format. Ensure that an point is hashable by a python dict.
Parameters
----------
params : numpy array
array format of parameters
Returns
-------
tuple
tuple format of parameters
"""
return tuple(map(float, params)) return tuple(map(float, params))
class TargetSpace(): class TargetSpace():
""" """
Holds the param-space coordinates (X) and target values (Y) Holds the param-space coordinates (X) and target values (Y)
"""
def __init__(self, pbounds, random_state=None):
"""
Parameters Parameters
---------- ----------
pbounds : dict pbounds : dict
Dictionary with parameters names as keys and a tuple with minimum Dictionary with parameters names and legal values.
and maximum values.
random_state : int, RandomState, or None random_state : int, RandomState, or None
optionally specify a seed for a random number generator optionally specify a seed for a random number generator, by default None.
""" """
self.random_state = random_state
def __init__(self, pbounds, random_state=None):
self._random_state = random_state
# Get the name of the parameters # Get the name of the parameters
self._keys = sorted(pbounds) self._keys = sorted(pbounds)
# Create an array with parameters bounds # Create an array with parameters bounds
self._bounds = np.array( self._bounds = np.array(
[item[1] for item in sorted(pbounds.items(), key=lambda x: x[0])] [item[1] for item in sorted(pbounds.items(), key=lambda x: x[0])]
...@@ -71,54 +83,100 @@ class TargetSpace(): ...@@ -71,54 +83,100 @@ class TargetSpace():
self._cache = {} self._cache = {}
def __contains__(self, params): def __contains__(self, params):
''' """
check if a parameter is already registered check if a parameter is already registered
'''
Parameters
----------
params : numpy array
Returns
-------
bool
True if the parameter is already registered, else false
"""
return _hashable(params) in self._cache return _hashable(params) in self._cache
def len(self): def len(self):
''' """
length of registered params and targets length of registered params and targets
'''
Returns
-------
int
"""
assert len(self._params) == len(self._target) assert len(self._params) == len(self._target)
return len(self._target) return len(self._target)
@property @property
def params(self): def params(self):
''' """
params: numpy array registered parameters
'''
Returns
-------
numpy array
"""
return self._params return self._params
@property @property
def target(self): def target(self):
''' """
target: numpy array registered target values
'''
Returns
-------
numpy array
"""
return self._target return self._target
@property @property
def dim(self): def dim(self):
''' """
dim: int dimension of parameters
length of keys
''' Returns
-------
int
"""
return len(self._keys) return len(self._keys)
@property @property
def keys(self): def keys(self):
''' """
keys: numpy array keys of parameters
'''
Returns
-------
numpy array
"""
return self._keys return self._keys
@property @property
def bounds(self): def bounds(self):
'''bounds''' """
bounds of parameters
Returns
-------
numpy array
"""
return self._bounds return self._bounds
def params_to_array(self, params): def params_to_array(self, params):
''' dict to array ''' """
dict to array
Parameters
----------
params : dict
dict format of parameters
Returns
-------
numpy array
array format of parameters
"""
try: try:
assert set(params) == set(self.keys) assert set(params) == set(self.keys)
except AssertionError: except AssertionError:
...@@ -129,11 +187,20 @@ class TargetSpace(): ...@@ -129,11 +187,20 @@ class TargetSpace():
return np.asarray([params[key] for key in self.keys]) return np.asarray([params[key] for key in self.keys])
def array_to_params(self, x): def array_to_params(self, x):
''' """
array to dict array to dict
maintain int type if the paramters is defined as int in search_space.json maintain int type if the paramters is defined as int in search_space.json
''' Parameters
----------
x : numpy array
array format of parameters
Returns
-------
dict
dict format of parameters
"""
try: try:
assert len(x) == len(self.keys) assert len(x) == len(self.keys)
except AssertionError: except AssertionError:
...@@ -159,15 +226,15 @@ class TargetSpace(): ...@@ -159,15 +226,15 @@ class TargetSpace():
Parameters Parameters
---------- ----------
x : dict params : dict
parameters
y : float target : float
target function value target function value
""" """
x = self.params_to_array(params) x = self.params_to_array(params)
if x in self: if x in self:
#raise KeyError('Data point {} is not unique'.format(x))
print('Data point {} is not unique'.format(x)) print('Data point {} is not unique'.format(x))
# Insert data into unique dictionary # Insert data into unique dictionary
...@@ -180,32 +247,43 @@ class TargetSpace(): ...@@ -180,32 +247,43 @@ class TargetSpace():
""" """
Creates a random point within the bounds of the space. Creates a random point within the bounds of the space.
Returns
-------
numpy array
one groupe of parameter
""" """
params = np.empty(self.dim) params = np.empty(self.dim)
for col, _bound in enumerate(self._bounds): for col, _bound in enumerate(self._bounds):
if _bound['_type'] == 'choice': if _bound['_type'] == 'choice':
params[col] = parameter_expressions.choice( params[col] = parameter_expressions.choice(
_bound['_value'], self.random_state) _bound['_value'], self._random_state)
elif _bound['_type'] == 'randint': elif _bound['_type'] == 'randint':
params[col] = self.random_state.randint( params[col] = self._random_state.randint(
_bound['_value'][0], _bound['_value'][1], size=1) _bound['_value'][0], _bound['_value'][1], size=1)
elif _bound['_type'] == 'uniform': elif _bound['_type'] == 'uniform':
params[col] = parameter_expressions.uniform( params[col] = parameter_expressions.uniform(
_bound['_value'][0], _bound['_value'][1], self.random_state) _bound['_value'][0], _bound['_value'][1], self._random_state)
elif _bound['_type'] == 'quniform': elif _bound['_type'] == 'quniform':
params[col] = parameter_expressions.quniform( params[col] = parameter_expressions.quniform(
_bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self.random_state) _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state)
elif _bound['_type'] == 'loguniform': elif _bound['_type'] == 'loguniform':
params[col] = parameter_expressions.loguniform( params[col] = parameter_expressions.loguniform(
_bound['_value'][0], _bound['_value'][1], self.random_state) _bound['_value'][0], _bound['_value'][1], self._random_state)
elif _bound['_type'] == 'qloguniform': elif _bound['_type'] == 'qloguniform':
params[col] = parameter_expressions.qloguniform( params[col] = parameter_expressions.qloguniform(
_bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self.random_state) _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state)
return params return params
def max(self): def max(self):
"""Get maximum target value found and corresponding parametes.""" """
Get maximum target value found and its corresponding parameters.
Returns
-------
dict
target value and parameters, empty dict if nothing registered
"""
try: try:
res = { res = {
'target': self.target.max(), 'target': self.target.max(),
...@@ -218,7 +296,14 @@ class TargetSpace(): ...@@ -218,7 +296,14 @@ class TargetSpace():
return res return res
def res(self): def res(self):
"""Get all target values found and corresponding parametes.""" """
Get all target values found and corresponding parameters.
Returns
-------
list
a list of target values and their corresponding parameters
"""
params = [dict(zip(self.keys, p)) for p in self.params] params = [dict(zip(self.keys, p)) for p in self.params]
return [ return [
......
src/sdk/pynni/nni/gp_tuner/util.py
View file @ 751445d3
...@@ -17,9 +17,9 @@ ...@@ -17,9 +17,9 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
''' """
gp_tuner.py utility functions and classes for GPTuner
''' """
import warnings import warnings
import numpy as np import numpy as np
...@@ -28,9 +28,21 @@ from scipy.optimize import minimize ...@@ -28,9 +28,21 @@ from scipy.optimize import minimize
def _match_val_type(vals, bounds): def _match_val_type(vals, bounds):
''' """
Update values in the array, to match their corresponding type Update values in the array, to match their corresponding type, make sure the value is legal.
'''
Parameters
----------
vals : numpy array
values of parameters
bounds : numpy array
list of dictionary which stores parameters names and legal values.
Returns
-------
vals_new : list
The closest legal value to the original value
"""
vals_new = [] vals_new = []
for i, bound in enumerate(bounds): for i, bound in enumerate(bounds):
...@@ -52,32 +64,33 @@ def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points): ...@@ -52,32 +64,33 @@ def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points):
A function to find the maximum of the acquisition function A function to find the maximum of the acquisition function
It uses a combination of random sampling (cheap) and the 'L-BFGS-B' It uses a combination of random sampling (cheap) and the 'L-BFGS-B'
optimization method. First by sampling `n_warmup` (1e5) points at random, optimization method. First by sampling ``num_warmup`` points at random,
and then running L-BFGS-B from `n_iter` (250) random starting points. and then running L-BFGS-B from ``num_starting_points`` random starting points.
Parameters Parameters
---------- ----------
:param f_acq: f_acq : UtilityFunction.utility
The acquisition function object that return its point-wise value. The acquisition function object that return its point-wise value.
:param gp: gp : GaussianProcessRegressor
A gaussian process fitted to the relevant data. A gaussian process fitted to the relevant data.
:param y_max: y_max : float
The current maximum known value of the target function. The current maximum known value of the target function.
:param bounds: bounds : numpy array
The variables bounds to limit the search of the acq max. The variables bounds to limit the search of the acq max.
:param num_warmup: num_warmup : int
number of times to randomly sample the aquisition function number of times to randomly sample the aquisition function
:param num_starting_points: num_starting_points : int
number of times to run scipy.minimize number of times to run scipy.minimize
Returns Returns
------- -------
:return: x_max, The arg max of the acquisition function. numpy array
The parameter which achieves max of the acquisition function.
""" """
# Warm up with random points # Warm up with random points
...@@ -117,36 +130,70 @@ def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points): ...@@ -117,36 +130,70 @@ def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points):
class UtilityFunction(): class UtilityFunction():
""" """
An object to compute the acquisition functions. A class to compute different acquisition function values.
"""
def __init__(self, kind, kappa, xi): Parameters
""" ----------
If UCB is to be used, a constant kappa is needed. kind : string
specification of utility function to use
kappa : float
parameter usedd for 'ucb' acquisition function
xi : float
parameter usedd for 'ei' and 'poi' acquisition function
""" """
self.kappa = kappa
self.xi = xi def __init__(self, kind, kappa, xi):
self._kappa = kappa
self._xi = xi
if kind not in ['ucb', 'ei', 'poi']: if kind not in ['ucb', 'ei', 'poi']:
err = "The utility function " \ err = "The utility function " \
"{} has not been implemented, " \ "{} has not been implemented, " \
"please choose one of ucb, ei, or poi.".format(kind) "please choose one of ucb, ei, or poi.".format(kind)
raise NotImplementedError(err) raise NotImplementedError(err)
self.kind = kind self._kind = kind
def utility(self, x, gp, y_max): def utility(self, x, gp, y_max):
'''return utility function''' """
if self.kind == 'ucb': return utility function
return self._ucb(x, gp, self.kappa)
if self.kind == 'ei': Parameters
return self._ei(x, gp, y_max, self.xi) ----------
if self.kind == 'poi': x : numpy array
return self._poi(x, gp, y_max, self.xi) parameters
gp : GaussianProcessRegressor
y_max : float
maximum target value observed so far
Returns
-------
function
return corresponding function, return None if parameter is illegal
"""
if self._kind == 'ucb':
return self._ucb(x, gp, self._kappa)
if self._kind == 'ei':
return self._ei(x, gp, y_max, self._xi)
if self._kind == 'poi':
return self._poi(x, gp, y_max, self._xi)
return None return None
@staticmethod @staticmethod
def _ucb(x, gp, kappa): def _ucb(x, gp, kappa):
"""
Upper Confidence Bound (UCB) utility function
Parameters
----------
x : numpy array
parameters
gp : GaussianProcessRegressor
kappa : float
Returns
-------
float
"""
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("ignore") warnings.simplefilter("ignore")
mean, std = gp.predict(x, return_std=True) mean, std = gp.predict(x, return_std=True)
...@@ -155,6 +202,22 @@ class UtilityFunction(): ...@@ -155,6 +202,22 @@ class UtilityFunction():
@staticmethod @staticmethod
def _ei(x, gp, y_max, xi): def _ei(x, gp, y_max, xi):
"""
Expected Improvement (EI) utility function
Parameters
----------
x : numpy array
parameters
gp : GaussianProcessRegressor
y_max : float
maximum target value observed so far
xi : float
Returns
-------
float
"""
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("ignore") warnings.simplefilter("ignore")
mean, std = gp.predict(x, return_std=True) mean, std = gp.predict(x, return_std=True)
...@@ -164,6 +227,22 @@ class UtilityFunction(): ...@@ -164,6 +227,22 @@ class UtilityFunction():
@staticmethod @staticmethod
def _poi(x, gp, y_max, xi): def _poi(x, gp, y_max, xi):
"""
Possibility Of Improvement (POI) utility function
Parameters
----------
x : numpy array
parameters
gp : GaussianProcessRegressor
y_max : float
maximum target value observed so far
xi : float
Returns
-------
float
"""
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("ignore") warnings.simplefilter("ignore")
mean, std = gp.predict(x, return_std=True) mean, std = gp.predict(x, return_std=True)
......
src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py
View file @ 751445d3
...@@ -27,21 +27,21 @@ class MedianstopAssessor(Assessor): ...@@ -27,21 +27,21 @@ class MedianstopAssessor(Assessor):
Parameters Parameters
---------- ----------
optimize_mode: str optimize_mode : str
optimize mode, 'maximize' or 'minimize' optimize mode, 'maximize' or 'minimize'
start_step: int start_step : int
only after receiving start_step number of reported intermediate results only after receiving start_step number of reported intermediate results
""" """
def __init__(self, optimize_mode='maximize', start_step=0): def __init__(self, optimize_mode='maximize', start_step=0):
self.start_step = start_step self._start_step = start_step
self.running_history = dict() self._running_history = dict()
self.completed_avg_history = dict() self._completed_avg_history = dict()
if optimize_mode == 'maximize': if optimize_mode == 'maximize':
self.high_better = True self._high_better = True
elif optimize_mode == 'minimize': elif optimize_mode == 'minimize':
self.high_better = False self._high_better = False
else: else:
self.high_better = True self._high_better = True
logger.warning('unrecognized optimize_mode %s', optimize_mode) logger.warning('unrecognized optimize_mode %s', optimize_mode)
def _update_data(self, trial_job_id, trial_history): def _update_data(self, trial_job_id, trial_history):
...@@ -49,35 +49,35 @@ class MedianstopAssessor(Assessor): ...@@ -49,35 +49,35 @@ class MedianstopAssessor(Assessor):
Parameters Parameters
---------- ----------
trial_job_id: int trial_job_id : int
trial job id trial job id
trial_history: list trial_history : list
The history performance matrix of each trial The history performance matrix of each trial
""" """
if trial_job_id not in self.running_history: if trial_job_id not in self._running_history:
self.running_history[trial_job_id] = [] self._running_history[trial_job_id] = []
self.running_history[trial_job_id].extend(trial_history[len(self.running_history[trial_job_id]):]) self._running_history[trial_job_id].extend(trial_history[len(self._running_history[trial_job_id]):])
def trial_end(self, trial_job_id, success): def trial_end(self, trial_job_id, success):
"""trial_end """trial_end
Parameters Parameters
---------- ----------
trial_job_id: int trial_job_id : int
trial job id trial job id
success: bool success : bool
True if succssfully finish the experiment, False otherwise True if succssfully finish the experiment, False otherwise
""" """
if trial_job_id in self.running_history: if trial_job_id in self._running_history:
if success: if success:
cnt = 0 cnt = 0
history_sum = 0 history_sum = 0
self.completed_avg_history[trial_job_id] = [] self._completed_avg_history[trial_job_id] = []
for each in self.running_history[trial_job_id]: for each in self._running_history[trial_job_id]:
cnt += 1 cnt += 1
history_sum += each history_sum += each
self.completed_avg_history[trial_job_id].append(history_sum / cnt) self._completed_avg_history[trial_job_id].append(history_sum / cnt)
self.running_history.pop(trial_job_id) self._running_history.pop(trial_job_id)
else: else:
logger.warning('trial_end: trial_job_id does not exist in running_history') logger.warning('trial_end: trial_job_id does not exist in running_history')
...@@ -86,9 +86,9 @@ class MedianstopAssessor(Assessor): ...@@ -86,9 +86,9 @@ class MedianstopAssessor(Assessor):
Parameters Parameters
---------- ----------
trial_job_id: int trial_job_id : int
trial job id trial job id
trial_history: list trial_history : list
The history performance matrix of each trial The history performance matrix of each trial
Returns Returns
...@@ -102,7 +102,7 @@ class MedianstopAssessor(Assessor): ...@@ -102,7 +102,7 @@ class MedianstopAssessor(Assessor):
unrecognize exception in medianstop_assessor unrecognize exception in medianstop_assessor
""" """
curr_step = len(trial_history) curr_step = len(trial_history)
if curr_step < self.start_step: if curr_step < self._start_step:
return AssessResult.Good return AssessResult.Good
try: try:
...@@ -115,18 +115,18 @@ class MedianstopAssessor(Assessor): ...@@ -115,18 +115,18 @@ class MedianstopAssessor(Assessor):
logger.exception(error) logger.exception(error)
self._update_data(trial_job_id, num_trial_history) self._update_data(trial_job_id, num_trial_history)
if self.high_better: if self._high_better:
best_history = max(trial_history) best_history = max(trial_history)
else: else:
best_history = min(trial_history) best_history = min(trial_history)
avg_array = [] avg_array = []
for id_ in self.completed_avg_history: for id_ in self._completed_avg_history:
if len(self.completed_avg_history[id_]) >= curr_step: if len(self._completed_avg_history[id_]) >= curr_step:
avg_array.append(self.completed_avg_history[id_][curr_step - 1]) avg_array.append(self._completed_avg_history[id_][curr_step - 1])
if avg_array: if avg_array:
avg_array.sort() avg_array.sort()
if self.high_better: if self._high_better:
median = avg_array[(len(avg_array)-1) // 2] median = avg_array[(len(avg_array)-1) // 2]
return AssessResult.Bad if best_history < median else AssessResult.Good return AssessResult.Bad if best_history < median else AssessResult.Good
else: else:
......
src/sdk/pynni/nni/tuner.py
View file @ 751445d3
...@@ -79,7 +79,8 @@ class Tuner(Recoverable): ...@@ -79,7 +79,8 @@ class Tuner(Recoverable):
:class:`~nni.smac_tuner.smac_tuner.SMACTuner` :class:`~nni.smac_tuner.smac_tuner.SMACTuner`
:class:`~nni.gridsearch_tuner.gridsearch_tuner.GridSearchTuner` :class:`~nni.gridsearch_tuner.gridsearch_tuner.GridSearchTuner`
:class:`~nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner` :class:`~nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner`
:class:`~nni.metis_tuner.mets_tuner.MetisTuner` :class:`~nni.metis_tuner.metis_tuner.MetisTuner`
:class:`~nni.gp_tuner.gp_tuner.GPTuner`
""" """
def generate_parameters(self, parameter_id, **kwargs): def generate_parameters(self, parameter_id, **kwargs):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment