Skip to content

GitLab

Unverified Commit 751445d3 authored by Guoxin's avatar Guoxin Committed by GitHub
Browse files

docstr/pylint of GP Tuner & CurveFitting Assessor & MedianStop Assessor (#1692) 

# docstr/pylint of GP Tuner & CurveFitting Assessor & MedianStop Assessor
parent e6df29cf
Hide whitespace changes
Inline Side-by-side
Showing with 409 additions and 226 deletions
docs/en_US/sdk_reference.rst
View file @ 751445d3
...@@ -39,6 +39,9 @@ Tuner ...@@ -39,6 +39,9 @@ Tuner
.. autoclass:: nni.batch_tuner.batch_tuner.BatchTuner .. autoclass:: nni.batch_tuner.batch_tuner.BatchTuner
:members: :members:
.. autoclass:: nni.gp_tuner.gp_tuner.GPTuner
:members:
Assessor Assessor
------------------------ ------------------------
.. autoclass:: nni.assessor.Assessor .. autoclass:: nni.assessor.Assessor
......
src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py
View file @ 751445d3
...@@ -29,13 +29,13 @@ class CurvefittingAssessor(Assessor): ...@@ -29,13 +29,13 @@ class CurvefittingAssessor(Assessor):
Parameters Parameters
---------- ----------
epoch_num: int epoch_num : int
The total number of epoch The total number of epoch
optimize_mode: str optimize_mode : str
optimize mode, 'maximize' or 'minimize' optimize mode, 'maximize' or 'minimize'
start_step: int start_step : int
only after receiving start_step number of reported intermediate results only after receiving start_step number of reported intermediate results
threshold: float threshold : float
The threshold that we decide to early stop the worse performance curve. The threshold that we decide to early stop the worse performance curve.
""" """
def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95, gap=1): def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95, gap=1):
...@@ -70,9 +70,9 @@ class CurvefittingAssessor(Assessor): ...@@ -70,9 +70,9 @@ class CurvefittingAssessor(Assessor):
Parameters Parameters
---------- ----------
trial_job_id: int trial_job_id : int
trial job id trial job id
success: bool success : bool
True if succssfully finish the experiment, False otherwise True if succssfully finish the experiment, False otherwise
""" """
if success: if success:
...@@ -90,9 +90,9 @@ class CurvefittingAssessor(Assessor): ...@@ -90,9 +90,9 @@ class CurvefittingAssessor(Assessor):
Parameters Parameters
---------- ----------
trial_job_id: int trial_job_id : int
trial job id trial job id
trial_history: list trial_history : list
The history performance matrix of each trial The history performance matrix of each trial
Returns Returns
...@@ -105,7 +105,6 @@ class CurvefittingAssessor(Assessor): ...@@ -105,7 +105,6 @@ class CurvefittingAssessor(Assessor):
Exception Exception
unrecognize exception in curvefitting_assessor unrecognize exception in curvefitting_assessor
""" """
trial_job_id = trial_job_id
self.trial_history = trial_history self.trial_history = trial_history
if not self.set_best_performance: if not self.set_best_performance:
return AssessResult.Good return AssessResult.Good
......
src/sdk/pynni/nni/curvefitting_assessor/curvefunctions.py
View file @ 751445d3
...@@ -14,7 +14,9 @@ ...@@ -14,7 +14,9 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
A family of functions used by CurvefittingAssessor
"""
import numpy as np import numpy as np
all_models = {} all_models = {}
...@@ -29,10 +31,10 @@ def vap(x, a, b, c): ...@@ -29,10 +31,10 @@ def vap(x, a, b, c):
Parameters Parameters
---------- ----------
x: int x : int
a: float a : float
b: float b : float
c: float c : float
Returns Returns
------- -------
...@@ -50,10 +52,10 @@ def pow3(x, c, a, alpha): ...@@ -50,10 +52,10 @@ def pow3(x, c, a, alpha):
Parameters Parameters
---------- ----------
x: int x : int
c: float c : float
a: float a : float
alpha: float alpha : float
Returns Returns
------- -------
...@@ -71,9 +73,9 @@ def linear(x, a, b): ...@@ -71,9 +73,9 @@ def linear(x, a, b):
Parameters Parameters
---------- ----------
x: int x : int
a: float a : float
b: float b : float
Returns Returns
------- -------
...@@ -91,9 +93,9 @@ def logx_linear(x, a, b): ...@@ -91,9 +93,9 @@ def logx_linear(x, a, b):
Parameters Parameters
---------- ----------
x: int x : int
a: float a : float
b: float b : float
Returns Returns
------- -------
...@@ -112,10 +114,10 @@ def dr_hill_zero_background(x, theta, eta, kappa): ...@@ -112,10 +114,10 @@ def dr_hill_zero_background(x, theta, eta, kappa):
Parameters Parameters
---------- ----------
x: int x : int
theta: float theta : float
eta: float eta : float
kappa: float kappa : float
Returns Returns
------- -------
...@@ -133,10 +135,10 @@ def log_power(x, a, b, c): ...@@ -133,10 +135,10 @@ def log_power(x, a, b, c):
Parameters Parameters
---------- ----------
x: int x : int
a: float a : float
b: float b : float
c: float c : float
Returns Returns
------- -------
...@@ -154,11 +156,11 @@ def pow4(x, alpha, a, b, c): ...@@ -154,11 +156,11 @@ def pow4(x, alpha, a, b, c):
Parameters Parameters
---------- ----------
x: int x : int
alpha: float alpha : float
a: float a : float
b: float b : float
c: float c : float
Returns Returns
------- -------
...@@ -177,11 +179,11 @@ def mmf(x, alpha, beta, kappa, delta): ...@@ -177,11 +179,11 @@ def mmf(x, alpha, beta, kappa, delta):
Parameters Parameters
---------- ----------
x: int x : int
alpha: float alpha : float
beta: float beta : float
kappa: float kappa : float
delta: float delta : float
Returns Returns
------- -------
...@@ -199,11 +201,11 @@ def exp4(x, c, a, b, alpha): ...@@ -199,11 +201,11 @@ def exp4(x, c, a, b, alpha):
Parameters Parameters
---------- ----------
x: int x : int
c: float c : float
a: float a : float
b: float b : float
alpha: float alpha : float
Returns Returns
------- -------
...@@ -221,9 +223,9 @@ def ilog2(x, c, a): ...@@ -221,9 +223,9 @@ def ilog2(x, c, a):
Parameters Parameters
---------- ----------
x: int x : int
c: float c : float
a: float a : float
Returns Returns
------- -------
...@@ -242,11 +244,11 @@ def weibull(x, alpha, beta, kappa, delta): ...@@ -242,11 +244,11 @@ def weibull(x, alpha, beta, kappa, delta):
Parameters Parameters
---------- ----------
x: int x : int
alpha: float alpha : float
beta: float beta : float
kappa: float kappa : float
delta: float delta : float
Returns Returns
------- -------
...@@ -264,11 +266,11 @@ def janoschek(x, a, beta, k, delta): ...@@ -264,11 +266,11 @@ def janoschek(x, a, beta, k, delta):
Parameters Parameters
---------- ----------
x: int x : int
a: float a : float
beta: float beta : float
k: float k : float
delta: float delta : float
Returns Returns
------- -------
......
src/sdk/pynni/nni/curvefitting_assessor/model_factory.py
View file @ 751445d3
...@@ -40,7 +40,7 @@ class CurveModel: ...@@ -40,7 +40,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
target_pos: int target_pos : int
The point we need to predict The point we need to predict
""" """
def __init__(self, target_pos): def __init__(self, target_pos):
...@@ -120,14 +120,14 @@ class CurveModel: ...@@ -120,14 +120,14 @@ class CurveModel:
Parameters Parameters
---------- ----------
model: string model : string
name of the curve function model name of the curve function model
pos: int pos : int
the epoch number of the position you want to predict the epoch number of the position you want to predict
Returns Returns
------- -------
int: int
The expected matrix at pos The expected matrix at pos
""" """
if model_para_num[model] == 2: if model_para_num[model] == 2:
...@@ -143,9 +143,9 @@ class CurveModel: ...@@ -143,9 +143,9 @@ class CurveModel:
Parameters Parameters
---------- ----------
pos: int pos : int
the epoch number of the position you want to predict the epoch number of the position you want to predict
sample: list sample : list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns Returns
...@@ -165,7 +165,7 @@ class CurveModel: ...@@ -165,7 +165,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
samples: list samples : list
a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix,
representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}}
...@@ -187,7 +187,7 @@ class CurveModel: ...@@ -187,7 +187,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
sample: list sample : list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns Returns
...@@ -206,9 +206,9 @@ class CurveModel: ...@@ -206,9 +206,9 @@ class CurveModel:
Parameters Parameters
---------- ----------
pos: int pos : int
the epoch number of the position you want to predict the epoch number of the position you want to predict
sample: list sample : list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns Returns
...@@ -225,7 +225,7 @@ class CurveModel: ...@@ -225,7 +225,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
sample: list sample : list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns Returns
...@@ -244,7 +244,7 @@ class CurveModel: ...@@ -244,7 +244,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
samples: list samples : list
a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix,
representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}}
...@@ -267,7 +267,7 @@ class CurveModel: ...@@ -267,7 +267,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
samples: list samples : list
a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix,
representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}}
...@@ -322,7 +322,7 @@ class CurveModel: ...@@ -322,7 +322,7 @@ class CurveModel:
Parameters Parameters
---------- ----------
trial_history: list trial_history : list
The history performance matrix of each trial. The history performance matrix of each trial.
Returns Returns
......
src/sdk/pynni/nni/gp_tuner/gp_tuner.py
View file @ 751445d3
...@@ -17,9 +17,11 @@ ...@@ -17,9 +17,11 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
''' """
gp_tuner.py GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions.
'''
See :class:`GPTuner` for details.
"""
import warnings import warnings
import logging import logging
...@@ -38,18 +40,40 @@ logger = logging.getLogger("GP_Tuner_AutoML") ...@@ -38,18 +40,40 @@ logger = logging.getLogger("GP_Tuner_AutoML")
class GPTuner(Tuner): class GPTuner(Tuner):
''' """
GPTuner GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions.
'''
Parameters
----------
optimize_mode : str
optimize mode, 'maximize' or 'minimize', by default 'maximize'
utility : str
utility function (also called 'acquisition funcition') to use, which can be 'ei', 'ucb' or 'poi'. By default 'ei'.
kappa : float
value used by utility function 'ucb'. The bigger kappa is, the more the tuner will be exploratory. By default 5.
xi : float
used by utility function 'ei' and 'poi'. The bigger xi is, the more the tuner will be exploratory. By default 0.
nu : float
used to specify Matern kernel. The smaller nu, the less smooth the approximated function is. By default 2.5.
alpha : float
Used to specify Gaussian Process Regressor. Larger values correspond to increased noise level in the observations.
By default 1e-6.
cold_start_num : int
Number of random exploration to perform before Gaussian Process. By default 10.
selection_num_warm_up : int
Number of random points to evaluate for getting the point which maximizes the acquisition function. By default 100000
selection_num_starting_points : int
Number of times to run L-BFGS-B from a random starting point after the warmup. By default 250.
"""
def __init__(self, optimize_mode="maximize", utility='ei', kappa=5, xi=0, nu=2.5, alpha=1e-6, cold_start_num=10, def __init__(self, optimize_mode="maximize", utility='ei', kappa=5, xi=0, nu=2.5, alpha=1e-6, cold_start_num=10,
selection_num_warm_up=100000, selection_num_starting_points=250): selection_num_warm_up=100000, selection_num_starting_points=250):
self.optimize_mode = OptimizeMode(optimize_mode) self._optimize_mode = OptimizeMode(optimize_mode)
# utility function related # utility function related
self.utility = utility self._utility = utility
self.kappa = kappa self._kappa = kappa
self.xi = xi self._xi = xi
# target space # target space
self._space = None self._space = None
...@@ -72,30 +96,23 @@ class GPTuner(Tuner): ...@@ -72,30 +96,23 @@ class GPTuner(Tuner):
self._selection_num_starting_points = selection_num_starting_points self._selection_num_starting_points = selection_num_starting_points
# num of imported data # num of imported data
self.supplement_data_num = 0 self._supplement_data_num = 0
def update_search_space(self, search_space): def update_search_space(self, search_space):
"""Update the self.bounds and self.types by the search_space.json """
Update the self.bounds and self.types by the search_space.json file.
Parameters Override of the abstract method in :class:`~nni.tuner.Tuner`.
----------
search_space : dict
""" """
self._space = TargetSpace(search_space, self._random_state) self._space = TargetSpace(search_space, self._random_state)
def generate_parameters(self, parameter_id, **kwargs): def generate_parameters(self, parameter_id, **kwargs):
"""Generate next parameter for trial """
If the number of trial result is lower than cold start number, Method which provides one set of hyper-parameters.
gp will first randomly generate some parameters. If the number of trial result is lower than cold_start_number, GPTuner will first randomly generate some parameters.
Otherwise, choose the parameters by the Gussian Process Model Otherwise, choose the parameters by the Gussian Process Model.
Parameters Override of the abstract method in :class:`~nni.tuner.Tuner`.
----------
parameter_id : int
Returns
-------
result : dict
""" """
if self._space.len() < self._cold_start_num: if self._space.len() < self._cold_start_num:
results = self._space.random_sample() results = self._space.random_sample()
...@@ -107,7 +124,7 @@ class GPTuner(Tuner): ...@@ -107,7 +124,7 @@ class GPTuner(Tuner):
self._gp.fit(self._space.params, self._space.target) self._gp.fit(self._space.params, self._space.target)
util = UtilityFunction( util = UtilityFunction(
kind=self.utility, kappa=self.kappa, xi=self.xi) kind=self._utility, kappa=self._kappa, xi=self._xi)
results = acq_max( results = acq_max(
f_acq=util.utility, f_acq=util.utility,
...@@ -124,17 +141,13 @@ class GPTuner(Tuner): ...@@ -124,17 +141,13 @@ class GPTuner(Tuner):
return results return results
def receive_trial_result(self, parameter_id, parameters, value, **kwargs): def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
"""Tuner receive result from trial. """
Method invoked when a trial reports its final result.
Parameters
---------- Override of the abstract method in :class:`~nni.tuner.Tuner`.
parameter_id : int
parameters : dict
value : dict/float
if value is dict, it should have "default" key.
""" """
value = extract_scalar_reward(value) value = extract_scalar_reward(value)
if self.optimize_mode == OptimizeMode.Minimize: if self._optimize_mode == OptimizeMode.Minimize:
value = -value value = -value
logger.info("Received trial result.") logger.info("Received trial result.")
...@@ -143,26 +156,27 @@ class GPTuner(Tuner): ...@@ -143,26 +156,27 @@ class GPTuner(Tuner):
self._space.register(parameters, value) self._space.register(parameters, value)
def import_data(self, data): def import_data(self, data):
"""Import additional data for tuning """
Parameters Import additional data for tuning.
----------
data: Override of the abstract method in :class:`~nni.tuner.Tuner`.
a list of dictionarys, each of which has at least two keys, 'parameter' and 'value'
""" """
_completed_num = 0 _completed_num = 0
for trial_info in data: for trial_info in data:
logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) logger.info(
"Importing data, current processing progress %s / %s", _completed_num, len(data))
_completed_num += 1 _completed_num += 1
assert "parameter" in trial_info assert "parameter" in trial_info
_params = trial_info["parameter"] _params = trial_info["parameter"]
assert "value" in trial_info assert "value" in trial_info
_value = trial_info['value'] _value = trial_info['value']
if not _value: if not _value:
logger.info("Useless trial data, value is %s, skip this trial data.", _value) logger.info(
"Useless trial data, value is %s, skip this trial data.", _value)
continue continue
self.supplement_data_num += 1 self._supplement_data_num += 1
_parameter_id = '_'.join( _parameter_id = '_'.join(
["ImportData", str(self.supplement_data_num)]) ["ImportData", str(self._supplement_data_num)])
self.receive_trial_result( self.receive_trial_result(
parameter_id=_parameter_id, parameters=_params, value=_value) parameter_id=_parameter_id, parameters=_params, value=_value)
logger.info("Successfully import data to GP tuner.") logger.info("Successfully import data to GP tuner.")
src/sdk/pynni/nni/gp_tuner/target_space.py
View file @ 751445d3
...@@ -17,39 +17,51 @@ ...@@ -17,39 +17,51 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
''' """
target_space.py Tool class to hold the param-space coordinates (X) and target values (Y).
''' """
import numpy as np import numpy as np
import nni.parameter_expressions as parameter_expressions import nni.parameter_expressions as parameter_expressions
def _hashable(params): def _hashable(params):
""" ensure that an point is hashable by a python dict """ """
Transform list params to tuple format. Ensure that an point is hashable by a python dict.
Parameters
----------
params : numpy array
array format of parameters
Returns
-------
tuple
tuple format of parameters
"""
return tuple(map(float, params)) return tuple(map(float, params))
class TargetSpace(): class TargetSpace():
""" """
Holds the param-space coordinates (X) and target values (Y) Holds the param-space coordinates (X) and target values (Y)
Parameters
----------
pbounds : dict
Dictionary with parameters names and legal values.
random_state : int, RandomState, or None
optionally specify a seed for a random number generator, by default None.
""" """
def __init__(self, pbounds, random_state=None): def __init__(self, pbounds, random_state=None):
""" self._random_state = random_state
Parameters
----------
pbounds : dict
Dictionary with parameters names as keys and a tuple with minimum
and maximum values.
random_state : int, RandomState, or None
optionally specify a seed for a random number generator
"""
self.random_state = random_state
# Get the name of the parameters # Get the name of the parameters
self._keys = sorted(pbounds) self._keys = sorted(pbounds)
# Create an array with parameters bounds # Create an array with parameters bounds
self._bounds = np.array( self._bounds = np.array(
[item[1] for item in sorted(pbounds.items(), key=lambda x: x[0])] [item[1] for item in sorted(pbounds.items(), key=lambda x: x[0])]
...@@ -71,54 +83,100 @@ class TargetSpace(): ...@@ -71,54 +83,100 @@ class TargetSpace():
self._cache = {} self._cache = {}
def __contains__(self, params): def __contains__(self, params):
''' """
check if a parameter is already registered check if a parameter is already registered
'''
Parameters
----------
params : numpy array
Returns
-------
bool
True if the parameter is already registered, else false
"""
return _hashable(params) in self._cache return _hashable(params) in self._cache
def len(self): def len(self):
''' """
length of registered params and targets length of registered params and targets
'''
Returns
-------
int
"""
assert len(self._params) == len(self._target) assert len(self._params) == len(self._target)
return len(self._target) return len(self._target)
@property @property
def params(self): def params(self):
''' """
params: numpy array registered parameters
'''
Returns
-------
numpy array
"""
return self._params return self._params
@property @property
def target(self): def target(self):
''' """
target: numpy array registered target values
'''
Returns
-------
numpy array
"""
return self._target return self._target
@property @property
def dim(self): def dim(self):
''' """
dim: int dimension of parameters
length of keys
''' Returns
-------
int
"""
return len(self._keys) return len(self._keys)
@property @property
def keys(self): def keys(self):
''' """
keys: numpy array keys of parameters
'''
Returns
-------
numpy array
"""
return self._keys return self._keys
@property @property
def bounds(self): def bounds(self):
'''bounds''' """
bounds of parameters
Returns
-------
numpy array
"""
return self._bounds return self._bounds
def params_to_array(self, params): def params_to_array(self, params):
''' dict to array ''' """
dict to array
Parameters
----------
params : dict
dict format of parameters
Returns
-------
numpy array
array format of parameters
"""
try: try:
assert set(params) == set(self.keys) assert set(params) == set(self.keys)
except AssertionError: except AssertionError:
...@@ -129,11 +187,20 @@ class TargetSpace(): ...@@ -129,11 +187,20 @@ class TargetSpace():
return np.asarray([params[key] for key in self.keys]) return np.asarray([params[key] for key in self.keys])
def array_to_params(self, x): def array_to_params(self, x):
''' """
array to dict array to dict
maintain int type if the paramters is defined as int in search_space.json maintain int type if the paramters is defined as int in search_space.json
''' Parameters
----------
x : numpy array
array format of parameters
Returns
-------
dict
dict format of parameters
"""
try: try:
assert len(x) == len(self.keys) assert len(x) == len(self.keys)
except AssertionError: except AssertionError:
...@@ -159,15 +226,15 @@ class TargetSpace(): ...@@ -159,15 +226,15 @@ class TargetSpace():
Parameters Parameters
---------- ----------
x : dict params : dict
parameters
y : float target : float
target function value target function value
""" """
x = self.params_to_array(params) x = self.params_to_array(params)
if x in self: if x in self:
#raise KeyError('Data point {} is not unique'.format(x))
print('Data point {} is not unique'.format(x)) print('Data point {} is not unique'.format(x))
# Insert data into unique dictionary # Insert data into unique dictionary
...@@ -180,32 +247,43 @@ class TargetSpace(): ...@@ -180,32 +247,43 @@ class TargetSpace():
""" """
Creates a random point within the bounds of the space. Creates a random point within the bounds of the space.
Returns
-------
numpy array
one groupe of parameter
""" """
params = np.empty(self.dim) params = np.empty(self.dim)
for col, _bound in enumerate(self._bounds): for col, _bound in enumerate(self._bounds):
if _bound['_type'] == 'choice': if _bound['_type'] == 'choice':
params[col] = parameter_expressions.choice( params[col] = parameter_expressions.choice(
_bound['_value'], self.random_state) _bound['_value'], self._random_state)
elif _bound['_type'] == 'randint': elif _bound['_type'] == 'randint':
params[col] = self.random_state.randint( params[col] = self._random_state.randint(
_bound['_value'][0], _bound['_value'][1], size=1) _bound['_value'][0], _bound['_value'][1], size=1)
elif _bound['_type'] == 'uniform': elif _bound['_type'] == 'uniform':
params[col] = parameter_expressions.uniform( params[col] = parameter_expressions.uniform(
_bound['_value'][0], _bound['_value'][1], self.random_state) _bound['_value'][0], _bound['_value'][1], self._random_state)
elif _bound['_type'] == 'quniform': elif _bound['_type'] == 'quniform':
params[col] = parameter_expressions.quniform( params[col] = parameter_expressions.quniform(
_bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self.random_state) _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state)
elif _bound['_type'] == 'loguniform': elif _bound['_type'] == 'loguniform':
params[col] = parameter_expressions.loguniform( params[col] = parameter_expressions.loguniform(
_bound['_value'][0], _bound['_value'][1], self.random_state) _bound['_value'][0], _bound['_value'][1], self._random_state)
elif _bound['_type'] == 'qloguniform': elif _bound['_type'] == 'qloguniform':
params[col] = parameter_expressions.qloguniform( params[col] = parameter_expressions.qloguniform(
_bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self.random_state) _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state)
return params return params
def max(self): def max(self):
"""Get maximum target value found and corresponding parametes.""" """
Get maximum target value found and its corresponding parameters.
Returns
-------
dict
target value and parameters, empty dict if nothing registered
"""
try: try:
res = { res = {
'target': self.target.max(), 'target': self.target.max(),
...@@ -218,7 +296,14 @@ class TargetSpace(): ...@@ -218,7 +296,14 @@ class TargetSpace():
return res return res
def res(self): def res(self):
"""Get all target values found and corresponding parametes.""" """
Get all target values found and corresponding parameters.
Returns
-------
list
a list of target values and their corresponding parameters
"""
params = [dict(zip(self.keys, p)) for p in self.params] params = [dict(zip(self.keys, p)) for p in self.params]
return [ return [
......
src/sdk/pynni/nni/gp_tuner/util.py
View file @ 751445d3
...@@ -17,9 +17,9 @@ ...@@ -17,9 +17,9 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
''' """
gp_tuner.py utility functions and classes for GPTuner
''' """
import warnings import warnings
import numpy as np import numpy as np
...@@ -28,9 +28,21 @@ from scipy.optimize import minimize ...@@ -28,9 +28,21 @@ from scipy.optimize import minimize
def _match_val_type(vals, bounds): def _match_val_type(vals, bounds):
''' """
Update values in the array, to match their corresponding type Update values in the array, to match their corresponding type, make sure the value is legal.
'''
Parameters
----------
vals : numpy array
values of parameters
bounds : numpy array
list of dictionary which stores parameters names and legal values.
Returns
-------
vals_new : list
The closest legal value to the original value
"""
vals_new = [] vals_new = []
for i, bound in enumerate(bounds): for i, bound in enumerate(bounds):
...@@ -52,32 +64,33 @@ def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points): ...@@ -52,32 +64,33 @@ def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points):
A function to find the maximum of the acquisition function A function to find the maximum of the acquisition function
It uses a combination of random sampling (cheap) and the 'L-BFGS-B' It uses a combination of random sampling (cheap) and the 'L-BFGS-B'
optimization method. First by sampling `n_warmup` (1e5) points at random, optimization method. First by sampling ``num_warmup`` points at random,
and then running L-BFGS-B from `n_iter` (250) random starting points. and then running L-BFGS-B from ``num_starting_points`` random starting points.
Parameters Parameters
---------- ----------
:param f_acq: f_acq : UtilityFunction.utility
The acquisition function object that return its point-wise value. The acquisition function object that return its point-wise value.
:param gp: gp : GaussianProcessRegressor
A gaussian process fitted to the relevant data. A gaussian process fitted to the relevant data.
:param y_max: y_max : float
The current maximum known value of the target function. The current maximum known value of the target function.
:param bounds: bounds : numpy array
The variables bounds to limit the search of the acq max. The variables bounds to limit the search of the acq max.
:param num_warmup: num_warmup : int
number of times to randomly sample the aquisition function number of times to randomly sample the aquisition function
:param num_starting_points: num_starting_points : int
number of times to run scipy.minimize number of times to run scipy.minimize
Returns Returns
------- -------
:return: x_max, The arg max of the acquisition function. numpy array
The parameter which achieves max of the acquisition function.
""" """
# Warm up with random points # Warm up with random points
...@@ -117,36 +130,70 @@ def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points): ...@@ -117,36 +130,70 @@ def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points):
class UtilityFunction(): class UtilityFunction():
""" """
An object to compute the acquisition functions. A class to compute different acquisition function values.
Parameters
----------
kind : string
specification of utility function to use
kappa : float
parameter usedd for 'ucb' acquisition function
xi : float
parameter usedd for 'ei' and 'poi' acquisition function
""" """
def __init__(self, kind, kappa, xi): def __init__(self, kind, kappa, xi):
""" self._kappa = kappa
If UCB is to be used, a constant kappa is needed. self._xi = xi
"""
self.kappa = kappa
self.xi = xi
if kind not in ['ucb', 'ei', 'poi']: if kind not in ['ucb', 'ei', 'poi']:
err = "The utility function " \ err = "The utility function " \
"{} has not been implemented, " \ "{} has not been implemented, " \
"please choose one of ucb, ei, or poi.".format(kind) "please choose one of ucb, ei, or poi.".format(kind)
raise NotImplementedError(err) raise NotImplementedError(err)
self.kind = kind self._kind = kind
def utility(self, x, gp, y_max): def utility(self, x, gp, y_max):
'''return utility function''' """
if self.kind == 'ucb': return utility function
return self._ucb(x, gp, self.kappa)
if self.kind == 'ei': Parameters
return self._ei(x, gp, y_max, self.xi) ----------
if self.kind == 'poi': x : numpy array
return self._poi(x, gp, y_max, self.xi) parameters
gp : GaussianProcessRegressor
y_max : float
maximum target value observed so far
Returns
-------
function
return corresponding function, return None if parameter is illegal
"""
if self._kind == 'ucb':
return self._ucb(x, gp, self._kappa)
if self._kind == 'ei':
return self._ei(x, gp, y_max, self._xi)
if self._kind == 'poi':
return self._poi(x, gp, y_max, self._xi)
return None return None
@staticmethod @staticmethod
def _ucb(x, gp, kappa): def _ucb(x, gp, kappa):
"""
Upper Confidence Bound (UCB) utility function
Parameters
----------
x : numpy array
parameters
gp : GaussianProcessRegressor
kappa : float
Returns
-------
float
"""
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("ignore") warnings.simplefilter("ignore")
mean, std = gp.predict(x, return_std=True) mean, std = gp.predict(x, return_std=True)
...@@ -155,6 +202,22 @@ class UtilityFunction(): ...@@ -155,6 +202,22 @@ class UtilityFunction():
@staticmethod @staticmethod
def _ei(x, gp, y_max, xi): def _ei(x, gp, y_max, xi):
"""
Expected Improvement (EI) utility function
Parameters
----------
x : numpy array
parameters
gp : GaussianProcessRegressor
y_max : float
maximum target value observed so far
xi : float
Returns
-------
float
"""
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("ignore") warnings.simplefilter("ignore")
mean, std = gp.predict(x, return_std=True) mean, std = gp.predict(x, return_std=True)
...@@ -164,6 +227,22 @@ class UtilityFunction(): ...@@ -164,6 +227,22 @@ class UtilityFunction():
@staticmethod @staticmethod
def _poi(x, gp, y_max, xi): def _poi(x, gp, y_max, xi):
"""
Possibility Of Improvement (POI) utility function
Parameters
----------
x : numpy array
parameters
gp : GaussianProcessRegressor
y_max : float
maximum target value observed so far
xi : float
Returns
-------
float
"""
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("ignore") warnings.simplefilter("ignore")
mean, std = gp.predict(x, return_std=True) mean, std = gp.predict(x, return_std=True)
......
src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py
View file @ 751445d3
...@@ -27,21 +27,21 @@ class MedianstopAssessor(Assessor): ...@@ -27,21 +27,21 @@ class MedianstopAssessor(Assessor):
Parameters Parameters
---------- ----------
optimize_mode: str optimize_mode : str
optimize mode, 'maximize' or 'minimize' optimize mode, 'maximize' or 'minimize'
start_step: int start_step : int
only after receiving start_step number of reported intermediate results only after receiving start_step number of reported intermediate results
""" """
def __init__(self, optimize_mode='maximize', start_step=0): def __init__(self, optimize_mode='maximize', start_step=0):
self.start_step = start_step self._start_step = start_step
self.running_history = dict() self._running_history = dict()
self.completed_avg_history = dict() self._completed_avg_history = dict()
if optimize_mode == 'maximize': if optimize_mode == 'maximize':
self.high_better = True self._high_better = True
elif optimize_mode == 'minimize': elif optimize_mode == 'minimize':
self.high_better = False self._high_better = False
else: else:
self.high_better = True self._high_better = True
logger.warning('unrecognized optimize_mode %s', optimize_mode) logger.warning('unrecognized optimize_mode %s', optimize_mode)
def _update_data(self, trial_job_id, trial_history): def _update_data(self, trial_job_id, trial_history):
...@@ -49,35 +49,35 @@ class MedianstopAssessor(Assessor): ...@@ -49,35 +49,35 @@ class MedianstopAssessor(Assessor):
Parameters Parameters
---------- ----------
trial_job_id: int trial_job_id : int
trial job id trial job id
trial_history: list trial_history : list
The history performance matrix of each trial The history performance matrix of each trial
""" """
if trial_job_id not in self.running_history: if trial_job_id not in self._running_history:
self.running_history[trial_job_id] = [] self._running_history[trial_job_id] = []
self.running_history[trial_job_id].extend(trial_history[len(self.running_history[trial_job_id]):]) self._running_history[trial_job_id].extend(trial_history[len(self._running_history[trial_job_id]):])
def trial_end(self, trial_job_id, success): def trial_end(self, trial_job_id, success):
"""trial_end """trial_end
Parameters Parameters
---------- ----------
trial_job_id: int trial_job_id : int
trial job id trial job id
success: bool success : bool
True if succssfully finish the experiment, False otherwise True if succssfully finish the experiment, False otherwise
""" """
if trial_job_id in self.running_history: if trial_job_id in self._running_history:
if success: if success:
cnt = 0 cnt = 0
history_sum = 0 history_sum = 0
self.completed_avg_history[trial_job_id] = [] self._completed_avg_history[trial_job_id] = []
for each in self.running_history[trial_job_id]: for each in self._running_history[trial_job_id]:
cnt += 1 cnt += 1
history_sum += each history_sum += each
self.completed_avg_history[trial_job_id].append(history_sum / cnt) self._completed_avg_history[trial_job_id].append(history_sum / cnt)
self.running_history.pop(trial_job_id) self._running_history.pop(trial_job_id)
else: else:
logger.warning('trial_end: trial_job_id does not exist in running_history') logger.warning('trial_end: trial_job_id does not exist in running_history')
...@@ -86,9 +86,9 @@ class MedianstopAssessor(Assessor): ...@@ -86,9 +86,9 @@ class MedianstopAssessor(Assessor):
Parameters Parameters
---------- ----------
trial_job_id: int trial_job_id : int
trial job id trial job id
trial_history: list trial_history : list
The history performance matrix of each trial The history performance matrix of each trial
Returns Returns
...@@ -102,7 +102,7 @@ class MedianstopAssessor(Assessor): ...@@ -102,7 +102,7 @@ class MedianstopAssessor(Assessor):
unrecognize exception in medianstop_assessor unrecognize exception in medianstop_assessor
""" """
curr_step = len(trial_history) curr_step = len(trial_history)
if curr_step < self.start_step: if curr_step < self._start_step:
return AssessResult.Good return AssessResult.Good
try: try:
...@@ -115,18 +115,18 @@ class MedianstopAssessor(Assessor): ...@@ -115,18 +115,18 @@ class MedianstopAssessor(Assessor):
logger.exception(error) logger.exception(error)
self._update_data(trial_job_id, num_trial_history) self._update_data(trial_job_id, num_trial_history)
if self.high_better: if self._high_better:
best_history = max(trial_history) best_history = max(trial_history)
else: else:
best_history = min(trial_history) best_history = min(trial_history)
avg_array = [] avg_array = []
for id_ in self.completed_avg_history: for id_ in self._completed_avg_history:
if len(self.completed_avg_history[id_]) >= curr_step: if len(self._completed_avg_history[id_]) >= curr_step:
avg_array.append(self.completed_avg_history[id_][curr_step - 1]) avg_array.append(self._completed_avg_history[id_][curr_step - 1])
if avg_array: if avg_array:
avg_array.sort() avg_array.sort()
if self.high_better: if self._high_better:
median = avg_array[(len(avg_array)-1) // 2] median = avg_array[(len(avg_array)-1) // 2]
return AssessResult.Bad if best_history < median else AssessResult.Good return AssessResult.Bad if best_history < median else AssessResult.Good
else: else:
......
src/sdk/pynni/nni/tuner.py
View file @ 751445d3
...@@ -79,7 +79,8 @@ class Tuner(Recoverable): ...@@ -79,7 +79,8 @@ class Tuner(Recoverable):
:class:`~nni.smac_tuner.smac_tuner.SMACTuner` :class:`~nni.smac_tuner.smac_tuner.SMACTuner`
:class:`~nni.gridsearch_tuner.gridsearch_tuner.GridSearchTuner` :class:`~nni.gridsearch_tuner.gridsearch_tuner.GridSearchTuner`
:class:`~nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner` :class:`~nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner`
:class:`~nni.metis_tuner.mets_tuner.MetisTuner` :class:`~nni.metis_tuner.metis_tuner.MetisTuner`
:class:`~nni.gp_tuner.gp_tuner.GPTuner`
""" """
def generate_parameters(self, parameter_id, **kwargs): def generate_parameters(self, parameter_id, **kwargs):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment