Skip to content

GitLab

"docs/en_US/WebUI.md" did not exist on "efa479b0a91ff4c2b8a0f4cbb58c4f48a3747d47"
Unverified Commit 751445d3 authored by Guoxin's avatar Guoxin Committed by GitHub
Browse files

docstr/pylint of GP Tuner & CurveFitting Assessor & MedianStop Assessor (#1692) 

# docstr/pylint of GP Tuner & CurveFitting Assessor & MedianStop Assessor
parent e6df29cf
Show whitespace changes
Inline Side-by-side
Showing with 409 additions and 226 deletions
docs/en_US/sdk_reference.rst
View file @ 751445d3
......@@ -39,6 +39,9 @@ Tuner
.. autoclass:: nni.batch_tuner.batch_tuner.BatchTuner
:members:
.. autoclass:: nni.gp_tuner.gp_tuner.GPTuner
:members:
Assessor
------------------------
.. autoclass:: nni.assessor.Assessor
......
src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py
View file @ 751445d3
......@@ -29,13 +29,13 @@ class CurvefittingAssessor(Assessor):
Parameters
----------
epoch_num: int
epoch_num : int
The total number of epoch
optimize_mode: str
optimize_mode : str
optimize mode, 'maximize' or 'minimize'
start_step: int
start_step : int
only after receiving start_step number of reported intermediate results
threshold: float
threshold : float
The threshold that we decide to early stop the worse performance curve.
"""
def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95, gap=1):
......@@ -70,9 +70,9 @@ class CurvefittingAssessor(Assessor):
Parameters
----------
trial_job_id: int
trial_job_id : int
trial job id
success: bool
success : bool
True if succssfully finish the experiment, False otherwise
"""
if success:
......@@ -90,9 +90,9 @@ class CurvefittingAssessor(Assessor):
Parameters
----------
trial_job_id: int
trial_job_id : int
trial job id
trial_history: list
trial_history : list
The history performance matrix of each trial
Returns
......@@ -105,7 +105,6 @@ class CurvefittingAssessor(Assessor):
Exception
unrecognize exception in curvefitting_assessor
"""
trial_job_id = trial_job_id
self.trial_history = trial_history
if not self.set_best_performance:
return AssessResult.Good
......
src/sdk/pynni/nni/curvefitting_assessor/curvefunctions.py
View file @ 751445d3
......@@ -14,7 +14,9 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
A family of functions used by CurvefittingAssessor
"""
import numpy as np
all_models = {}
......@@ -29,10 +31,10 @@ def vap(x, a, b, c):
Parameters
----------
x: int
a: float
b: float
c: float
x : int
a : float
b : float
c : float
Returns
-------
......@@ -50,10 +52,10 @@ def pow3(x, c, a, alpha):
Parameters
----------
x: int
c: float
a: float
alpha: float
x : int
c : float
a : float
alpha : float
Returns
-------
......@@ -71,9 +73,9 @@ def linear(x, a, b):
Parameters
----------
x: int
a: float
b: float
x : int
a : float
b : float
Returns
-------
......@@ -91,9 +93,9 @@ def logx_linear(x, a, b):
Parameters
----------
x: int
a: float
b: float
x : int
a : float
b : float
Returns
-------
......@@ -112,10 +114,10 @@ def dr_hill_zero_background(x, theta, eta, kappa):
Parameters
----------
x: int
theta: float
eta: float
kappa: float
x : int
theta : float
eta : float
kappa : float
Returns
-------
......@@ -133,10 +135,10 @@ def log_power(x, a, b, c):
Parameters
----------
x: int
a: float
b: float
c: float
x : int
a : float
b : float
c : float
Returns
-------
......@@ -154,11 +156,11 @@ def pow4(x, alpha, a, b, c):
Parameters
----------
x: int
alpha: float
a: float
b: float
c: float
x : int
alpha : float
a : float
b : float
c : float
Returns
-------
......@@ -177,11 +179,11 @@ def mmf(x, alpha, beta, kappa, delta):
Parameters
----------
x: int
alpha: float
beta: float
kappa: float
delta: float
x : int
alpha : float
beta : float
kappa : float
delta : float
Returns
-------
......@@ -199,11 +201,11 @@ def exp4(x, c, a, b, alpha):
Parameters
----------
x: int
c: float
a: float
b: float
alpha: float
x : int
c : float
a : float
b : float
alpha : float
Returns
-------
......@@ -221,9 +223,9 @@ def ilog2(x, c, a):
Parameters
----------
x: int
c: float
a: float
x : int
c : float
a : float
Returns
-------
......@@ -242,11 +244,11 @@ def weibull(x, alpha, beta, kappa, delta):
Parameters
----------
x: int
alpha: float
beta: float
kappa: float
delta: float
x : int
alpha : float
beta : float
kappa : float
delta : float
Returns
-------
......@@ -264,11 +266,11 @@ def janoschek(x, a, beta, k, delta):
Parameters
----------
x: int
a: float
beta: float
k: float
delta: float
x : int
a : float
beta : float
k : float
delta : float
Returns
-------
......
src/sdk/pynni/nni/curvefitting_assessor/model_factory.py
View file @ 751445d3
......@@ -40,7 +40,7 @@ class CurveModel:
Parameters
----------
target_pos: int
target_pos : int
The point we need to predict
"""
def __init__(self, target_pos):
......@@ -120,14 +120,14 @@ class CurveModel:
Parameters
----------
model: string
model : string
name of the curve function model
pos: int
pos : int
the epoch number of the position you want to predict
Returns
-------
int:
int
The expected matrix at pos
"""
if model_para_num[model] == 2:
......@@ -143,9 +143,9 @@ class CurveModel:
Parameters
----------
pos: int
pos : int
the epoch number of the position you want to predict
sample: list
sample : list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns
......@@ -165,7 +165,7 @@ class CurveModel:
Parameters
----------
samples: list
samples : list
a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix,
representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}}
......@@ -187,7 +187,7 @@ class CurveModel:
Parameters
----------
sample: list
sample : list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns
......@@ -206,9 +206,9 @@ class CurveModel:
Parameters
----------
pos: int
pos : int
the epoch number of the position you want to predict
sample: list
sample : list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns
......@@ -225,7 +225,7 @@ class CurveModel:
Parameters
----------
sample: list
sample : list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns
......@@ -244,7 +244,7 @@ class CurveModel:
Parameters
----------
samples: list
samples : list
a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix,
representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}}
......@@ -267,7 +267,7 @@ class CurveModel:
Parameters
----------
samples: list
samples : list
a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix,
representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}}
......@@ -322,7 +322,7 @@ class CurveModel:
Parameters
----------
trial_history: list
trial_history : list
The history performance matrix of each trial.
Returns
......
src/sdk/pynni/nni/gp_tuner/gp_tuner.py
View file @ 751445d3
......@@ -17,9 +17,11 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
'''
gp_tuner.py
'''
"""
GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions.
See :class:`GPTuner` for details.
"""
import warnings
import logging
......@@ -38,18 +40,40 @@ logger = logging.getLogger("GP_Tuner_AutoML")
class GPTuner(Tuner):
'''
GPTuner
'''
"""
GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions.
Parameters
----------
optimize_mode : str
optimize mode, 'maximize' or 'minimize', by default 'maximize'
utility : str
utility function (also called 'acquisition funcition') to use, which can be 'ei', 'ucb' or 'poi'. By default 'ei'.
kappa : float
value used by utility function 'ucb'. The bigger kappa is, the more the tuner will be exploratory. By default 5.
xi : float
used by utility function 'ei' and 'poi'. The bigger xi is, the more the tuner will be exploratory. By default 0.
nu : float
used to specify Matern kernel. The smaller nu, the less smooth the approximated function is. By default 2.5.
alpha : float
Used to specify Gaussian Process Regressor. Larger values correspond to increased noise level in the observations.
By default 1e-6.
cold_start_num : int
Number of random exploration to perform before Gaussian Process. By default 10.
selection_num_warm_up : int
Number of random points to evaluate for getting the point which maximizes the acquisition function. By default 100000
selection_num_starting_points : int
Number of times to run L-BFGS-B from a random starting point after the warmup. By default 250.
"""
def __init__(self, optimize_mode="maximize", utility='ei', kappa=5, xi=0, nu=2.5, alpha=1e-6, cold_start_num=10,
selection_num_warm_up=100000, selection_num_starting_points=250):
self.optimize_mode = OptimizeMode(optimize_mode)
self._optimize_mode = OptimizeMode(optimize_mode)
# utility function related
self.utility = utility
self.kappa = kappa
self.xi = xi
self._utility = utility
self._kappa = kappa
self._xi = xi
# target space
self._space = None
......@@ -72,30 +96,23 @@ class GPTuner(Tuner):
self._selection_num_starting_points = selection_num_starting_points
# num of imported data
self.supplement_data_num = 0
self._supplement_data_num = 0
def update_search_space(self, search_space):
"""Update the self.bounds and self.types by the search_space.json
"""
Update the self.bounds and self.types by the search_space.json file.
Parameters
----------
search_space : dict
Override of the abstract method in :class:`~nni.tuner.Tuner`.
"""
self._space = TargetSpace(search_space, self._random_state)
def generate_parameters(self, parameter_id, **kwargs):
"""Generate next parameter for trial
If the number of trial result is lower than cold start number,
gp will first randomly generate some parameters.
Otherwise, choose the parameters by the Gussian Process Model
Parameters
----------
parameter_id : int
"""
Method which provides one set of hyper-parameters.
If the number of trial result is lower than cold_start_number, GPTuner will first randomly generate some parameters.
Otherwise, choose the parameters by the Gussian Process Model.
Returns
-------
result : dict
Override of the abstract method in :class:`~nni.tuner.Tuner`.
"""
if self._space.len() < self._cold_start_num:
results = self._space.random_sample()
......@@ -107,7 +124,7 @@ class GPTuner(Tuner):
self._gp.fit(self._space.params, self._space.target)
util = UtilityFunction(
kind=self.utility, kappa=self.kappa, xi=self.xi)
kind=self._utility, kappa=self._kappa, xi=self._xi)
results = acq_max(
f_acq=util.utility,
......@@ -124,17 +141,13 @@ class GPTuner(Tuner):
return results
def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
"""Tuner receive result from trial.
"""
Method invoked when a trial reports its final result.
Parameters
----------
parameter_id : int
parameters : dict
value : dict/float
if value is dict, it should have "default" key.
Override of the abstract method in :class:`~nni.tuner.Tuner`.
"""
value = extract_scalar_reward(value)
if self.optimize_mode == OptimizeMode.Minimize:
if self._optimize_mode == OptimizeMode.Minimize:
value = -value
logger.info("Received trial result.")
......@@ -143,26 +156,27 @@ class GPTuner(Tuner):
self._space.register(parameters, value)
def import_data(self, data):
"""Import additional data for tuning
Parameters
----------
data:
a list of dictionarys, each of which has at least two keys, 'parameter' and 'value'
"""
Import additional data for tuning.
Override of the abstract method in :class:`~nni.tuner.Tuner`.
"""
_completed_num = 0
for trial_info in data:
logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data))
logger.info(
"Importing data, current processing progress %s / %s", _completed_num, len(data))
_completed_num += 1
assert "parameter" in trial_info
_params = trial_info["parameter"]
assert "value" in trial_info
_value = trial_info['value']
if not _value:
logger.info("Useless trial data, value is %s, skip this trial data.", _value)
logger.info(
"Useless trial data, value is %s, skip this trial data.", _value)
continue
self.supplement_data_num += 1
self._supplement_data_num += 1
_parameter_id = '_'.join(
["ImportData", str(self.supplement_data_num)])
["ImportData", str(self._supplement_data_num)])
self.receive_trial_result(
parameter_id=_parameter_id, parameters=_params, value=_value)
logger.info("Successfully import data to GP tuner.")
src/sdk/pynni/nni/gp_tuner/target_space.py
View file @ 751445d3
......@@ -17,39 +17,51 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
'''
target_space.py
'''
"""
Tool class to hold the param-space coordinates (X) and target values (Y).
"""
import numpy as np
import nni.parameter_expressions as parameter_expressions
def _hashable(params):
""" ensure that an point is hashable by a python dict """
"""
Transform list params to tuple format. Ensure that an point is hashable by a python dict.
Parameters
----------
params : numpy array
array format of parameters
Returns
-------
tuple
tuple format of parameters
"""
return tuple(map(float, params))
class TargetSpace():
"""
Holds the param-space coordinates (X) and target values (Y)
"""
def __init__(self, pbounds, random_state=None):
"""
Parameters
----------
pbounds : dict
Dictionary with parameters names as keys and a tuple with minimum
and maximum values.
Dictionary with parameters names and legal values.
random_state : int, RandomState, or None
optionally specify a seed for a random number generator
optionally specify a seed for a random number generator, by default None.
"""
self.random_state = random_state
def __init__(self, pbounds, random_state=None):
self._random_state = random_state
# Get the name of the parameters
self._keys = sorted(pbounds)
# Create an array with parameters bounds
self._bounds = np.array(
[item[1] for item in sorted(pbounds.items(), key=lambda x: x[0])]
......@@ -71,54 +83,100 @@ class TargetSpace():
self._cache = {}
def __contains__(self, params):
'''
"""
check if a parameter is already registered
'''
Parameters
----------
params : numpy array
Returns
-------
bool
True if the parameter is already registered, else false
"""
return _hashable(params) in self._cache
def len(self):
'''
"""
length of registered params and targets
'''
Returns
-------
int
"""
assert len(self._params) == len(self._target)
return len(self._target)
@property
def params(self):
'''
params: numpy array
'''
"""
registered parameters
Returns
-------
numpy array
"""
return self._params
@property
def target(self):
'''
target: numpy array
'''
"""
registered target values
Returns
-------
numpy array
"""
return self._target
@property
def dim(self):
'''
dim: int
length of keys
'''
"""
dimension of parameters
Returns
-------
int
"""
return len(self._keys)
@property
def keys(self):
'''
keys: numpy array
'''
"""
keys of parameters
Returns
-------
numpy array
"""
return self._keys
@property
def bounds(self):
'''bounds'''
"""
bounds of parameters
Returns
-------
numpy array
"""
return self._bounds
def params_to_array(self, params):
''' dict to array '''
"""
dict to array
Parameters
----------
params : dict
dict format of parameters
Returns
-------
numpy array
array format of parameters
"""
try:
assert set(params) == set(self.keys)
except AssertionError:
......@@ -129,11 +187,20 @@ class TargetSpace():
return np.asarray([params[key] for key in self.keys])
def array_to_params(self, x):
'''
"""
array to dict
maintain int type if the paramters is defined as int in search_space.json
'''
Parameters
----------
x : numpy array
array format of parameters
Returns
-------
dict
dict format of parameters
"""
try:
assert len(x) == len(self.keys)
except AssertionError:
......@@ -159,15 +226,15 @@ class TargetSpace():
Parameters
----------
x : dict
params : dict
parameters
y : float
target : float
target function value
"""
x = self.params_to_array(params)
if x in self:
#raise KeyError('Data point {} is not unique'.format(x))
print('Data point {} is not unique'.format(x))
# Insert data into unique dictionary
......@@ -180,32 +247,43 @@ class TargetSpace():
"""
Creates a random point within the bounds of the space.
Returns
-------
numpy array
one groupe of parameter
"""
params = np.empty(self.dim)
for col, _bound in enumerate(self._bounds):
if _bound['_type'] == 'choice':
params[col] = parameter_expressions.choice(
_bound['_value'], self.random_state)
_bound['_value'], self._random_state)
elif _bound['_type'] == 'randint':
params[col] = self.random_state.randint(
params[col] = self._random_state.randint(
_bound['_value'][0], _bound['_value'][1], size=1)
elif _bound['_type'] == 'uniform':
params[col] = parameter_expressions.uniform(
_bound['_value'][0], _bound['_value'][1], self.random_state)
_bound['_value'][0], _bound['_value'][1], self._random_state)
elif _bound['_type'] == 'quniform':
params[col] = parameter_expressions.quniform(
_bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self.random_state)
_bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state)
elif _bound['_type'] == 'loguniform':
params[col] = parameter_expressions.loguniform(
_bound['_value'][0], _bound['_value'][1], self.random_state)
_bound['_value'][0], _bound['_value'][1], self._random_state)
elif _bound['_type'] == 'qloguniform':
params[col] = parameter_expressions.qloguniform(
_bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self.random_state)
_bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state)
return params
def max(self):
"""Get maximum target value found and corresponding parametes."""
"""
Get maximum target value found and its corresponding parameters.
Returns
-------
dict
target value and parameters, empty dict if nothing registered
"""
try:
res = {
'target': self.target.max(),
......@@ -218,7 +296,14 @@ class TargetSpace():
return res
def res(self):
"""Get all target values found and corresponding parametes."""
"""
Get all target values found and corresponding parameters.
Returns
-------
list
a list of target values and their corresponding parameters
"""
params = [dict(zip(self.keys, p)) for p in self.params]
return [
......
src/sdk/pynni/nni/gp_tuner/util.py
View file @ 751445d3
......@@ -17,9 +17,9 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
'''
gp_tuner.py
'''
"""
utility functions and classes for GPTuner
"""
import warnings
import numpy as np
......@@ -28,9 +28,21 @@ from scipy.optimize import minimize
def _match_val_type(vals, bounds):
'''
Update values in the array, to match their corresponding type
'''
"""
Update values in the array, to match their corresponding type, make sure the value is legal.
Parameters
----------
vals : numpy array
values of parameters
bounds : numpy array
list of dictionary which stores parameters names and legal values.
Returns
-------
vals_new : list
The closest legal value to the original value
"""
vals_new = []
for i, bound in enumerate(bounds):
......@@ -52,32 +64,33 @@ def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points):
A function to find the maximum of the acquisition function
It uses a combination of random sampling (cheap) and the 'L-BFGS-B'
optimization method. First by sampling `n_warmup` (1e5) points at random,
and then running L-BFGS-B from `n_iter` (250) random starting points.
optimization method. First by sampling ``num_warmup`` points at random,
and then running L-BFGS-B from ``num_starting_points`` random starting points.
Parameters
----------
:param f_acq:
f_acq : UtilityFunction.utility
The acquisition function object that return its point-wise value.
:param gp:
gp : GaussianProcessRegressor
A gaussian process fitted to the relevant data.
:param y_max:
y_max : float
The current maximum known value of the target function.
:param bounds:
bounds : numpy array
The variables bounds to limit the search of the acq max.
:param num_warmup:
num_warmup : int
number of times to randomly sample the aquisition function
:param num_starting_points:
num_starting_points : int
number of times to run scipy.minimize
Returns
-------
:return: x_max, The arg max of the acquisition function.
numpy array
The parameter which achieves max of the acquisition function.
"""
# Warm up with random points
......@@ -117,36 +130,70 @@ def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points):
class UtilityFunction():
"""
An object to compute the acquisition functions.
"""
A class to compute different acquisition function values.
def __init__(self, kind, kappa, xi):
"""
If UCB is to be used, a constant kappa is needed.
Parameters
----------
kind : string
specification of utility function to use
kappa : float
parameter usedd for 'ucb' acquisition function
xi : float
parameter usedd for 'ei' and 'poi' acquisition function
"""
self.kappa = kappa
self.xi = xi
def __init__(self, kind, kappa, xi):
self._kappa = kappa
self._xi = xi
if kind not in ['ucb', 'ei', 'poi']:
err = "The utility function " \
"{} has not been implemented, " \
"please choose one of ucb, ei, or poi.".format(kind)
raise NotImplementedError(err)
self.kind = kind
self._kind = kind
def utility(self, x, gp, y_max):
'''return utility function'''
if self.kind == 'ucb':
return self._ucb(x, gp, self.kappa)
if self.kind == 'ei':
return self._ei(x, gp, y_max, self.xi)
if self.kind == 'poi':
return self._poi(x, gp, y_max, self.xi)
"""
return utility function
Parameters
----------
x : numpy array
parameters
gp : GaussianProcessRegressor
y_max : float
maximum target value observed so far
Returns
-------
function
return corresponding function, return None if parameter is illegal
"""
if self._kind == 'ucb':
return self._ucb(x, gp, self._kappa)
if self._kind == 'ei':
return self._ei(x, gp, y_max, self._xi)
if self._kind == 'poi':
return self._poi(x, gp, y_max, self._xi)
return None
@staticmethod
def _ucb(x, gp, kappa):
"""
Upper Confidence Bound (UCB) utility function
Parameters
----------
x : numpy array
parameters
gp : GaussianProcessRegressor
kappa : float
Returns
-------
float
"""
with warnings.catch_warnings():
warnings.simplefilter("ignore")
mean, std = gp.predict(x, return_std=True)
......@@ -155,6 +202,22 @@ class UtilityFunction():
@staticmethod
def _ei(x, gp, y_max, xi):
"""
Expected Improvement (EI) utility function
Parameters
----------
x : numpy array
parameters
gp : GaussianProcessRegressor
y_max : float
maximum target value observed so far
xi : float
Returns
-------
float
"""
with warnings.catch_warnings():
warnings.simplefilter("ignore")
mean, std = gp.predict(x, return_std=True)
......@@ -164,6 +227,22 @@ class UtilityFunction():
@staticmethod
def _poi(x, gp, y_max, xi):
"""
Possibility Of Improvement (POI) utility function
Parameters
----------
x : numpy array
parameters
gp : GaussianProcessRegressor
y_max : float
maximum target value observed so far
xi : float
Returns
-------
float
"""
with warnings.catch_warnings():
warnings.simplefilter("ignore")
mean, std = gp.predict(x, return_std=True)
......
src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py
View file @ 751445d3
......@@ -27,21 +27,21 @@ class MedianstopAssessor(Assessor):
Parameters
----------
optimize_mode: str
optimize_mode : str
optimize mode, 'maximize' or 'minimize'
start_step: int
start_step : int
only after receiving start_step number of reported intermediate results
"""
def __init__(self, optimize_mode='maximize', start_step=0):
self.start_step = start_step
self.running_history = dict()
self.completed_avg_history = dict()
self._start_step = start_step
self._running_history = dict()
self._completed_avg_history = dict()
if optimize_mode == 'maximize':
self.high_better = True
self._high_better = True
elif optimize_mode == 'minimize':
self.high_better = False
self._high_better = False
else:
self.high_better = True
self._high_better = True
logger.warning('unrecognized optimize_mode %s', optimize_mode)
def _update_data(self, trial_job_id, trial_history):
......@@ -49,35 +49,35 @@ class MedianstopAssessor(Assessor):
Parameters
----------
trial_job_id: int
trial_job_id : int
trial job id
trial_history: list
trial_history : list
The history performance matrix of each trial
"""
if trial_job_id not in self.running_history:
self.running_history[trial_job_id] = []
self.running_history[trial_job_id].extend(trial_history[len(self.running_history[trial_job_id]):])
if trial_job_id not in self._running_history:
self._running_history[trial_job_id] = []
self._running_history[trial_job_id].extend(trial_history[len(self._running_history[trial_job_id]):])
def trial_end(self, trial_job_id, success):
"""trial_end
Parameters
----------
trial_job_id: int
trial_job_id : int
trial job id
success: bool
success : bool
True if succssfully finish the experiment, False otherwise
"""
if trial_job_id in self.running_history:
if trial_job_id in self._running_history:
if success:
cnt = 0
history_sum = 0
self.completed_avg_history[trial_job_id] = []
for each in self.running_history[trial_job_id]:
self._completed_avg_history[trial_job_id] = []
for each in self._running_history[trial_job_id]:
cnt += 1
history_sum += each
self.completed_avg_history[trial_job_id].append(history_sum / cnt)
self.running_history.pop(trial_job_id)
self._completed_avg_history[trial_job_id].append(history_sum / cnt)
self._running_history.pop(trial_job_id)
else:
logger.warning('trial_end: trial_job_id does not exist in running_history')
......@@ -86,9 +86,9 @@ class MedianstopAssessor(Assessor):
Parameters
----------
trial_job_id: int
trial_job_id : int
trial job id
trial_history: list
trial_history : list
The history performance matrix of each trial
Returns
......@@ -102,7 +102,7 @@ class MedianstopAssessor(Assessor):
unrecognize exception in medianstop_assessor
"""
curr_step = len(trial_history)
if curr_step < self.start_step:
if curr_step < self._start_step:
return AssessResult.Good
try:
......@@ -115,18 +115,18 @@ class MedianstopAssessor(Assessor):
logger.exception(error)
self._update_data(trial_job_id, num_trial_history)
if self.high_better:
if self._high_better:
best_history = max(trial_history)
else:
best_history = min(trial_history)
avg_array = []
for id_ in self.completed_avg_history:
if len(self.completed_avg_history[id_]) >= curr_step:
avg_array.append(self.completed_avg_history[id_][curr_step - 1])
for id_ in self._completed_avg_history:
if len(self._completed_avg_history[id_]) >= curr_step:
avg_array.append(self._completed_avg_history[id_][curr_step - 1])
if avg_array:
avg_array.sort()
if self.high_better:
if self._high_better:
median = avg_array[(len(avg_array)-1) // 2]
return AssessResult.Bad if best_history < median else AssessResult.Good
else:
......
src/sdk/pynni/nni/tuner.py
View file @ 751445d3
......@@ -79,7 +79,8 @@ class Tuner(Recoverable):
:class:`~nni.smac_tuner.smac_tuner.SMACTuner`
:class:`~nni.gridsearch_tuner.gridsearch_tuner.GridSearchTuner`
:class:`~nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner`
:class:`~nni.metis_tuner.mets_tuner.MetisTuner`
:class:`~nni.metis_tuner.metis_tuner.MetisTuner`
:class:`~nni.gp_tuner.gp_tuner.GPTuner`
"""
def generate_parameters(self, parameter_id, **kwargs):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment