Skip to content

GitLab

Unverified Commit 416ecd5a authored by Miguel Trejo Marrufo's avatar Miguel Trejo Marrufo Committed by GitHub
Browse files

[python-package] remove 'fobj' in favor of passing custom objective function... 


[python-package] remove 'fobj' in favor of passing custom objective function in params (fixes #3244) (#5052)

* feat: support custom metrics in params

* feat: support objective in params

* test: custom objective and metric

* fix: imports are incorrectly sorted

* feat: convert eval metrics str and set to list

* feat: convert single callable eval_metric to list

* test: single callable objective in params
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* feat: callable fobj in basic cv function
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* test: cv support objective callable
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* fix: assert in cv_res
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* docs: objective callable in params
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* recover test_boost_from_average_with_single_leaf_trees
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* linters fail
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* remove metrics helper functions
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* feat: choose objective through _choose_param_values
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* test: test objective through _choose_param_values
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* test: test objective is callabe in train
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* test: parametrize choose_param_value with objective aliases
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* test: cv booster metric is none
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* fix: if string and callable choose callable
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* test train uses custom objective metrics
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* test: cv uses custom objective metrics
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* refactor: remove fobj parameter in train and cv
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* refactor: objective through params in sklearn API
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* custom objective function in advanced_example
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* fix whitespackes lint

* objective is none not a particular case for predict method
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* replace scipy.expit with custom implementation
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* test: set num_boost_round value to 20
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* fix: custom objective default_value is none
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* refactor: remove self._fobj
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* custom_objective default value is None
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* refactor: variables name reference dummy_obj
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* linter errors

* fix: process objective parameter when calling predict
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>

* linter errors

* fix: objective is None during predict call
Signed-off-by: default avatarMiguel Trejo <armando.trejo.marrufo@gmail.com>
parent fc0c8fd4
Show whitespace changes
Inline Side-by-side
Showing with 270 additions and 115 deletions
examples/python-guide/advanced_example.py
View file @ 416ecd5a
# coding: utf-8
import copy
import json
import pickle
from pathlib import Path
......@@ -159,11 +160,14 @@ def binary_error(preds, train_data):
return 'error', np.mean(labels != (preds > 0.5)), False
gbm = lgb.train(params,
# Pass custom objective function through params
params_custom_obj = copy.deepcopy(params)
params_custom_obj['objective'] = loglikelihood
gbm = lgb.train(params_custom_obj,
lgb_train,
num_boost_round=10,
init_model=gbm,
fobj=loglikelihood,
feval=binary_error,
valid_sets=lgb_eval)
......@@ -183,11 +187,14 @@ def accuracy(preds, train_data):
return 'accuracy', np.mean(labels == (preds > 0.5)), True
gbm = lgb.train(params,
# Pass custom objective function through params
params_custom_obj = copy.deepcopy(params)
params_custom_obj['objective'] = loglikelihood
gbm = lgb.train(params_custom_obj,
lgb_train,
num_boost_round=10,
init_model=gbm,
fobj=loglikelihood,
feval=[binary_error, accuracy],
valid_sets=lgb_eval)
......
python-package/lightgbm/basic.py
View file @ 416ecd5a
......@@ -3185,7 +3185,7 @@ class Booster:
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
The predicted values.
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
If ``fobj`` is specified, predicted values are returned before any transformation,
If custom objective function is used, predicted values are returned before any transformation,
e.g. they are raw margin instead of probability of positive class for binary task in this case.
eval_data : Dataset
A ``Dataset`` to evaluate.
......@@ -3231,7 +3231,7 @@ class Booster:
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
The predicted values.
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
If ``fobj`` is specified, predicted values are returned before any transformation,
If custom objective function is used, predicted values are returned before any transformation,
e.g. they are raw margin instead of probability of positive class for binary task in this case.
eval_data : Dataset
The training dataset.
......@@ -3262,7 +3262,7 @@ class Booster:
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
The predicted values.
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
If ``fobj`` is specified, predicted values are returned before any transformation,
If custom objective function is used, predicted values are returned before any transformation,
e.g. they are raw margin instead of probability of positive class for binary task in this case.
eval_data : Dataset
The validation dataset.
......
python-package/lightgbm/engine.py
View file @ 416ecd5a
......@@ -12,10 +12,6 @@ from . import callback
from .basic import Booster, Dataset, LightGBMError, _choose_param_value, _ConfigAliases, _InnerPredictor, _log_warning
from .compat import SKLEARN_INSTALLED, _LGBMGroupKFold, _LGBMStratifiedKFold
_LGBM_CustomObjectiveFunction = Callable[
[np.ndarray, Dataset],
Tuple[np.ndarray, np.ndarray]
]
_LGBM_CustomMetricFunction = Callable[
[np.ndarray, Dataset],
Tuple[str, float, bool]
......@@ -28,7 +24,6 @@ def train(
num_boost_round: int = 100,
valid_sets: Optional[List[Dataset]] = None,
valid_names: Optional[List[str]] = None,
fobj: Optional[_LGBM_CustomObjectiveFunction] = None,
feval: Optional[Union[_LGBM_CustomMetricFunction, List[_LGBM_CustomMetricFunction]]] = None,
init_model: Optional[Union[str, Path, Booster]] = None,
feature_name: Union[List[str], str] = 'auto',
......@@ -41,7 +36,8 @@ def train(
Parameters
----------
params : dict
Parameters for training.
Parameters for training. Values passed through ``params`` take precedence over those
supplied via arguments.
train_set : Dataset
Data to be trained on.
num_boost_round : int, optional (default=100)
......@@ -50,27 +46,6 @@ def train(
List of data to be evaluated on during training.
valid_names : list of str, or None, optional (default=None)
Names of ``valid_sets``.
fobj : callable or None, optional (default=None)
Customized objective function.
Should accept two parameters: preds, train_data,
and return (grad, hess).
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
The predicted values.
Predicted values are returned before any transformation,
e.g. they are raw margin instead of probability of positive class for binary task.
train_data : Dataset
The training dataset.
grad : numpy 1-D array or numpy 2-D array (for multi-class task)
The value of the first order derivative (gradient) of the loss
with respect to the elements of preds for each sample point.
hess : numpy 1-D array or numpy 2-D array (for multi-class task)
The value of the second order derivative (Hessian) of the loss
with respect to the elements of preds for each sample point.
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
and grad and hess should be returned in the same format.
feval : callable, list of callable, or None, optional (default=None)
Customized evaluation function.
Each evaluation function should accept two parameters: preds, eval_data,
......@@ -79,7 +54,7 @@ def train(
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
The predicted values.
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
If ``fobj`` is specified, predicted values are returned before any transformation,
If custom objective function is used, predicted values are returned before any transformation,
e.g. they are raw margin instead of probability of positive class for binary task in this case.
eval_data : Dataset
A ``Dataset`` to evaluate.
......@@ -118,6 +93,27 @@ def train(
List of callback functions that are applied at each iteration.
See Callbacks in Python API for more information.
Note
----
A custom objective function can be provided for the ``objective`` parameter.
It should accept two parameters: preds, train_data and return (grad, hess).
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
The predicted values.
Predicted values are returned before any transformation,
e.g. they are raw margin instead of probability of positive class for binary task.
train_data : Dataset
The training dataset.
grad : numpy 1-D array or numpy 2-D array (for multi-class task)
The value of the first order derivative (gradient) of the loss
with respect to the elements of preds for each sample point.
hess : numpy 1-D array or numpy 2-D array (for multi-class task)
The value of the second order derivative (Hessian) of the loss
with respect to the elements of preds for each sample point.
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
and grad and hess should be returned in the same format.
Returns
-------
booster : Booster
......@@ -125,10 +121,15 @@ def train(
"""
# create predictor first
params = copy.deepcopy(params)
if fobj is not None:
for obj_alias in _ConfigAliases.get("objective"):
params.pop(obj_alias, None)
params['objective'] = 'none'
params = _choose_param_value(
main_param_name='objective',
params=params,
default_value=None
)
fobj = None
if callable(params["objective"]):
fobj = params["objective"]
params["objective"] = 'none'
for alias in _ConfigAliases.get("num_iterations"):
if alias in params:
num_boost_round = params.pop(alias)
......@@ -374,7 +375,7 @@ def _agg_cv_result(raw_results):
def cv(params, train_set, num_boost_round=100,
folds=None, nfold=5, stratified=True, shuffle=True,
metrics=None, fobj=None, feval=None, init_model=None,
metrics=None, feval=None, init_model=None,
feature_name='auto', categorical_feature='auto',
fpreproc=None, seed=0, callbacks=None, eval_train_metric=False,
return_cvbooster=False):
......@@ -383,7 +384,8 @@ def cv(params, train_set, num_boost_round=100,
Parameters
----------
params : dict
Parameters for Booster.
Parameters for training. Values passed through ``params`` take precedence over those
supplied via arguments.
train_set : Dataset
Data to be trained on.
num_boost_round : int, optional (default=100)
......@@ -403,27 +405,6 @@ def cv(params, train_set, num_boost_round=100,
metrics : str, list of str, or None, optional (default=None)
Evaluation metrics to be monitored while CV.
If not None, the metric in ``params`` will be overridden.
fobj : callable or None, optional (default=None)
Customized objective function.
Should accept two parameters: preds, train_data,
and return (grad, hess).
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
The predicted values.
Predicted values are returned before any transformation,
e.g. they are raw margin instead of probability of positive class for binary task.
train_data : Dataset
The training dataset.
grad : numpy 1-D array or numpy 2-D array (for multi-class task)
The value of the first order derivative (gradient) of the loss
with respect to the elements of preds for each sample point.
hess : numpy 1-D array or numpy 2-D array (for multi-class task)
The value of the second order derivative (Hessian) of the loss
with respect to the elements of preds for each sample point.
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
and grad and hess should be returned in the same format.
feval : callable, list of callable, or None, optional (default=None)
Customized evaluation function.
Each evaluation function should accept two parameters: preds, eval_data,
......@@ -432,7 +413,7 @@ def cv(params, train_set, num_boost_round=100,
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
The predicted values.
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
If ``fobj`` is specified, predicted values are returned before any transformation,
If custom objective function is used, predicted values are returned before any transformation,
e.g. they are raw margin instead of probability of positive class for binary task in this case.
eval_data : Dataset
A ``Dataset`` to evaluate.
......@@ -474,6 +455,27 @@ def cv(params, train_set, num_boost_round=100,
return_cvbooster : bool, optional (default=False)
Whether to return Booster models trained on each fold through ``CVBooster``.
Note
----
A custom objective function can be provided for the ``objective`` parameter.
It should accept two parameters: preds, train_data and return (grad, hess).
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
The predicted values.
Predicted values are returned before any transformation,
e.g. they are raw margin instead of probability of positive class for binary task.
train_data : Dataset
The training dataset.
grad : numpy 1-D array or numpy 2-D array (for multi-class task)
The value of the first order derivative (gradient) of the loss
with respect to the elements of preds for each sample point.
hess : numpy 1-D array or numpy 2-D array (for multi-class task)
The value of the second order derivative (Hessian) of the loss
with respect to the elements of preds for each sample point.
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
and grad and hess should be returned in the same format.
Returns
-------
eval_hist : dict
......@@ -486,12 +488,16 @@ def cv(params, train_set, num_boost_round=100,
"""
if not isinstance(train_set, Dataset):
raise TypeError("Training only accepts Dataset object")
params = copy.deepcopy(params)
if fobj is not None:
for obj_alias in _ConfigAliases.get("objective"):
params.pop(obj_alias, None)
params['objective'] = 'none'
params = _choose_param_value(
main_param_name='objective',
params=params,
default_value=None
)
fobj = None
if callable(params["objective"]):
fobj = params["objective"]
params["objective"] = 'none'
for alias in _ConfigAliases.get("num_iterations"):
if alias in params:
_log_warning(f"Found '{alias}' in params. Will use it instead of 'num_boost_round' argument")
......
python-package/lightgbm/sklearn.py
View file @ 416ecd5a
......@@ -596,11 +596,10 @@ class LGBMModel(_LGBMModelBase):
raise ValueError("Unknown LGBMModel type.")
if callable(self._objective):
if stage == "fit":
self._fobj = _ObjectiveFunctionWrapper(self._objective)
params['objective'] = 'None' # objective = nullptr for unknown objective
params['objective'] = _ObjectiveFunctionWrapper(self._objective)
else:
params['objective'] = 'None'
else:
if stage == "fit":
self._fobj = None
params['objective'] = self._objective
params.pop('importance_type', None)
......@@ -756,7 +755,6 @@ class LGBMModel(_LGBMModelBase):
num_boost_round=self.n_estimators,
valid_sets=valid_sets,
valid_names=eval_names,
fobj=self._fobj,
feval=eval_metrics_callable,
init_model=init_model,
feature_name=feature_name,
......
tests/python_package_test/test_basic.py
View file @ 416ecd5a
......@@ -14,7 +14,7 @@ from sklearn.model_selection import train_test_split
import lightgbm as lgb
from lightgbm.compat import PANDAS_INSTALLED, pd_DataFrame, pd_Series
from .utils import load_breast_cancer
from .utils import dummy_obj, load_breast_cancer, mse_obj
def test_basic(tmp_path):
......@@ -513,6 +513,36 @@ def test_choose_param_value():
assert original_params == expected_params
@pytest.mark.parametrize("objective_alias", lgb.basic._ConfigAliases.get("objective"))
def test_choose_param_value_objective(objective_alias):
# If callable is found in objective
params = {objective_alias: dummy_obj}
params = lgb.basic._choose_param_value(
main_param_name="objective",
params=params,
default_value=None
)
assert params['objective'] == dummy_obj
# Value in params should be preferred to the default_value passed from keyword arguments
params = {objective_alias: dummy_obj}
params = lgb.basic._choose_param_value(
main_param_name="objective",
params=params,
default_value=mse_obj
)
assert params['objective'] == dummy_obj
# None of objective or its aliases in params, but default_value is callable.
params = {}
params = lgb.basic._choose_param_value(
main_param_name="objective",
params=params,
default_value=mse_obj
)
assert params['objective'] == mse_obj
@pytest.mark.parametrize('collection', ['1d_np', '2d_np', 'pd_float', 'pd_str', '1d_list', '2d_list'])
@pytest.mark.parametrize('dtype', [np.float32, np.float64])
def test_list_to_1d_numpy(collection, dtype):
......
tests/python_package_test/test_engine.py
View file @ 416ecd5a
......@@ -19,14 +19,18 @@ from sklearn.model_selection import GroupKFold, TimeSeriesSplit, train_test_spli
import lightgbm as lgb
from .utils import (load_boston, load_breast_cancer, load_digits, load_iris, make_synthetic_regression,
sklearn_multiclass_custom_objective, softmax)
from .utils import (dummy_obj, load_boston, load_breast_cancer, load_digits, load_iris, logistic_sigmoid,
make_synthetic_regression, mse_obj, sklearn_multiclass_custom_objective, softmax)
decreasing_generator = itertools.count(0, -1)
def dummy_obj(preds, train_data):
return np.ones(preds.shape), np.ones(preds.shape)
def logloss_obj(preds, train_data):
y_true = train_data.get_label()
y_pred = logistic_sigmoid(preds)
grad = y_pred - y_true
hess = y_pred * (1.0 - y_pred)
return grad, hess
def multi_logloss(y_true, y_pred):
......@@ -1882,7 +1886,7 @@ def test_metrics():
lgb_valid = lgb.Dataset(X_test, y_test, reference=lgb_train)
evals_result = {}
params_verbose = {'verbose': -1}
params_dummy_obj_verbose = {'verbose': -1, 'objective': dummy_obj}
params_obj_verbose = {'objective': 'binary', 'verbose': -1}
params_obj_metric_log_verbose = {'objective': 'binary', 'metric': 'binary_logloss', 'verbose': -1}
params_obj_metric_err_verbose = {'objective': 'binary', 'metric': 'binary_error', 'verbose': -1}
......@@ -1891,11 +1895,11 @@ def test_metrics():
'metric': ['binary_logloss', 'binary_error'],
'verbose': -1}
params_obj_metric_none_verbose = {'objective': 'binary', 'metric': 'None', 'verbose': -1}
params_metric_log_verbose = {'metric': 'binary_logloss', 'verbose': -1}
params_metric_err_verbose = {'metric': 'binary_error', 'verbose': -1}
params_metric_inv_verbose = {'metric_types': 'invalid_metric', 'verbose': -1}
params_metric_multi_verbose = {'metric': ['binary_logloss', 'binary_error'], 'verbose': -1}
params_metric_none_verbose = {'metric': 'None', 'verbose': -1}
params_dummy_obj_metric_log_verbose = {'objective': dummy_obj, 'metric': 'binary_logloss', 'verbose': -1}
params_dummy_obj_metric_err_verbose = {'metric': 'binary_error', 'objective': dummy_obj, 'verbose': -1}
params_dummy_obj_metric_inv_verbose = {'objective': dummy_obj, 'metric_types': 'invalid_metric', 'verbose': -1}
params_dummy_obj_metric_multi_verbose = {'objective': dummy_obj, 'metric': ['binary_logloss', 'binary_error'], 'verbose': -1}
params_dummy_obj_metric_none_verbose = {'objective': dummy_obj, 'metric': 'None', 'verbose': -1}
def get_cv_result(params=params_obj_verbose, **kwargs):
return lgb.cv(params, lgb_train, num_boost_round=2, **kwargs)
......@@ -1959,32 +1963,32 @@ def test_metrics():
# fobj, no feval
# no default metric
res = get_cv_result(params=params_verbose, fobj=dummy_obj)
res = get_cv_result(params=params_dummy_obj_verbose)
assert len(res) == 0
# metric in params
res = get_cv_result(params=params_metric_err_verbose, fobj=dummy_obj)
res = get_cv_result(params=params_dummy_obj_metric_err_verbose)
assert len(res) == 2
assert 'valid binary_error-mean' in res
# metric in args
res = get_cv_result(params=params_verbose, fobj=dummy_obj, metrics='binary_error')
res = get_cv_result(params=params_dummy_obj_verbose, metrics='binary_error')
assert len(res) == 2
assert 'valid binary_error-mean' in res
# metric in args overwrites its' alias in params
res = get_cv_result(params=params_metric_inv_verbose, fobj=dummy_obj, metrics='binary_error')
res = get_cv_result(params=params_dummy_obj_metric_inv_verbose, metrics='binary_error')
assert len(res) == 2
assert 'valid binary_error-mean' in res
# multiple metrics in params
res = get_cv_result(params=params_metric_multi_verbose, fobj=dummy_obj)
res = get_cv_result(params=params_dummy_obj_metric_multi_verbose)
assert len(res) == 4
assert 'valid binary_logloss-mean' in res
assert 'valid binary_error-mean' in res
# multiple metrics in args
res = get_cv_result(params=params_verbose, fobj=dummy_obj,
res = get_cv_result(params=params_dummy_obj_verbose,
metrics=['binary_logloss', 'binary_error'])
assert len(res) == 4
assert 'valid binary_logloss-mean' in res
......@@ -2042,39 +2046,39 @@ def test_metrics():
# fobj, feval
# no default metric, only custom one
res = get_cv_result(params=params_verbose, fobj=dummy_obj, feval=constant_metric)
res = get_cv_result(params=params_dummy_obj_verbose, feval=constant_metric)
assert len(res) == 2
assert 'valid error-mean' in res
# metric in params with custom one
res = get_cv_result(params=params_metric_err_verbose, fobj=dummy_obj, feval=constant_metric)
res = get_cv_result(params=params_dummy_obj_metric_err_verbose, feval=constant_metric)
assert len(res) == 4
assert 'valid binary_error-mean' in res
assert 'valid error-mean' in res
# metric in args with custom one
res = get_cv_result(params=params_verbose, fobj=dummy_obj,
res = get_cv_result(params=params_dummy_obj_verbose,
feval=constant_metric, metrics='binary_error')
assert len(res) == 4
assert 'valid binary_error-mean' in res
assert 'valid error-mean' in res
# metric in args overwrites one in params, custom one is evaluated too
res = get_cv_result(params=params_metric_inv_verbose, fobj=dummy_obj,
res = get_cv_result(params=params_dummy_obj_metric_inv_verbose,
feval=constant_metric, metrics='binary_error')
assert len(res) == 4
assert 'valid binary_error-mean' in res
assert 'valid error-mean' in res
# multiple metrics in params with custom one
res = get_cv_result(params=params_metric_multi_verbose, fobj=dummy_obj, feval=constant_metric)
res = get_cv_result(params=params_dummy_obj_metric_multi_verbose, feval=constant_metric)
assert len(res) == 6
assert 'valid binary_logloss-mean' in res
assert 'valid binary_error-mean' in res
assert 'valid error-mean' in res
# multiple metrics in args with custom one
res = get_cv_result(params=params_verbose, fobj=dummy_obj, feval=constant_metric,
res = get_cv_result(params=params_dummy_obj_verbose, feval=constant_metric,
metrics=['binary_logloss', 'binary_error'])
assert len(res) == 6
assert 'valid binary_logloss-mean' in res
......@@ -2082,7 +2086,7 @@ def test_metrics():
assert 'valid error-mean' in res
# custom metric is evaluated despite 'None' is passed
res = get_cv_result(params=params_metric_none_verbose, fobj=dummy_obj, feval=constant_metric)
res = get_cv_result(params=params_dummy_obj_metric_none_verbose, feval=constant_metric)
assert len(res) == 2
assert 'valid error-mean' in res
......@@ -2116,16 +2120,16 @@ def test_metrics():
# fobj, no feval
# no default metric
train_booster(params=params_verbose, fobj=dummy_obj)
train_booster(params=params_dummy_obj_verbose)
assert len(evals_result) == 0
# metric in params
train_booster(params=params_metric_log_verbose, fobj=dummy_obj)
train_booster(params=params_dummy_obj_metric_log_verbose)
assert len(evals_result['valid_0']) == 1
assert 'binary_logloss' in evals_result['valid_0']
# multiple metrics in params
train_booster(params=params_metric_multi_verbose, fobj=dummy_obj)
train_booster(params=params_dummy_obj_metric_multi_verbose)
assert len(evals_result['valid_0']) == 2
assert 'binary_logloss' in evals_result['valid_0']
assert 'binary_error' in evals_result['valid_0']
......@@ -2163,25 +2167,25 @@ def test_metrics():
# fobj, feval
# no default metric, only custom one
train_booster(params=params_verbose, fobj=dummy_obj, feval=constant_metric)
train_booster(params=params_dummy_obj_verbose, feval=constant_metric)
assert len(evals_result['valid_0']) == 1
assert 'error' in evals_result['valid_0']
# metric in params with custom one
train_booster(params=params_metric_log_verbose, fobj=dummy_obj, feval=constant_metric)
train_booster(params=params_dummy_obj_metric_log_verbose, feval=constant_metric)
assert len(evals_result['valid_0']) == 2
assert 'binary_logloss' in evals_result['valid_0']
assert 'error' in evals_result['valid_0']
# multiple metrics in params with custom one
train_booster(params=params_metric_multi_verbose, fobj=dummy_obj, feval=constant_metric)
train_booster(params=params_dummy_obj_metric_multi_verbose, feval=constant_metric)
assert len(evals_result['valid_0']) == 3
assert 'binary_logloss' in evals_result['valid_0']
assert 'binary_error' in evals_result['valid_0']
assert 'error' in evals_result['valid_0']
# custom metric is evaluated despite 'None' is passed
train_booster(params=params_metric_none_verbose, fobj=dummy_obj, feval=constant_metric)
train_booster(params=params_dummy_obj_metric_none_verbose, feval=constant_metric)
assert len(evals_result) == 1
assert 'error' in evals_result['valid_0']
......@@ -2190,9 +2194,12 @@ def test_metrics():
obj_multi_aliases = ['multiclass', 'softmax', 'multiclassova', 'multiclass_ova', 'ova', 'ovr']
for obj_multi_alias in obj_multi_aliases:
# Custom objective replaces multiclass
params_obj_class_3_verbose = {'objective': obj_multi_alias, 'num_class': 3, 'verbose': -1}
params_obj_class_1_verbose = {'objective': obj_multi_alias, 'num_class': 1, 'verbose': -1}
params_dummy_obj_class_3_verbose = {'objective': dummy_obj, 'num_class': 3, 'verbose': -1}
params_dummy_obj_class_1_verbose = {'objective': dummy_obj, 'num_class': 1, 'verbose': -1}
params_obj_verbose = {'objective': obj_multi_alias, 'verbose': -1}
params_dummy_obj_verbose = {'objective': dummy_obj, 'verbose': -1}
# multiclass default metric
res = get_cv_result(params_obj_class_3_verbose)
assert len(res) == 2
......@@ -2203,20 +2210,20 @@ def test_metrics():
assert 'valid multi_logloss-mean' in res
assert 'valid error-mean' in res
# multiclass metric alias with custom one for custom objective
res = get_cv_result(params_obj_class_3_verbose, fobj=dummy_obj, feval=constant_metric)
res = get_cv_result(params_dummy_obj_class_3_verbose, feval=constant_metric)
assert len(res) == 2
assert 'valid error-mean' in res
# no metric for invalid class_num
res = get_cv_result(params_obj_class_1_verbose, fobj=dummy_obj)
res = get_cv_result(params_dummy_obj_class_1_verbose)
assert len(res) == 0
# custom metric for invalid class_num
res = get_cv_result(params_obj_class_1_verbose, fobj=dummy_obj, feval=constant_metric)
res = get_cv_result(params_dummy_obj_class_1_verbose, feval=constant_metric)
assert len(res) == 2
assert 'valid error-mean' in res
# multiclass metric alias with custom one with invalid class_num
with pytest.raises(lgb.basic.LightGBMError):
get_cv_result(params_obj_class_1_verbose, metrics=obj_multi_alias,
fobj=dummy_obj, feval=constant_metric)
get_cv_result(params_dummy_obj_class_1_verbose, metrics=obj_multi_alias,
feval=constant_metric)
# multiclass default metric without num_class
with pytest.raises(lgb.basic.LightGBMError):
get_cv_result(params_obj_verbose)
......@@ -2237,20 +2244,20 @@ def test_metrics():
with pytest.raises(lgb.basic.LightGBMError):
get_cv_result(params_class_3_verbose)
# no metric with non-default num_class for custom objective
res = get_cv_result(params_class_3_verbose, fobj=dummy_obj)
res = get_cv_result(params_dummy_obj_class_3_verbose)
assert len(res) == 0
for metric_multi_alias in obj_multi_aliases + ['multi_logloss']:
# multiclass metric alias for custom objective
res = get_cv_result(params_class_3_verbose, metrics=metric_multi_alias, fobj=dummy_obj)
res = get_cv_result(params_dummy_obj_class_3_verbose, metrics=metric_multi_alias)
assert len(res) == 2
assert 'valid multi_logloss-mean' in res
# multiclass metric for custom objective
res = get_cv_result(params_class_3_verbose, metrics='multi_error', fobj=dummy_obj)
res = get_cv_result(params_dummy_obj_class_3_verbose, metrics='multi_error')
assert len(res) == 2
assert 'valid multi_error-mean' in res
# binary metric with non-default num_class for custom objective
with pytest.raises(lgb.basic.LightGBMError):
get_cv_result(params_class_3_verbose, metrics='binary_error', fobj=dummy_obj)
get_cv_result(params_dummy_obj_class_3_verbose, metrics='binary_error')
def test_multiple_feval_train():
......@@ -2278,6 +2285,97 @@ def test_multiple_feval_train():
assert 'decreasing_metric' in evals_result['valid_0']
def test_objective_callable_train_binary_classification():
X, y = load_breast_cancer(return_X_y=True)
params = {
'verbose': -1,
'objective': logloss_obj,
'learning_rate': 0.01
}
train_dataset = lgb.Dataset(X, y)
booster = lgb.train(
params=params,
train_set=train_dataset,
num_boost_round=20
)
y_pred = logistic_sigmoid(booster.predict(X))
logloss_error = log_loss(y, y_pred)
rocauc_error = roc_auc_score(y, y_pred)
assert booster.params['objective'] == 'none'
assert logloss_error == pytest.approx(0.55, 0.1)
assert rocauc_error == pytest.approx(0.99, 0.5)
def test_objective_callable_train_regression():
X, y = make_synthetic_regression()
params = {
'verbose': -1,
'objective': mse_obj
}
lgb_train = lgb.Dataset(X, y)
booster = lgb.train(
params,
lgb_train,
num_boost_round=20
)
y_pred = booster.predict(X)
mse_error = mean_squared_error(y, y_pred)
assert booster.params['objective'] == 'none'
assert mse_error == pytest.approx(286, 1)
def test_objective_callable_cv_binary_classification():
X, y = load_breast_cancer(return_X_y=True)
params = {
'verbose': -1,
'objective': logloss_obj,
'learning_rate': 0.01
}
train_dataset = lgb.Dataset(X, y)
cv_res = lgb.cv(
params,
train_dataset,
num_boost_round=20,
nfold=3,
return_cvbooster=True
)
cv_booster = cv_res['cvbooster'].boosters
cv_logloss_errors = [
log_loss(y, logistic_sigmoid(cb.predict(X))) < 0.56 for cb in cv_booster
]
cv_objs = [
cb.params['objective'] == 'none' for cb in cv_booster
]
assert all(cv_objs)
assert all(cv_logloss_errors)
def test_objective_callable_cv_regression():
X, y = make_synthetic_regression()
lgb_train = lgb.Dataset(X, y)
params = {
'verbose': -1,
'objective': mse_obj
}
cv_res = lgb.cv(
params,
lgb_train,
num_boost_round=20,
nfold=3,
stratified=False,
return_cvbooster=True
)
cv_booster = cv_res['cvbooster'].boosters
cv_mse_errors = [
mean_squared_error(y, cb.predict(X)) < 463 for cb in cv_booster
]
cv_objs = [
cb.params['objective'] == 'none' for cb in cv_booster
]
assert all(cv_objs)
assert all(cv_mse_errors)
def test_multiple_feval_cv():
X, y = load_breast_cancer(return_X_y=True)
......@@ -2334,7 +2432,8 @@ def test_multiclass_custom_objective():
builtin_obj_bst = lgb.train(params, ds, num_boost_round=10)
builtin_obj_preds = builtin_obj_bst.predict(X)
custom_obj_bst = lgb.train(params, ds, num_boost_round=10, fobj=custom_obj)
params['objective'] = custom_obj
custom_obj_bst = lgb.train(params, ds, num_boost_round=10)
custom_obj_preds = softmax(custom_obj_bst.predict(X))
np.testing.assert_allclose(builtin_obj_preds, custom_obj_preds, rtol=0.01)
......
tests/python_package_test/utils.py
View file @ 416ecd5a
......@@ -119,12 +119,27 @@ def make_synthetic_regression(n_samples=100):
return sklearn.datasets.make_regression(n_samples, n_features=4, n_informative=2, random_state=42)
def dummy_obj(preds, train_data):
return np.ones(preds.shape), np.ones(preds.shape)
def mse_obj(y_pred, dtrain):
y_true = dtrain.get_label()
grad = (y_pred - y_true)
hess = np.ones(len(grad))
return grad, hess
def softmax(x):
row_wise_max = np.max(x, axis=1).reshape(-1, 1)
exp_x = np.exp(x - row_wise_max)
return exp_x / np.sum(exp_x, axis=1).reshape(-1, 1)
def logistic_sigmoid(x):
return 1.0 / (1.0 + np.exp(-x))
def sklearn_multiclass_custom_objective(y_true, y_pred):
num_rows, num_class = y_pred.shape
prob = softmax(y_pred)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment