test_engine.py 170 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
# coding: utf-8
wxchan's avatar
wxchan committed
2
import copy
3
import itertools
4
import json
wxchan's avatar
wxchan committed
5
import math
6
import pickle
7
import platform
8
import random
9
import re
10
from os import getenv
11
from pathlib import Path
12
from shutil import copyfile
wxchan's avatar
wxchan committed
13
14

import numpy as np
15
import psutil
16
import pytest
17
from scipy.sparse import csr_matrix, isspmatrix_csc, isspmatrix_csr
18
from sklearn.datasets import load_svmlight_file, make_blobs, make_multilabel_classification
19
20
from sklearn.metrics import average_precision_score, log_loss, mean_absolute_error, mean_squared_error, roc_auc_score
from sklearn.model_selection import GroupKFold, TimeSeriesSplit, train_test_split
wxchan's avatar
wxchan committed
21

22
import lightgbm as lgb
23
from lightgbm.compat import PANDAS_INSTALLED, pd_DataFrame, pd_Series
24

25
26
27
28
29
30
31
32
33
34
35
36
37
from .utils import (
    SERIALIZERS,
    dummy_obj,
    load_breast_cancer,
    load_digits,
    load_iris,
    logistic_sigmoid,
    make_synthetic_regression,
    mse_obj,
    pickle_and_unpickle_object,
    sklearn_multiclass_custom_objective,
    softmax,
)
wxchan's avatar
wxchan committed
38

39
40
41
decreasing_generator = itertools.count(0, -1)


42
43
44
45
46
47
def logloss_obj(preds, train_data):
    y_true = train_data.get_label()
    y_pred = logistic_sigmoid(preds)
    grad = y_pred - y_true
    hess = y_pred * (1.0 - y_pred)
    return grad, hess
48
49


wxchan's avatar
wxchan committed
50
51
52
def multi_logloss(y_true, y_pred):
    return np.mean([-math.log(y_pred[i][y]) for i, y in enumerate(y_true)])

wxchan's avatar
wxchan committed
53

Belinda Trotta's avatar
Belinda Trotta committed
54
55
56
57
58
59
60
def top_k_error(y_true, y_pred, k):
    if k == y_pred.shape[1]:
        return 0
    max_rest = np.max(-np.partition(-y_pred, k)[:, k:], axis=1)
    return 1 - np.mean((y_pred[np.arange(len(y_true)), y_true] > max_rest))


61
def constant_metric(preds, train_data):
62
    return ("error", 0.0, False)
63
64
65


def decreasing_metric(preds, train_data):
66
    return ("decreasing_metric", next(decreasing_generator), False)
67
68


69
70
71
72
def categorize(continuous_x):
    return np.digitize(continuous_x, bins=np.arange(0, 1, 0.01))


73
74
75
76
def test_binary():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
77
78
79
80
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
        "num_iteration": 50,  # test num_iteration in dict here
81
82
83
84
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    evals_result = {}
85
    gbm = lgb.train(
86
        params, lgb_train, num_boost_round=20, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
87
    )
88
89
    ret = log_loss(y_test, gbm.predict(X_test))
    assert ret < 0.14
90
91
    assert len(evals_result["valid_0"]["binary_logloss"]) == 50
    assert evals_result["valid_0"]["binary_logloss"][-1] == pytest.approx(ret)
92
93
94
95
96
97


def test_rf():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
98
99
100
101
102
103
104
105
        "boosting_type": "rf",
        "objective": "binary",
        "bagging_freq": 1,
        "bagging_fraction": 0.5,
        "feature_fraction": 0.5,
        "num_leaves": 50,
        "metric": "binary_logloss",
        "verbose": -1,
106
107
108
109
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    evals_result = {}
110
    gbm = lgb.train(
111
        params, lgb_train, num_boost_round=50, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
112
    )
113
114
    ret = log_loss(y_test, gbm.predict(X_test))
    assert ret < 0.19
115
    assert evals_result["valid_0"]["binary_logloss"][-1] == pytest.approx(ret)
116
117


118
@pytest.mark.parametrize("objective", ["regression", "regression_l1", "huber", "fair", "poisson", "quantile"])
119
def test_regression(objective):
120
121
    X, y = make_synthetic_regression()
    y = np.abs(y)
122
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
123
    params = {"objective": objective, "metric": "l2", "verbose": -1}
124
125
126
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    evals_result = {}
127
    gbm = lgb.train(
128
        params, lgb_train, num_boost_round=50, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
129
    )
130
    ret = mean_squared_error(y_test, gbm.predict(X_test))
131
    if objective == "huber":
132
        assert ret < 430
133
    elif objective == "fair":
134
        assert ret < 296
135
    elif objective == "poisson":
136
        assert ret < 193
137
    elif objective == "quantile":
138
        assert ret < 1311
139
    else:
140
        assert ret < 343
141
    assert evals_result["valid_0"]["l2"][-1] == pytest.approx(ret)
142
143
144
145
146
147
148
149
150
151
152
153


def test_missing_value_handle():
    X_train = np.zeros((100, 1))
    y_train = np.zeros(100)
    trues = random.sample(range(100), 20)
    for idx in trues:
        X_train[idx, 0] = np.nan
        y_train[idx] = 1
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

154
    params = {"metric": "l2", "verbose": -1, "boost_from_average": False}
155
    evals_result = {}
156
    gbm = lgb.train(
157
        params, lgb_train, num_boost_round=20, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
158
    )
159
160
    ret = mean_squared_error(y_train, gbm.predict(X_train))
    assert ret < 0.005
161
    assert evals_result["valid_0"]["l2"][-1] == pytest.approx(ret)
162
163
164
165
166
167
168
169
170
171
172
173


def test_missing_value_handle_more_na():
    X_train = np.ones((100, 1))
    y_train = np.ones(100)
    trues = random.sample(range(100), 80)
    for idx in trues:
        X_train[idx, 0] = np.nan
        y_train[idx] = 0
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

174
    params = {"metric": "l2", "verbose": -1, "boost_from_average": False}
175
    evals_result = {}
176
    gbm = lgb.train(
177
        params, lgb_train, num_boost_round=20, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
178
    )
179
180
    ret = mean_squared_error(y_train, gbm.predict(X_train))
    assert ret < 0.005
181
    assert evals_result["valid_0"]["l2"][-1] == pytest.approx(ret)
182
183
184
185
186
187
188
189
190
191
192
193


def test_missing_value_handle_na():
    x = [0, 1, 2, 3, 4, 5, 6, 7, np.nan]
    y = [1, 1, 1, 1, 0, 0, 0, 0, 1]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
194
195
196
197
198
199
200
201
202
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "zero_as_missing": False,
203
204
    }
    evals_result = {}
205
    gbm = lgb.train(
206
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
207
    )
208
209
210
211
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
212
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
213
214
215
216
217
218
219
220
221
222
223
224


def test_missing_value_handle_zero():
    x = [0, 1, 2, 3, 4, 5, 6, 7, np.nan]
    y = [0, 1, 1, 1, 0, 0, 0, 0, 0]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
225
226
227
228
229
230
231
232
233
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "zero_as_missing": True,
234
235
    }
    evals_result = {}
236
    gbm = lgb.train(
237
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
238
    )
239
240
241
242
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
243
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
244
245
246
247
248
249
250
251
252
253
254
255


def test_missing_value_handle_none():
    x = [0, 1, 2, 3, 4, 5, 6, 7, np.nan]
    y = [0, 1, 1, 1, 0, 0, 0, 0, 0]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
256
257
258
259
260
261
262
263
264
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "use_missing": False,
265
266
    }
    evals_result = {}
267
    gbm = lgb.train(
268
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
269
    )
270
271
272
273
274
    pred = gbm.predict(X_train)
    assert pred[0] == pytest.approx(pred[1])
    assert pred[-1] == pytest.approx(pred[0])
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.83
275
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
276
277


278
279
280
281
282
283
284
285
286
287
288
289
290
291
@pytest.mark.parametrize(
    "use_quantized_grad",
    [
        pytest.param(
            True,
            marks=pytest.mark.skipif(
                getenv("TASK", "") == "cuda",
                reason="Skip because quantized training with categorical features is not supported for cuda version",
            ),
        ),
        False,
    ],
)
def test_categorical_handle(use_quantized_grad):
292
293
294
295
296
297
298
299
300
    x = [0, 1, 2, 3, 4, 5, 6, 7]
    y = [0, 1, 0, 1, 0, 1, 0, 1]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
301
302
303
304
305
306
307
308
309
310
311
312
313
314
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "min_data_per_group": 1,
        "cat_smooth": 1,
        "cat_l2": 0,
        "max_cat_to_onehot": 1,
        "zero_as_missing": True,
        "categorical_column": 0,
315
        "use_quantized_grad": use_quantized_grad,
316
317
    }
    evals_result = {}
318
    gbm = lgb.train(
319
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
320
    )
321
322
323
324
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
325
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
326
327


328
329
330
331
332
333
334
335
336
337
338
339
340
341
@pytest.mark.parametrize(
    "use_quantized_grad",
    [
        pytest.param(
            True,
            marks=pytest.mark.skipif(
                getenv("TASK", "") == "cuda",
                reason="Skip because quantized training with categorical features is not supported for cuda version",
            ),
        ),
        False,
    ],
)
def test_categorical_handle_na(use_quantized_grad):
342
343
344
345
346
347
348
349
350
    x = [0, np.nan, 0, np.nan, 0, np.nan]
    y = [0, 1, 0, 1, 0, 1]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
351
352
353
354
355
356
357
358
359
360
361
362
363
364
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "min_data_per_group": 1,
        "cat_smooth": 1,
        "cat_l2": 0,
        "max_cat_to_onehot": 1,
        "zero_as_missing": False,
        "categorical_column": 0,
365
        "use_quantized_grad": use_quantized_grad,
366
367
    }
    evals_result = {}
368
    gbm = lgb.train(
369
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
370
    )
371
372
373
374
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
375
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
376
377


378
379
380
381
382
383
384
385
386
387
388
389
390
391
@pytest.mark.parametrize(
    "use_quantized_grad",
    [
        pytest.param(
            True,
            marks=pytest.mark.skipif(
                getenv("TASK", "") == "cuda",
                reason="Skip because quantized training with categorical features is not supported for cuda version",
            ),
        ),
        False,
    ],
)
def test_categorical_non_zero_inputs(use_quantized_grad):
392
393
394
395
396
397
398
399
400
    x = [1, 1, 1, 1, 1, 1, 2, 2]
    y = [1, 1, 1, 1, 1, 1, 0, 0]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
401
402
403
404
405
406
407
408
409
410
411
412
413
414
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "min_data_per_group": 1,
        "cat_smooth": 1,
        "cat_l2": 0,
        "max_cat_to_onehot": 1,
        "zero_as_missing": False,
        "categorical_column": 0,
415
        "use_quantized_grad": use_quantized_grad,
416
417
    }
    evals_result = {}
418
    gbm = lgb.train(
419
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
420
    )
421
422
423
424
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
425
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
426
427
428
429
430


def test_multiclass():
    X, y = load_digits(n_class=10, return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
431
    params = {"objective": "multiclass", "metric": "multi_logloss", "num_class": 10, "verbose": -1}
432
433
434
    lgb_train = lgb.Dataset(X_train, y_train, params=params)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, params=params)
    evals_result = {}
435
    gbm = lgb.train(
436
        params, lgb_train, num_boost_round=50, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
437
    )
438
439
    ret = multi_logloss(y_test, gbm.predict(X_test))
    assert ret < 0.16
440
    assert evals_result["valid_0"]["multi_logloss"][-1] == pytest.approx(ret)
441
442
443
444
445
446


def test_multiclass_rf():
    X, y = load_digits(n_class=10, return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
447
448
449
450
451
452
453
454
455
456
457
        "boosting_type": "rf",
        "objective": "multiclass",
        "metric": "multi_logloss",
        "bagging_freq": 1,
        "bagging_fraction": 0.6,
        "feature_fraction": 0.6,
        "num_class": 10,
        "num_leaves": 50,
        "min_data": 1,
        "verbose": -1,
        "gpu_use_dp": True,
458
459
460
461
    }
    lgb_train = lgb.Dataset(X_train, y_train, params=params)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, params=params)
    evals_result = {}
462
    gbm = lgb.train(
463
        params, lgb_train, num_boost_round=50, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
464
    )
465
466
    ret = multi_logloss(y_test, gbm.predict(X_test))
    assert ret < 0.23
467
    assert evals_result["valid_0"]["multi_logloss"][-1] == pytest.approx(ret)
468
469
470
471
472


def test_multiclass_prediction_early_stopping():
    X, y = load_digits(n_class=10, return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
473
    params = {"objective": "multiclass", "metric": "multi_logloss", "num_class": 10, "verbose": -1}
474
    lgb_train = lgb.Dataset(X_train, y_train, params=params)
475
    gbm = lgb.train(params, lgb_train, num_boost_round=50)
476

477
    pred_parameter = {"pred_early_stop": True, "pred_early_stop_freq": 5, "pred_early_stop_margin": 1.5}
478
479
480
481
    ret = multi_logloss(y_test, gbm.predict(X_test, **pred_parameter))
    assert ret < 0.8
    assert ret > 0.6  # loss will be higher than when evaluating the full model

482
    pred_parameter["pred_early_stop_margin"] = 5.5
483
484
485
486
487
488
    ret = multi_logloss(y_test, gbm.predict(X_test, **pred_parameter))
    assert ret < 0.2


def test_multi_class_error():
    X, y = load_digits(n_class=10, return_X_y=True)
489
    params = {"objective": "multiclass", "num_classes": 10, "metric": "multi_error", "num_leaves": 4, "verbose": -1}
490
491
492
493
    lgb_data = lgb.Dataset(X, label=y)
    est = lgb.train(params, lgb_data, num_boost_round=10)
    predict_default = est.predict(X)
    results = {}
494
    est = lgb.train(
495
        dict(params, multi_error_top_k=1),
496
497
498
        lgb_data,
        num_boost_round=10,
        valid_sets=[lgb_data],
499
        callbacks=[lgb.record_evaluation(results)],
500
    )
501
502
503
504
505
    predict_1 = est.predict(X)
    # check that default gives same result as k = 1
    np.testing.assert_allclose(predict_1, predict_default)
    # check against independent calculation for k = 1
    err = top_k_error(y, predict_1, 1)
506
    assert results["training"]["multi_error"][-1] == pytest.approx(err)
507
508
    # check against independent calculation for k = 2
    results = {}
509
    est = lgb.train(
510
        dict(params, multi_error_top_k=2),
511
512
513
        lgb_data,
        num_boost_round=10,
        valid_sets=[lgb_data],
514
        callbacks=[lgb.record_evaluation(results)],
515
    )
516
517
    predict_2 = est.predict(X)
    err = top_k_error(y, predict_2, 2)
518
    assert results["training"]["multi_error@2"][-1] == pytest.approx(err)
519
520
    # check against independent calculation for k = 10
    results = {}
521
    est = lgb.train(
522
        dict(params, multi_error_top_k=10),
523
524
525
        lgb_data,
        num_boost_round=10,
        valid_sets=[lgb_data],
526
        callbacks=[lgb.record_evaluation(results)],
527
    )
528
529
    predict_3 = est.predict(X)
    err = top_k_error(y, predict_3, 10)
530
    assert results["training"]["multi_error@10"][-1] == pytest.approx(err)
531
532
533
534
    # check cases where predictions are equal
    X = np.array([[0, 0], [0, 0]])
    y = np.array([0, 1])
    lgb_data = lgb.Dataset(X, label=y)
535
    params["num_classes"] = 2
536
    results = {}
537
538
    lgb.train(params, lgb_data, num_boost_round=10, valid_sets=[lgb_data], callbacks=[lgb.record_evaluation(results)])
    assert results["training"]["multi_error"][-1] == pytest.approx(1)
539
    results = {}
540
    lgb.train(
541
        dict(params, multi_error_top_k=2),
542
543
544
        lgb_data,
        num_boost_round=10,
        valid_sets=[lgb_data],
545
        callbacks=[lgb.record_evaluation(results)],
546
    )
547
    assert results["training"]["multi_error@2"][-1] == pytest.approx(0)
548
549


550
551
552
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Skip due to differences in implementation details of CUDA version"
)
553
554
555
556
557
558
def test_auc_mu():
    # should give same result as binary auc for 2 classes
    X, y = load_digits(n_class=10, return_X_y=True)
    y_new = np.zeros((len(y)))
    y_new[y != 0] = 1
    lgb_X = lgb.Dataset(X, label=y_new)
559
    params = {"objective": "multiclass", "metric": "auc_mu", "verbose": -1, "num_classes": 2, "seed": 0}
560
    results_auc_mu = {}
561
562
    lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_auc_mu)])
    params = {"objective": "binary", "metric": "auc", "verbose": -1, "seed": 0}
563
    results_auc = {}
564
565
    lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_auc)])
    np.testing.assert_allclose(results_auc_mu["training"]["auc_mu"], results_auc["training"]["auc"])
566
567
    # test the case where all predictions are equal
    lgb_X = lgb.Dataset(X[:10], label=y_new[:10])
568
569
570
571
572
573
574
575
    params = {
        "objective": "multiclass",
        "metric": "auc_mu",
        "verbose": -1,
        "num_classes": 2,
        "min_data_in_leaf": 20,
        "seed": 0,
    }
576
    results_auc_mu = {}
577
578
    lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_auc_mu)])
    assert results_auc_mu["training"]["auc_mu"][-1] == pytest.approx(0.5)
579
580
581
582
583
584
    # test that weighted data gives different auc_mu
    lgb_X = lgb.Dataset(X, label=y)
    lgb_X_weighted = lgb.Dataset(X, label=y, weight=np.abs(np.random.normal(size=y.shape)))
    results_unweighted = {}
    results_weighted = {}
    params = dict(params, num_classes=10, num_leaves=5)
585
    lgb.train(
586
        params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_unweighted)]
587
588
589
590
591
592
    )
    lgb.train(
        params,
        lgb_X_weighted,
        num_boost_round=10,
        valid_sets=[lgb_X_weighted],
593
        callbacks=[lgb.record_evaluation(results_weighted)],
594
    )
595
596
    assert results_weighted["training"]["auc_mu"][-1] < 1
    assert results_unweighted["training"]["auc_mu"][-1] != results_weighted["training"]["auc_mu"][-1]
597
598
    # test that equal data weights give same auc_mu as unweighted data
    lgb_X_weighted = lgb.Dataset(X, label=y, weight=np.ones(y.shape) * 0.5)
599
600
601
602
603
    lgb.train(
        params,
        lgb_X_weighted,
        num_boost_round=10,
        valid_sets=[lgb_X_weighted],
604
605
606
607
        callbacks=[lgb.record_evaluation(results_weighted)],
    )
    assert results_unweighted["training"]["auc_mu"][-1] == pytest.approx(
        results_weighted["training"]["auc_mu"][-1], abs=1e-5
608
    )
609
610
611
612
    # should give 1 when accuracy = 1
    X = X[:10, :]
    y = y[:10]
    lgb_X = lgb.Dataset(X, label=y)
613
    params = {"objective": "multiclass", "metric": "auc_mu", "num_classes": 10, "min_data_in_leaf": 1, "verbose": -1}
614
    results = {}
615
616
    lgb.train(params, lgb_X, num_boost_round=100, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results)])
    assert results["training"]["auc_mu"][-1] == pytest.approx(1)
617
    # test loading class weights
618
    Xy = np.loadtxt(
619
        str(Path(__file__).absolute().parents[2] / "examples" / "multiclass_classification" / "multiclass.train")
620
    )
621
622
623
    y = Xy[:, 0]
    X = Xy[:, 1:]
    lgb_X = lgb.Dataset(X, label=y)
624
625
626
627
628
629
630
631
    params = {
        "objective": "multiclass",
        "metric": "auc_mu",
        "auc_mu_weights": [0, 2, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0],
        "num_classes": 5,
        "verbose": -1,
        "seed": 0,
    }
632
    results_weight = {}
633
634
    lgb.train(params, lgb_X, num_boost_round=5, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_weight)])
    params["auc_mu_weights"] = []
635
    results_no_weight = {}
636
    lgb.train(
637
        params, lgb_X, num_boost_round=5, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_no_weight)]
638
    )
639
    assert results_weight["training"]["auc_mu"][-1] != results_no_weight["training"]["auc_mu"][-1]
640
641


642
def test_ranking_prediction_early_stopping():
643
644
645
646
647
    rank_example_dir = Path(__file__).absolute().parents[2] / "examples" / "lambdarank"
    X_train, y_train = load_svmlight_file(str(rank_example_dir / "rank.train"))
    q_train = np.loadtxt(str(rank_example_dir / "rank.train.query"))
    X_test, _ = load_svmlight_file(str(rank_example_dir / "rank.test"))
    params = {"objective": "rank_xendcg", "verbose": -1}
648
649
650
    lgb_train = lgb.Dataset(X_train, y_train, group=q_train, params=params)
    gbm = lgb.train(params, lgb_train, num_boost_round=50)

651
    pred_parameter = {"pred_early_stop": True, "pred_early_stop_freq": 5, "pred_early_stop_margin": 1.5}
652
653
654
655
656
657
658
659
    ret_early = gbm.predict(X_test, **pred_parameter)

    pred_parameter["pred_early_stop_margin"] = 5.5
    ret_early_more_strict = gbm.predict(X_test, **pred_parameter)
    with pytest.raises(AssertionError):
        np.testing.assert_allclose(ret_early, ret_early_more_strict)


660
661
662
663
664
665
666
# Simulates position bias for a given ranking dataset.
# The ouput dataset is identical to the input one with the exception for the relevance labels.
# The new labels are generated according to an instance of a cascade user model:
# for each query, the user is simulated to be traversing the list of documents ranked by a baseline ranker
# (in our example it is simply the ordering by some feature correlated with relevance, e.g., 34)
# and clicks on that document (new_label=1) with some probability 'pclick' depending on its true relevance;
# at each position the user may stop the traversal with some probability pstop. For the non-clicked documents,
667
# new_label=0. Thus the generated new labels are biased towards the baseline ranker.
668
669
670
671
672
673
674
675
676
677
678
679
680
681
# The positions of the documents in the ranked lists produced by the baseline, are returned.
def simulate_position_bias(file_dataset_in, file_query_in, file_dataset_out, baseline_feature):
    # a mapping of a document's true relevance (defined on a 5-grade scale) into the probability of clicking it
    def get_pclick(label):
        if label == 0:
            return 0.4
        elif label == 1:
            return 0.6
        elif label == 2:
            return 0.7
        elif label == 3:
            return 0.8
        else:
            return 0.9
682

683
684
    # an instantiation of a cascade model where the user stops with probability 0.2 after observing each document
    pstop = 0.2
685

686
687
    f_dataset_in = open(file_dataset_in, "r")
    f_dataset_out = open(file_dataset_out, "w")
688
689
690
    random.seed(10)
    positions_all = []
    for line in open(file_query_in):
691
        docs_num = int(line)
692
        lines = []
693
        index_values = []
694
695
696
697
698
699
        positions = [0] * docs_num
        for index in range(docs_num):
            features = f_dataset_in.readline().split()
            lines.append(features)
            val = 0.0
            for feature_val in features:
700
                feature_val_split = feature_val.split(":")
701
702
703
704
                if int(feature_val_split[0]) == baseline_feature:
                    val = float(feature_val_split[1])
            index_values.append([index, val])
        index_values.sort(key=lambda x: -x[1])
705
        stop = False
706
707
708
709
710
711
712
        for pos in range(docs_num):
            index = index_values[pos][0]
            new_label = 0
            if not stop:
                label = int(lines[index][0])
                pclick = get_pclick(label)
                if random.random() < pclick:
713
                    new_label = 1
714
715
716
717
                stop = random.random() < pstop
            lines[index][0] = str(new_label)
            positions[index] = pos
        for features in lines:
718
            f_dataset_out.write(" ".join(features) + "\n")
719
720
721
722
723
        positions_all.extend(positions)
    f_dataset_out.close()
    return positions_all


724
725
726
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Positions in learning to rank is not supported in CUDA version yet"
)
727
def test_ranking_with_position_information_with_file(tmp_path):
728
    rank_example_dir = Path(__file__).absolute().parents[2] / "examples" / "lambdarank"
729
    params = {
730
731
732
733
734
735
736
737
        "objective": "lambdarank",
        "verbose": -1,
        "eval_at": [3],
        "metric": "ndcg",
        "bagging_freq": 1,
        "bagging_fraction": 0.9,
        "min_data_in_leaf": 50,
        "min_sum_hessian_in_leaf": 5.0,
738
739
740
    }

    # simulate position bias for the train dataset and put the train dataset with biased labels to temp directory
741
742
743
744
745
746
747
748
749
    positions = simulate_position_bias(
        str(rank_example_dir / "rank.train"),
        str(rank_example_dir / "rank.train.query"),
        str(tmp_path / "rank.train"),
        baseline_feature=34,
    )
    copyfile(str(rank_example_dir / "rank.train.query"), str(tmp_path / "rank.train.query"))
    copyfile(str(rank_example_dir / "rank.test"), str(tmp_path / "rank.test"))
    copyfile(str(rank_example_dir / "rank.test.query"), str(tmp_path / "rank.test.query"))
750

751
752
753
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
    gbm_baseline = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
754

755
    f_positions_out = open(str(tmp_path / "rank.train.position"), "w")
756
    for pos in positions:
757
        f_positions_out.write(str(pos) + "\n")
758
759
    f_positions_out.close()

760
761
762
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
    gbm_unbiased_with_file = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
763

764
    # the performance of the unbiased LambdaMART should outperform the plain LambdaMART on the dataset with position bias
765
    assert gbm_baseline.best_score["valid_0"]["ndcg@3"] + 0.03 <= gbm_unbiased_with_file.best_score["valid_0"]["ndcg@3"]
766
767

    # add extra row to position file
768
769
    with open(str(tmp_path / "rank.train.position"), "a") as file:
        file.write("pos_1000\n")
770
        file.close()
771
772
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
773
    with pytest.raises(lgb.basic.LightGBMError, match=r"Positions size \(3006\) doesn't match data size"):
774
        lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
775
776


777
778
779
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Positions in learning to rank is not supported in CUDA version yet"
)
780
def test_ranking_with_position_information_with_dataset_constructor(tmp_path):
781
    rank_example_dir = Path(__file__).absolute().parents[2] / "examples" / "lambdarank"
782
    params = {
783
784
785
786
787
788
789
790
791
792
793
        "objective": "lambdarank",
        "verbose": -1,
        "eval_at": [3],
        "metric": "ndcg",
        "bagging_freq": 1,
        "bagging_fraction": 0.9,
        "min_data_in_leaf": 50,
        "min_sum_hessian_in_leaf": 5.0,
        "num_threads": 1,
        "deterministic": True,
        "seed": 0,
794
795
796
    }

    # simulate position bias for the train dataset and put the train dataset with biased labels to temp directory
797
798
799
800
801
802
803
804
805
    positions = simulate_position_bias(
        str(rank_example_dir / "rank.train"),
        str(rank_example_dir / "rank.train.query"),
        str(tmp_path / "rank.train"),
        baseline_feature=34,
    )
    copyfile(str(rank_example_dir / "rank.train.query"), str(tmp_path / "rank.train.query"))
    copyfile(str(rank_example_dir / "rank.test"), str(tmp_path / "rank.test"))
    copyfile(str(rank_example_dir / "rank.test.query"), str(tmp_path / "rank.test.query"))
806

807
808
809
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
    gbm_baseline = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
810
811
812
813

    positions = np.array(positions)

    # test setting positions through Dataset constructor with numpy array
814
815
816
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params, position=positions)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
    gbm_unbiased = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
817
818

    # the performance of the unbiased LambdaMART should outperform the plain LambdaMART on the dataset with position bias
819
    assert gbm_baseline.best_score["valid_0"]["ndcg@3"] + 0.03 <= gbm_unbiased.best_score["valid_0"]["ndcg@3"]
820
821
822

    if PANDAS_INSTALLED:
        # test setting positions through Dataset constructor with pandas Series
823
824
825
826
827
828
        lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params, position=pd_Series(positions))
        lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
        gbm_unbiased_pandas_series = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
        assert (
            gbm_unbiased.best_score["valid_0"]["ndcg@3"] == gbm_unbiased_pandas_series.best_score["valid_0"]["ndcg@3"]
        )
829
830

    # test setting positions through set_position
831
832
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
833
    lgb_train.set_position(positions)
834
835
    gbm_unbiased_set_position = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
    assert gbm_unbiased.best_score["valid_0"]["ndcg@3"] == gbm_unbiased_set_position.best_score["valid_0"]["ndcg@3"]
836
837
838
839
840
841

    # test get_position works
    positions_from_get = lgb_train.get_position()
    np.testing.assert_array_equal(positions_from_get, positions)


842
843
def test_early_stopping():
    X, y = load_breast_cancer(return_X_y=True)
844
    params = {"objective": "binary", "metric": "binary_logloss", "verbose": -1}
845
846
847
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
848
    valid_set_name = "valid_set"
849
    # no early stopping
850
851
852
853
854
855
856
857
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=10,
        valid_sets=lgb_eval,
        valid_names=valid_set_name,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
    )
858
859
    assert gbm.best_iteration == 10
    assert valid_set_name in gbm.best_score
860
    assert "binary_logloss" in gbm.best_score[valid_set_name]
861
    # early stopping occurs
862
863
864
865
866
867
868
869
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=40,
        valid_sets=lgb_eval,
        valid_names=valid_set_name,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
    )
870
871
    assert gbm.best_iteration <= 39
    assert valid_set_name in gbm.best_score
872
    assert "binary_logloss" in gbm.best_score[valid_set_name]
873
874


875
@pytest.mark.parametrize("use_valid", [True, False])
876
877
878
879
880
881
882
883
884
def test_early_stopping_ignores_training_set(use_valid):
    x = np.linspace(-1, 1, 100)
    X = x.reshape(-1, 1)
    y = x**2
    X_train, X_valid = X[:80], X[80:]
    y_train, y_valid = y[:80], y[80:]
    train_ds = lgb.Dataset(X_train, y_train)
    valid_ds = lgb.Dataset(X_valid, y_valid)
    valid_sets = [train_ds]
885
    valid_names = ["train"]
886
887
    if use_valid:
        valid_sets.append(valid_ds)
888
        valid_names.append("valid")
889
890
891
892
    eval_result = {}

    def train_fn():
        return lgb.train(
893
            {"num_leaves": 5},
894
895
896
897
            train_ds,
            num_boost_round=2,
            valid_sets=valid_sets,
            valid_names=valid_names,
898
            callbacks=[lgb.early_stopping(1), lgb.record_evaluation(eval_result)],
899
        )
900

901
902
903
    if use_valid:
        bst = train_fn()
        assert bst.best_iteration == 1
904
905
        assert eval_result["train"]["l2"][1] < eval_result["train"]["l2"][0]  # train improved
        assert eval_result["valid"]["l2"][1] > eval_result["valid"]["l2"][0]  # valid didn't
906
    else:
907
        with pytest.warns(UserWarning, match="Only training set found, disabling early stopping."):
908
909
910
911
912
            bst = train_fn()
        assert bst.current_iteration() == 2
        assert bst.best_iteration == 0


913
@pytest.mark.parametrize("first_metric_only", [True, False])
914
915
916
917
def test_early_stopping_via_global_params(first_metric_only):
    X, y = load_breast_cancer(return_X_y=True)
    num_trees = 5
    params = {
918
919
920
921
922
923
        "num_trees": num_trees,
        "objective": "binary",
        "metric": "None",
        "verbose": -1,
        "early_stopping_round": 2,
        "first_metric_only": first_metric_only,
924
925
926
927
    }
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
928
929
930
931
    valid_set_name = "valid_set"
    gbm = lgb.train(
        params, lgb_train, feval=[decreasing_metric, constant_metric], valid_sets=lgb_eval, valid_names=valid_set_name
    )
932
933
934
935
936
    if first_metric_only:
        assert gbm.best_iteration == num_trees
    else:
        assert gbm.best_iteration == 1
    assert valid_set_name in gbm.best_score
937
938
    assert "decreasing_metric" in gbm.best_score[valid_set_name]
    assert "error" in gbm.best_score[valid_set_name]
939
940


941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
@pytest.mark.parametrize("early_stopping_round", [-10, -1, 0, None, "None"])
def test_early_stopping_is_not_enabled_for_non_positive_stopping_rounds(early_stopping_round):
    X, y = load_breast_cancer(return_X_y=True)
    num_trees = 5
    params = {
        "num_trees": num_trees,
        "objective": "binary",
        "metric": "None",
        "verbose": -1,
        "early_stopping_round": early_stopping_round,
        "first_metric_only": True,
    }
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    valid_set_name = "valid_set"

    if early_stopping_round is None:
        gbm = lgb.train(
            params,
            lgb_train,
            feval=[constant_metric],
            valid_sets=lgb_eval,
            valid_names=valid_set_name,
        )
        assert "early_stopping_round" not in gbm.params
        assert gbm.num_trees() == num_trees
    elif early_stopping_round == "None":
        with pytest.raises(TypeError, match="early_stopping_round should be an integer. Got 'str'"):
            gbm = lgb.train(
                params,
                lgb_train,
                feval=[constant_metric],
                valid_sets=lgb_eval,
                valid_names=valid_set_name,
            )
    elif early_stopping_round <= 0:
        gbm = lgb.train(
            params,
            lgb_train,
            feval=[constant_metric],
            valid_sets=lgb_eval,
            valid_names=valid_set_name,
        )
        assert gbm.params["early_stopping_round"] == early_stopping_round
        assert gbm.num_trees() == num_trees


989
990
991
@pytest.mark.parametrize("first_only", [True, False])
@pytest.mark.parametrize("single_metric", [True, False])
@pytest.mark.parametrize("greater_is_better", [True, False])
992
993
994
995
def test_early_stopping_min_delta(first_only, single_metric, greater_is_better):
    if single_metric and not first_only:
        pytest.skip("first_metric_only doesn't affect single metric.")
    metric2min_delta = {
996
997
998
999
        "auc": 0.001,
        "binary_logloss": 0.01,
        "average_precision": 0.001,
        "mape": 0.01,
1000
1001
1002
    }
    if single_metric:
        if greater_is_better:
1003
            metric = "auc"
1004
        else:
1005
            metric = "binary_logloss"
1006
1007
1008
    else:
        if first_only:
            if greater_is_better:
1009
                metric = ["auc", "binary_logloss"]
1010
            else:
1011
                metric = ["binary_logloss", "auc"]
1012
1013
        else:
            if greater_is_better:
1014
                metric = ["auc", "average_precision"]
1015
            else:
1016
                metric = ["binary_logloss", "mape"]
1017
1018
1019
1020
1021
1022

    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=0)
    train_ds = lgb.Dataset(X_train, y_train)
    valid_ds = lgb.Dataset(X_valid, y_valid, reference=train_ds)

1023
    params = {"objective": "binary", "metric": metric, "verbose": -1}
1024
1025
1026
1027
1028
1029
    if isinstance(metric, str):
        min_delta = metric2min_delta[metric]
    elif first_only:
        min_delta = metric2min_delta[metric[0]]
    else:
        min_delta = [metric2min_delta[m] for m in metric]
1030
1031
1032
1033
1034
    train_kwargs = {
        "params": params,
        "train_set": train_ds,
        "num_boost_round": 50,
        "valid_sets": [train_ds, valid_ds],
1035
        "valid_names": ["training", "valid"],
1036
    }
1037
1038
1039

    # regular early stopping
    evals_result = {}
1040
    train_kwargs["callbacks"] = [
1041
        lgb.callback.early_stopping(10, first_only, verbose=False),
1042
        lgb.record_evaluation(evals_result),
1043
1044
    ]
    bst = lgb.train(**train_kwargs)
1045
    scores = np.vstack(list(evals_result["valid"].values())).T
1046
1047
1048

    # positive min_delta
    delta_result = {}
1049
    train_kwargs["callbacks"] = [
1050
        lgb.callback.early_stopping(10, first_only, verbose=False, min_delta=min_delta),
1051
        lgb.record_evaluation(delta_result),
1052
1053
    ]
    delta_bst = lgb.train(**train_kwargs)
1054
    delta_scores = np.vstack(list(delta_result["valid"].values())).T
1055
1056
1057
1058
1059
1060

    if first_only:
        scores = scores[:, 0]
        delta_scores = delta_scores[:, 0]

    assert delta_bst.num_trees() < bst.num_trees()
1061
    np.testing.assert_allclose(scores[: len(delta_scores)], delta_scores)
1062
1063
1064
1065
1066
1067
1068
1069
    last_score = delta_scores[-1]
    best_score = delta_scores[delta_bst.num_trees() - 1]
    if greater_is_better:
        assert np.less_equal(last_score, best_score + min_delta).any()
    else:
        assert np.greater_equal(last_score, best_score - min_delta).any()


1070
1071
1072
1073
1074
1075
def test_early_stopping_can_be_triggered_via_custom_callback():
    X, y = make_synthetic_regression()

    def _early_stop_after_seventh_iteration(env):
        if env.iteration == 6:
            exc = lgb.EarlyStopException(
1076
                best_iteration=6, best_score=[("some_validation_set", "some_metric", 0.708, True)]
1077
1078
1079
1080
            )
            raise exc

    bst = lgb.train(
1081
        params={"objective": "regression", "verbose": -1, "num_leaves": 2},
1082
1083
        train_set=lgb.Dataset(X, label=y),
        num_boost_round=23,
1084
        callbacks=[_early_stop_after_seventh_iteration],
1085
1086
1087
1088
1089
1090
1091
    )
    assert bst.num_trees() == 7
    assert bst.best_score["some_validation_set"]["some_metric"] == 0.708
    assert bst.best_iteration == 7
    assert bst.current_iteration() == 7


1092
def test_continue_train():
1093
    X, y = make_synthetic_regression()
1094
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
1095
    params = {"objective": "regression", "metric": "l1", "verbose": -1}
1096
1097
1098
    lgb_train = lgb.Dataset(X_train, y_train, free_raw_data=False)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, free_raw_data=False)
    init_gbm = lgb.train(params, lgb_train, num_boost_round=20)
1099
    model_name = "model.txt"
1100
1101
    init_gbm.save_model(model_name)
    evals_result = {}
1102
1103
1104
1105
1106
1107
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=30,
        valid_sets=lgb_eval,
        # test custom eval metrics
1108
        feval=(lambda p, d: ("custom_mae", mean_absolute_error(p, d.get_label()), False)),
1109
        callbacks=[lgb.record_evaluation(evals_result)],
1110
        init_model="model.txt",
1111
    )
1112
    ret = mean_absolute_error(y_test, gbm.predict(X_test))
1113
    assert ret < 13.6
1114
1115
    assert evals_result["valid_0"]["l1"][-1] == pytest.approx(ret)
    np.testing.assert_allclose(evals_result["valid_0"]["l1"], evals_result["valid_0"]["custom_mae"])
1116
1117
1118


def test_continue_train_reused_dataset():
1119
    X, y = make_synthetic_regression()
1120
    params = {"objective": "regression", "verbose": -1}
1121
1122
1123
1124
1125
1126
1127
1128
1129
    lgb_train = lgb.Dataset(X, y, free_raw_data=False)
    init_gbm = lgb.train(params, lgb_train, num_boost_round=5)
    init_gbm_2 = lgb.train(params, lgb_train, num_boost_round=5, init_model=init_gbm)
    init_gbm_3 = lgb.train(params, lgb_train, num_boost_round=5, init_model=init_gbm_2)
    gbm = lgb.train(params, lgb_train, num_boost_round=5, init_model=init_gbm_3)
    assert gbm.current_iteration() == 20


def test_continue_train_dart():
1130
    X, y = make_synthetic_regression()
1131
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
1132
    params = {"boosting_type": "dart", "objective": "regression", "metric": "l1", "verbose": -1}
1133
1134
1135
1136
    lgb_train = lgb.Dataset(X_train, y_train, free_raw_data=False)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, free_raw_data=False)
    init_gbm = lgb.train(params, lgb_train, num_boost_round=50)
    evals_result = {}
1137
1138
1139
1140
1141
1142
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=50,
        valid_sets=lgb_eval,
        callbacks=[lgb.record_evaluation(evals_result)],
1143
        init_model=init_gbm,
1144
    )
1145
    ret = mean_absolute_error(y_test, gbm.predict(X_test))
1146
    assert ret < 13.6
1147
    assert evals_result["valid_0"]["l1"][-1] == pytest.approx(ret)
1148
1149
1150
1151
1152


def test_continue_train_multiclass():
    X, y = load_iris(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
1153
    params = {"objective": "multiclass", "metric": "multi_logloss", "num_class": 3, "verbose": -1}
1154
1155
1156
1157
    lgb_train = lgb.Dataset(X_train, y_train, params=params, free_raw_data=False)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, params=params, free_raw_data=False)
    init_gbm = lgb.train(params, lgb_train, num_boost_round=20)
    evals_result = {}
1158
1159
1160
1161
1162
1163
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=30,
        valid_sets=lgb_eval,
        callbacks=[lgb.record_evaluation(evals_result)],
1164
        init_model=init_gbm,
1165
    )
1166
1167
    ret = multi_logloss(y_test, gbm.predict(X_test))
    assert ret < 0.1
1168
    assert evals_result["valid_0"]["multi_logloss"][-1] == pytest.approx(ret)
1169
1170
1171


def test_cv():
1172
    X_train, y_train = make_synthetic_regression()
1173
    params = {"verbose": -1}
1174
1175
    lgb_train = lgb.Dataset(X_train, y_train)
    # shuffle = False, override metric in params
1176
1177
1178
1179
1180
1181
1182
    params_with_metric = {"metric": "l2", "verbose": -1}
    cv_res = lgb.cv(
        params_with_metric, lgb_train, num_boost_round=10, nfold=3, stratified=False, shuffle=False, metrics="l1"
    )
    assert "valid l1-mean" in cv_res
    assert "valid l2-mean" not in cv_res
    assert len(cv_res["valid l1-mean"]) == 10
1183
    # shuffle = True, callbacks
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
    cv_res = lgb.cv(
        params,
        lgb_train,
        num_boost_round=10,
        nfold=3,
        stratified=False,
        shuffle=True,
        metrics="l1",
        callbacks=[lgb.reset_parameter(learning_rate=lambda i: 0.1 - 0.001 * i)],
    )
    assert "valid l1-mean" in cv_res
    assert len(cv_res["valid l1-mean"]) == 10
1196
    # enable display training loss
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
    cv_res = lgb.cv(
        params_with_metric,
        lgb_train,
        num_boost_round=10,
        nfold=3,
        stratified=False,
        shuffle=False,
        metrics="l1",
        eval_train_metric=True,
    )
    assert "train l1-mean" in cv_res
    assert "valid l1-mean" in cv_res
    assert "train l2-mean" not in cv_res
    assert "valid l2-mean" not in cv_res
    assert len(cv_res["train l1-mean"]) == 10
    assert len(cv_res["valid l1-mean"]) == 10
1213
1214
1215
    # self defined folds
    tss = TimeSeriesSplit(3)
    folds = tss.split(X_train)
1216
1217
    cv_res_gen = lgb.cv(params_with_metric, lgb_train, num_boost_round=10, folds=folds)
    cv_res_obj = lgb.cv(params_with_metric, lgb_train, num_boost_round=10, folds=tss)
1218
    np.testing.assert_allclose(cv_res_gen["valid l2-mean"], cv_res_obj["valid l2-mean"])
Andrew Ziem's avatar
Andrew Ziem committed
1219
    # LambdaRank
1220
1221
1222
1223
    rank_example_dir = Path(__file__).absolute().parents[2] / "examples" / "lambdarank"
    X_train, y_train = load_svmlight_file(str(rank_example_dir / "rank.train"))
    q_train = np.loadtxt(str(rank_example_dir / "rank.train.query"))
    params_lambdarank = {"objective": "lambdarank", "verbose": -1, "eval_at": 3}
1224
1225
    lgb_train = lgb.Dataset(X_train, y_train, group=q_train)
    # ... with l2 metric
1226
    cv_res_lambda = lgb.cv(params_lambdarank, lgb_train, num_boost_round=10, nfold=3, metrics="l2")
1227
    assert len(cv_res_lambda) == 2
1228
    assert not np.isnan(cv_res_lambda["valid l2-mean"]).any()
1229
    # ... with NDCG (default) metric
1230
    cv_res_lambda = lgb.cv(params_lambdarank, lgb_train, num_boost_round=10, nfold=3)
1231
    assert len(cv_res_lambda) == 2
1232
    assert not np.isnan(cv_res_lambda["valid ndcg@3-mean"]).any()
1233
    # self defined folds with lambdarank
1234
1235
    cv_res_lambda_obj = lgb.cv(params_lambdarank, lgb_train, num_boost_round=10, folds=GroupKFold(n_splits=3))
    np.testing.assert_allclose(cv_res_lambda["valid ndcg@3-mean"], cv_res_lambda_obj["valid ndcg@3-mean"])
1236
1237


1238
1239
def test_cv_works_with_init_model(tmp_path):
    X, y = make_synthetic_regression()
1240
    params = {"objective": "regression", "verbose": -1}
1241
1242
    num_train_rounds = 2
    lgb_train = lgb.Dataset(X, y, free_raw_data=False)
1243
    bst = lgb.train(params=params, train_set=lgb_train, num_boost_round=num_train_rounds)
1244
    preds_raw = bst.predict(X, raw_score=True)
1245
    model_path_txt = str(tmp_path / "lgb.model")
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
    bst.save_model(model_path_txt)

    num_cv_rounds = 5
    cv_kwargs = {
        "num_boost_round": num_cv_rounds,
        "nfold": 3,
        "stratified": False,
        "shuffle": False,
        "seed": 708,
        "return_cvbooster": True,
1256
        "params": params,
1257
1258
1259
    }

    # init_model from an in-memory Booster
1260
    cv_res = lgb.cv(train_set=lgb_train, init_model=bst, **cv_kwargs)
1261
1262
1263
    cv_bst_w_in_mem_init_model = cv_res["cvbooster"]
    assert cv_bst_w_in_mem_init_model.current_iteration() == [num_train_rounds + num_cv_rounds] * 3
    for booster in cv_bst_w_in_mem_init_model.boosters:
1264
        np.testing.assert_allclose(preds_raw, booster.predict(X, raw_score=True, num_iteration=num_train_rounds))
1265
1266

    # init_model from a text file
1267
    cv_res = lgb.cv(train_set=lgb_train, init_model=model_path_txt, **cv_kwargs)
1268
1269
1270
    cv_bst_w_file_init_model = cv_res["cvbooster"]
    assert cv_bst_w_file_init_model.current_iteration() == [num_train_rounds + num_cv_rounds] * 3
    for booster in cv_bst_w_file_init_model.boosters:
1271
        np.testing.assert_allclose(preds_raw, booster.predict(X, raw_score=True, num_iteration=num_train_rounds))
1272
1273
1274
1275

    # predictions should be identical
    for i in range(3):
        np.testing.assert_allclose(
1276
            cv_bst_w_in_mem_init_model.boosters[i].predict(X), cv_bst_w_file_init_model.boosters[i].predict(X)
1277
1278
1279
        )


1280
1281
1282
1283
def test_cvbooster():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1284
1285
1286
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
1287
    }
1288
    nfold = 3
1289
1290
    lgb_train = lgb.Dataset(X_train, y_train)
    # with early stopping
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
    cv_res = lgb.cv(
        params,
        lgb_train,
        num_boost_round=25,
        nfold=nfold,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
        return_cvbooster=True,
    )
    assert "cvbooster" in cv_res
    cvb = cv_res["cvbooster"]
1301
1302
    assert isinstance(cvb, lgb.CVBooster)
    assert isinstance(cvb.boosters, list)
1303
    assert len(cvb.boosters) == nfold
1304
1305
1306
    assert all(isinstance(bst, lgb.Booster) for bst in cvb.boosters)
    assert cvb.best_iteration > 0
    # predict by each fold booster
1307
    preds = cvb.predict(X_test)
1308
    assert isinstance(preds, list)
1309
1310
1311
1312
1313
1314
    assert len(preds) == nfold
    # check that each booster predicted using the best iteration
    for fold_preds, bst in zip(preds, cvb.boosters):
        assert bst.best_iteration == cvb.best_iteration
        expected = bst.predict(X_test, num_iteration=cvb.best_iteration)
        np.testing.assert_allclose(fold_preds, expected)
1315
1316
1317
1318
1319
    # fold averaging
    avg_pred = np.mean(preds, axis=0)
    ret = log_loss(y_test, avg_pred)
    assert ret < 0.13
    # without early stopping
1320
1321
    cv_res = lgb.cv(params, lgb_train, num_boost_round=20, nfold=3, return_cvbooster=True)
    cvb = cv_res["cvbooster"]
1322
1323
1324
1325
1326
1327
1328
    assert cvb.best_iteration == -1
    preds = cvb.predict(X_test)
    avg_pred = np.mean(preds, axis=0)
    ret = log_loss(y_test, avg_pred)
    assert ret < 0.15


1329
1330
1331
1332
def test_cvbooster_save_load(tmp_path):
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, _ = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1333
1334
1335
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
1336
1337
1338
1339
    }
    nfold = 3
    lgb_train = lgb.Dataset(X_train, y_train)

1340
1341
1342
1343
1344
1345
1346
1347
1348
    cv_res = lgb.cv(
        params,
        lgb_train,
        num_boost_round=10,
        nfold=nfold,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
        return_cvbooster=True,
    )
    cvbooster = cv_res["cvbooster"]
1349
1350
1351
    preds = cvbooster.predict(X_test)
    best_iteration = cvbooster.best_iteration

1352
    model_path_txt = str(tmp_path / "lgb.model")
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364

    cvbooster.save_model(model_path_txt)
    model_string = cvbooster.model_to_string()
    del cvbooster

    cvbooster_from_txt_file = lgb.CVBooster(model_file=model_path_txt)
    cvbooster_from_string = lgb.CVBooster().model_from_string(model_string)
    for cvbooster_loaded in [cvbooster_from_txt_file, cvbooster_from_string]:
        assert best_iteration == cvbooster_loaded.best_iteration
        np.testing.assert_array_equal(preds, cvbooster_loaded.predict(X_test))


1365
@pytest.mark.parametrize("serializer", SERIALIZERS)
1366
1367
1368
1369
def test_cvbooster_picklable(serializer):
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, _ = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1370
1371
1372
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
1373
1374
1375
1376
    }
    nfold = 3
    lgb_train = lgb.Dataset(X_train, y_train)

1377
1378
1379
1380
1381
1382
1383
1384
1385
    cv_res = lgb.cv(
        params,
        lgb_train,
        num_boost_round=10,
        nfold=nfold,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
        return_cvbooster=True,
    )
    cvbooster = cv_res["cvbooster"]
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
    preds = cvbooster.predict(X_test)
    best_iteration = cvbooster.best_iteration

    cvbooster_from_disk = pickle_and_unpickle_object(obj=cvbooster, serializer=serializer)
    del cvbooster

    assert best_iteration == cvbooster_from_disk.best_iteration

    preds_from_disk = cvbooster_from_disk.predict(X_test)
    np.testing.assert_array_equal(preds, preds_from_disk)


1398
def test_feature_name():
1399
    X_train, y_train = make_synthetic_regression()
1400
    params = {"verbose": -1}
1401
    lgb_train = lgb.Dataset(X_train, y_train)
1402
    feature_names = [f"f_{i}" for i in range(X_train.shape[-1])]
1403
1404
1405
    gbm = lgb.train(params, lgb_train, num_boost_round=5, feature_name=feature_names)
    assert feature_names == gbm.feature_name()
    # test feature_names with whitespaces
1406
    feature_names_with_space = [f"f {i}" for i in range(X_train.shape[-1])]
1407
1408
1409
1410
1411
1412
1413
1414
    gbm = lgb.train(params, lgb_train, num_boost_round=5, feature_name=feature_names_with_space)
    assert feature_names == gbm.feature_name()


def test_feature_name_with_non_ascii():
    X_train = np.random.normal(size=(100, 4))
    y_train = np.random.random(100)
    # This has non-ascii strings.
1415
1416
    feature_names = ["F_零", "F_一", "F_二", "F_三"]
    params = {"verbose": -1}
1417
1418
1419
1420
    lgb_train = lgb.Dataset(X_train, y_train)

    gbm = lgb.train(params, lgb_train, num_boost_round=5, feature_name=feature_names)
    assert feature_names == gbm.feature_name()
1421
    gbm.save_model("lgb.model")
1422

1423
    gbm2 = lgb.Booster(model_file="lgb.model")
1424
1425
1426
    assert feature_names == gbm2.feature_name()


1427
def test_parameters_are_loaded_from_model_file(tmp_path, capsys):
1428
1429
1430
1431
    X = np.hstack([np.random.rand(100, 1), np.random.randint(0, 5, (100, 2))])
    y = np.random.rand(100)
    ds = lgb.Dataset(X, y)
    params = {
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
        "bagging_fraction": 0.8,
        "bagging_freq": 2,
        "boosting": "rf",
        "feature_contri": [0.5, 0.5, 0.5],
        "feature_fraction": 0.7,
        "boost_from_average": False,
        "interaction_constraints": [[0, 1], [0]],
        "metric": ["l2", "rmse"],
        "num_leaves": 5,
        "num_threads": 1,
1442
    }
1443
    model_file = tmp_path / "model.txt"
1444
1445
    orig_bst = lgb.train(params, ds, num_boost_round=1, categorical_feature=[1, 2])
    orig_bst.save_model(model_file)
1446
    with model_file.open("rt") as f:
1447
        model_contents = f.readlines()
1448
1449
1450
    params_start = model_contents.index("parameters:\n")
    model_contents.insert(params_start + 1, "[max_conflict_rate: 0]\n")
    with model_file.open("wt") as f:
1451
        f.writelines(model_contents)
1452
    bst = lgb.Booster(model_file=model_file)
1453
1454
1455
    expected_msg = "[LightGBM] [Warning] Ignoring unrecognized parameter 'max_conflict_rate' found in model string."
    stdout = capsys.readouterr().out
    assert expected_msg in stdout
1456
1457
    set_params = {k: bst.params[k] for k in params.keys()}
    assert set_params == params
1458
    assert bst.params["categorical_feature"] == [1, 2]
1459
1460

    # check that passing parameters to the constructor raises warning and ignores them
1461
1462
    with pytest.warns(UserWarning, match="Ignoring params argument"):
        bst2 = lgb.Booster(params={"num_leaves": 7}, model_file=model_file)
1463
1464
    assert bst.params == bst2.params

1465
1466
1467
1468
1469
    # check inference isn't affected by unknown parameter
    orig_preds = orig_bst.predict(X)
    preds = bst.predict(X)
    np.testing.assert_allclose(preds, orig_preds)

1470

1471
1472
def test_save_load_copy_pickle():
    def train_and_predict(init_model=None, return_model=False):
1473
        X, y = make_synthetic_regression()
1474
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
1475
        params = {"objective": "regression", "metric": "l2", "verbose": -1}
1476
        lgb_train = lgb.Dataset(X_train, y_train)
1477
1478
1479
1480
1481
1482
        gbm_template = lgb.train(params, lgb_train, num_boost_round=10, init_model=init_model)
        return gbm_template if return_model else mean_squared_error(y_test, gbm_template.predict(X_test))

    gbm = train_and_predict(return_model=True)
    ret_origin = train_and_predict(init_model=gbm)
    other_ret = []
1483
1484
    gbm.save_model("lgb.model")
    with open("lgb.model") as f:  # check all params are logged into model file correctly
1485
        assert f.read().find("[num_iterations: 10]") != -1
1486
1487
    other_ret.append(train_and_predict(init_model="lgb.model"))
    gbm_load = lgb.Booster(model_file="lgb.model")
1488
1489
1490
    other_ret.append(train_and_predict(init_model=gbm_load))
    other_ret.append(train_and_predict(init_model=copy.copy(gbm)))
    other_ret.append(train_and_predict(init_model=copy.deepcopy(gbm)))
1491
    with open("lgb.pkl", "wb") as f:
1492
        pickle.dump(gbm, f)
1493
    with open("lgb.pkl", "rb") as f:
1494
1495
1496
1497
1498
1499
1500
1501
        gbm_pickle = pickle.load(f)
    other_ret.append(train_and_predict(init_model=gbm_pickle))
    gbm_pickles = pickle.loads(pickle.dumps(gbm))
    other_ret.append(train_and_predict(init_model=gbm_pickles))
    for ret in other_ret:
        assert ret_origin == pytest.approx(ret)


1502
1503
1504
def test_all_expected_params_are_written_out_to_model_text(tmp_path):
    X, y = make_synthetic_regression()
    params = {
1505
1506
1507
1508
1509
1510
        "objective": "mape",
        "metric": ["l2", "mae"],
        "seed": 708,
        "data_sample_strategy": "bagging",
        "sub_row": 0.8234,
        "verbose": -1,
1511
1512
    }
    dtrain = lgb.Dataset(data=X, label=y)
1513
    gbm = lgb.train(params=params, train_set=dtrain, num_boost_round=3)
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651

    model_txt_from_memory = gbm.model_to_string()
    model_file = tmp_path / "out.model"
    gbm.save_model(filename=model_file)
    with open(model_file, "r") as f:
        model_txt_from_file = f.read()

    assert model_txt_from_memory == model_txt_from_file

    # entries whose values should reflect params passed to lgb.train()
    non_default_param_entries = [
        "[objective: mape]",
        # 'l1' was passed in with alias 'mae'
        "[metric: l2,l1]",
        "[data_sample_strategy: bagging]",
        "[seed: 708]",
        # NOTE: this was passed in with alias 'sub_row'
        "[bagging_fraction: 0.8234]",
        "[num_iterations: 3]",
    ]

    # entries with default values of params
    default_param_entries = [
        "[boosting: gbdt]",
        "[tree_learner: serial]",
        "[data: ]",
        "[valid: ]",
        "[learning_rate: 0.1]",
        "[num_leaves: 31]",
        "[num_threads: 0]",
        "[deterministic: 0]",
        "[histogram_pool_size: -1]",
        "[max_depth: -1]",
        "[min_data_in_leaf: 20]",
        "[min_sum_hessian_in_leaf: 0.001]",
        "[pos_bagging_fraction: 1]",
        "[neg_bagging_fraction: 1]",
        "[bagging_freq: 0]",
        "[bagging_seed: 15415]",
        "[feature_fraction: 1]",
        "[feature_fraction_bynode: 1]",
        "[feature_fraction_seed: 32671]",
        "[extra_trees: 0]",
        "[extra_seed: 6642]",
        "[early_stopping_round: 0]",
        "[first_metric_only: 0]",
        "[max_delta_step: 0]",
        "[lambda_l1: 0]",
        "[lambda_l2: 0]",
        "[linear_lambda: 0]",
        "[min_gain_to_split: 0]",
        "[drop_rate: 0.1]",
        "[max_drop: 50]",
        "[skip_drop: 0.5]",
        "[xgboost_dart_mode: 0]",
        "[uniform_drop: 0]",
        "[drop_seed: 20623]",
        "[top_rate: 0.2]",
        "[other_rate: 0.1]",
        "[min_data_per_group: 100]",
        "[max_cat_threshold: 32]",
        "[cat_l2: 10]",
        "[cat_smooth: 10]",
        "[max_cat_to_onehot: 4]",
        "[top_k: 20]",
        "[monotone_constraints: ]",
        "[monotone_constraints_method: basic]",
        "[monotone_penalty: 0]",
        "[feature_contri: ]",
        "[forcedsplits_filename: ]",
        "[refit_decay_rate: 0.9]",
        "[cegb_tradeoff: 1]",
        "[cegb_penalty_split: 0]",
        "[cegb_penalty_feature_lazy: ]",
        "[cegb_penalty_feature_coupled: ]",
        "[path_smooth: 0]",
        "[interaction_constraints: ]",
        "[verbosity: -1]",
        "[saved_feature_importance_type: 0]",
        "[use_quantized_grad: 0]",
        "[num_grad_quant_bins: 4]",
        "[quant_train_renew_leaf: 0]",
        "[stochastic_rounding: 1]",
        "[linear_tree: 0]",
        "[max_bin: 255]",
        "[max_bin_by_feature: ]",
        "[min_data_in_bin: 3]",
        "[bin_construct_sample_cnt: 200000]",
        "[data_random_seed: 2350]",
        "[is_enable_sparse: 1]",
        "[enable_bundle: 1]",
        "[use_missing: 1]",
        "[zero_as_missing: 0]",
        "[feature_pre_filter: 1]",
        "[pre_partition: 0]",
        "[two_round: 0]",
        "[header: 0]",
        "[label_column: ]",
        "[weight_column: ]",
        "[group_column: ]",
        "[ignore_column: ]",
        "[categorical_feature: ]",
        "[forcedbins_filename: ]",
        "[precise_float_parser: 0]",
        "[parser_config_file: ]",
        "[objective_seed: 4309]",
        "[num_class: 1]",
        "[is_unbalance: 0]",
        "[scale_pos_weight: 1]",
        "[sigmoid: 1]",
        "[boost_from_average: 1]",
        "[reg_sqrt: 0]",
        "[alpha: 0.9]",
        "[fair_c: 1]",
        "[poisson_max_delta_step: 0.7]",
        "[tweedie_variance_power: 1.5]",
        "[lambdarank_truncation_level: 30]",
        "[lambdarank_norm: 1]",
        "[label_gain: ]",
        "[lambdarank_position_bias_regularization: 0]",
        "[eval_at: ]",
        "[multi_error_top_k: 1]",
        "[auc_mu_weights: ]",
        "[num_machines: 1]",
        "[local_listen_port: 12400]",
        "[time_out: 120]",
        "[machine_list_filename: ]",
        "[machines: ]",
        "[gpu_platform_id: -1]",
        "[gpu_device_id: -1]",
        "[num_gpu: 1]",
    ]
    all_param_entries = non_default_param_entries + default_param_entries

    # add device-specific entries
    #
    # passed-in force_col_wise / force_row_wise parameters are ignored on CUDA and GPU builds...
    # https://github.com/microsoft/LightGBM/blob/1d7ee63686272bceffd522284127573b511df6be/src/io/config.cpp#L375-L377
1652
1653
1654
1655
    if getenv("TASK", "") == "cuda":
        device_entries = ["[force_col_wise: 0]", "[force_row_wise: 1]", "[device_type: cuda]", "[gpu_use_dp: 1]"]
    elif getenv("TASK", "") == "gpu":
        device_entries = ["[force_col_wise: 1]", "[force_row_wise: 0]", "[device_type: gpu]", "[gpu_use_dp: 0]"]
1656
    else:
1657
        device_entries = ["[force_col_wise: 0]", "[force_row_wise: 0]", "[device_type: cpu]", "[gpu_use_dp: 0]"]
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679

    all_param_entries += device_entries

    # check that model text has all expected param entries
    for param_str in all_param_entries:
        assert param_str in model_txt_from_file
        assert param_str in model_txt_from_memory

    # since Booster.model_to_string() is used when pickling, check that parameters all
    # roundtrip pickling successfully too
    gbm_pkl = pickle_and_unpickle_object(gbm, serializer="joblib")
    model_txt_from_memory = gbm_pkl.model_to_string()
    model_file = tmp_path / "out-pkl.model"
    gbm_pkl.save_model(filename=model_file)
    with open(model_file, "r") as f:
        model_txt_from_file = f.read()

    for param_str in all_param_entries:
        assert param_str in model_txt_from_file
        assert param_str in model_txt_from_memory


1680
1681
1682
def test_pandas_categorical():
    pd = pytest.importorskip("pandas")
    np.random.seed(42)  # sometimes there is no difference how cols are treated (cat or not cat)
1683
1684
1685
1686
1687
1688
1689
1690
1691
    X = pd.DataFrame(
        {
            "A": np.random.permutation(["a", "b", "c", "d"] * 75),  # str
            "B": np.random.permutation([1, 2, 3] * 100),  # int
            "C": np.random.permutation([0.1, 0.2, -0.1, -0.1, 0.2] * 60),  # float
            "D": np.random.permutation([True, False] * 150),  # bool
            "E": pd.Categorical(np.random.permutation(["z", "y", "x", "w", "v"] * 60), ordered=True),
        }
    )  # str and ordered categorical
1692
    y = np.random.permutation([0, 1] * 150)
1693
1694
1695
1696
1697
1698
1699
1700
1701
    X_test = pd.DataFrame(
        {
            "A": np.random.permutation(["a", "b", "e"] * 20),  # unseen category
            "B": np.random.permutation([1, 3] * 30),
            "C": np.random.permutation([0.1, -0.1, 0.2, 0.2] * 15),
            "D": np.random.permutation([True, False] * 30),
            "E": pd.Categorical(np.random.permutation(["z", "y"] * 30), ordered=True),
        }
    )
1702
1703
1704
    np.random.seed()  # reset seed
    cat_cols_actual = ["A", "B", "C", "D"]
    cat_cols_to_store = cat_cols_actual + ["E"]
1705
1706
    X[cat_cols_actual] = X[cat_cols_actual].astype("category")
    X_test[cat_cols_actual] = X_test[cat_cols_actual].astype("category")
1707
    cat_values = [X[col].cat.categories.tolist() for col in cat_cols_to_store]
1708
    params = {"objective": "binary", "metric": "binary_logloss", "verbose": -1}
1709
1710
1711
    lgb_train = lgb.Dataset(X, y)
    gbm0 = lgb.train(params, lgb_train, num_boost_round=10)
    pred0 = gbm0.predict(X_test)
1712
    assert lgb_train.categorical_feature == "auto"
1713
1714
1715
1716
1717
    lgb_train = lgb.Dataset(X, pd.DataFrame(y))  # also test that label can be one-column pd.DataFrame
    gbm1 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=[0])
    pred1 = gbm1.predict(X_test)
    assert lgb_train.categorical_feature == [0]
    lgb_train = lgb.Dataset(X, pd.Series(y))  # also test that label can be pd.Series
1718
    gbm2 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=["A"])
1719
    pred2 = gbm2.predict(X_test)
1720
    assert lgb_train.categorical_feature == ["A"]
1721
    lgb_train = lgb.Dataset(X, y)
1722
    gbm3 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=["A", "B", "C", "D"])
1723
    pred3 = gbm3.predict(X_test)
1724
1725
1726
    assert lgb_train.categorical_feature == ["A", "B", "C", "D"]
    gbm3.save_model("categorical.model")
    gbm4 = lgb.Booster(model_file="categorical.model")
1727
1728
    pred4 = gbm4.predict(X_test)
    model_str = gbm4.model_to_string()
1729
    gbm4.model_from_string(model_str)
1730
1731
1732
1733
    pred5 = gbm4.predict(X_test)
    gbm5 = lgb.Booster(model_str=model_str)
    pred6 = gbm5.predict(X_test)
    lgb_train = lgb.Dataset(X, y)
1734
    gbm6 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=["A", "B", "C", "D", "E"])
1735
    pred7 = gbm6.predict(X_test)
1736
    assert lgb_train.categorical_feature == ["A", "B", "C", "D", "E"]
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
    lgb_train = lgb.Dataset(X, y)
    gbm7 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=[])
    pred8 = gbm7.predict(X_test)
    assert lgb_train.categorical_feature == []
    with pytest.raises(AssertionError):
        np.testing.assert_allclose(pred0, pred1)
    with pytest.raises(AssertionError):
        np.testing.assert_allclose(pred0, pred2)
    np.testing.assert_allclose(pred1, pred2)
    np.testing.assert_allclose(pred0, pred3)
    np.testing.assert_allclose(pred0, pred4)
    np.testing.assert_allclose(pred0, pred5)
    np.testing.assert_allclose(pred0, pred6)
    with pytest.raises(AssertionError):
        np.testing.assert_allclose(pred0, pred7)  # ordered cat features aren't treated as cat features by default
    with pytest.raises(AssertionError):
1753
        np.testing.assert_allclose(pred0, pred8)
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
    assert gbm0.pandas_categorical == cat_values
    assert gbm1.pandas_categorical == cat_values
    assert gbm2.pandas_categorical == cat_values
    assert gbm3.pandas_categorical == cat_values
    assert gbm4.pandas_categorical == cat_values
    assert gbm5.pandas_categorical == cat_values
    assert gbm6.pandas_categorical == cat_values
    assert gbm7.pandas_categorical == cat_values


def test_pandas_sparse():
    pd = pytest.importorskip("pandas")
1766
1767
1768
1769
1770
1771
1772
    X = pd.DataFrame(
        {
            "A": pd.arrays.SparseArray(np.random.permutation([0, 1, 2] * 100)),
            "B": pd.arrays.SparseArray(np.random.permutation([0.0, 0.1, 0.2, -0.1, 0.2] * 60)),
            "C": pd.arrays.SparseArray(np.random.permutation([True, False] * 150)),
        }
    )
1773
    y = pd.Series(pd.arrays.SparseArray(np.random.permutation([0, 1] * 150)))
1774
1775
1776
1777
1778
1779
1780
    X_test = pd.DataFrame(
        {
            "A": pd.arrays.SparseArray(np.random.permutation([0, 2] * 30)),
            "B": pd.arrays.SparseArray(np.random.permutation([0.0, 0.1, 0.2, -0.1] * 15)),
            "C": pd.arrays.SparseArray(np.random.permutation([True, False] * 30)),
        }
    )
1781
1782
    for dtype in pd.concat([X.dtypes, X_test.dtypes, pd.Series(y.dtypes)]):
        assert pd.api.types.is_sparse(dtype)
1783
    params = {"objective": "binary", "verbose": -1}
1784
1785
1786
    lgb_train = lgb.Dataset(X, y)
    gbm = lgb.train(params, lgb_train, num_boost_round=10)
    pred_sparse = gbm.predict(X_test, raw_score=True)
1787
    if hasattr(X_test, "sparse"):
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
        pred_dense = gbm.predict(X_test.sparse.to_dense(), raw_score=True)
    else:
        pred_dense = gbm.predict(X_test.to_dense(), raw_score=True)
    np.testing.assert_allclose(pred_sparse, pred_dense)


def test_reference_chain():
    X = np.random.normal(size=(100, 2))
    y = np.random.normal(size=100)
    tmp_dat = lgb.Dataset(X, y)
    # take subsets and train
    tmp_dat_train = tmp_dat.subset(np.arange(80))
    tmp_dat_val = tmp_dat.subset(np.arange(80, 100)).subset(np.arange(18))
1801
    params = {"objective": "regression_l2", "metric": "rmse"}
1802
    evals_result = {}
1803
1804
1805
1806
1807
    lgb.train(
        params,
        tmp_dat_train,
        num_boost_round=20,
        valid_sets=[tmp_dat_train, tmp_dat_val],
1808
        callbacks=[lgb.record_evaluation(evals_result)],
1809
    )
1810
1811
    assert len(evals_result["training"]["rmse"]) == 20
    assert len(evals_result["valid_1"]["rmse"]) == 20
1812
1813
1814
1815
1816
1817


def test_contribs():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1818
1819
1820
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
1821
1822
1823
1824
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)

1825
1826
1827
1828
    assert (
        np.linalg.norm(gbm.predict(X_test, raw_score=True) - np.sum(gbm.predict(X_test, pred_contrib=True), axis=1))
        < 1e-4
    )
1829
1830
1831
1832
1833
1834


def test_contribs_sparse():
    n_features = 20
    n_samples = 100
    # generate CSR sparse dataset
1835
1836
1837
    X, y = make_multilabel_classification(
        n_samples=n_samples, sparse=True, n_features=n_features, n_classes=1, n_labels=2
    )
1838
1839
1840
    y = y.flatten()
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1841
1842
        "objective": "binary",
        "verbose": -1,
1843
1844
1845
1846
1847
1848
1849
1850
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)
    contribs_csr = gbm.predict(X_test, pred_contrib=True)
    assert isspmatrix_csr(contribs_csr)
    # convert data to dense and get back same contribs
    contribs_dense = gbm.predict(X_test.toarray(), pred_contrib=True)
    # validate the values are the same
1851
    if platform.machine() == "aarch64":
1852
1853
1854
        np.testing.assert_allclose(contribs_csr.toarray(), contribs_dense, rtol=1, atol=1e-12)
    else:
        np.testing.assert_allclose(contribs_csr.toarray(), contribs_dense)
1855
    assert np.linalg.norm(gbm.predict(X_test, raw_score=True) - np.sum(contribs_dense, axis=1)) < 1e-4
1856
1857
1858
1859
1860
    # validate using CSC matrix
    X_test_csc = X_test.tocsc()
    contribs_csc = gbm.predict(X_test_csc, pred_contrib=True)
    assert isspmatrix_csc(contribs_csc)
    # validate the values are the same
1861
    if platform.machine() == "aarch64":
1862
1863
1864
        np.testing.assert_allclose(contribs_csc.toarray(), contribs_dense, rtol=1, atol=1e-12)
    else:
        np.testing.assert_allclose(contribs_csc.toarray(), contribs_dense)
1865
1866
1867
1868
1869
1870
1871


def test_contribs_sparse_multiclass():
    n_features = 20
    n_samples = 100
    n_labels = 4
    # generate CSR sparse dataset
1872
1873
1874
    X, y = make_multilabel_classification(
        n_samples=n_samples, sparse=True, n_features=n_features, n_classes=1, n_labels=n_labels
    )
1875
1876
1877
    y = y.flatten()
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1878
1879
1880
        "objective": "multiclass",
        "num_class": n_labels,
        "verbose": -1,
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)
    contribs_csr = gbm.predict(X_test, pred_contrib=True)
    assert isinstance(contribs_csr, list)
    for perclass_contribs_csr in contribs_csr:
        assert isspmatrix_csr(perclass_contribs_csr)
    # convert data to dense and get back same contribs
    contribs_dense = gbm.predict(X_test.toarray(), pred_contrib=True)
    # validate the values are the same
1891
    contribs_csr_array = np.swapaxes(np.array([sparse_array.toarray() for sparse_array in contribs_csr]), 0, 1)
1892
1893
1894
1895
    contribs_csr_arr_re = contribs_csr_array.reshape(
        (contribs_csr_array.shape[0], contribs_csr_array.shape[1] * contribs_csr_array.shape[2])
    )
    if platform.machine() == "aarch64":
1896
1897
1898
        np.testing.assert_allclose(contribs_csr_arr_re, contribs_dense, rtol=1, atol=1e-12)
    else:
        np.testing.assert_allclose(contribs_csr_arr_re, contribs_dense)
1899
1900
1901
1902
1903
1904
1905
1906
1907
    contribs_dense_re = contribs_dense.reshape(contribs_csr_array.shape)
    assert np.linalg.norm(gbm.predict(X_test, raw_score=True) - np.sum(contribs_dense_re, axis=2)) < 1e-4
    # validate using CSC matrix
    X_test_csc = X_test.tocsc()
    contribs_csc = gbm.predict(X_test_csc, pred_contrib=True)
    assert isinstance(contribs_csc, list)
    for perclass_contribs_csc in contribs_csc:
        assert isspmatrix_csc(perclass_contribs_csc)
    # validate the values are the same
1908
    contribs_csc_array = np.swapaxes(np.array([sparse_array.toarray() for sparse_array in contribs_csc]), 0, 1)
1909
1910
1911
1912
    contribs_csc_array = contribs_csc_array.reshape(
        (contribs_csc_array.shape[0], contribs_csc_array.shape[1] * contribs_csc_array.shape[2])
    )
    if platform.machine() == "aarch64":
1913
1914
1915
        np.testing.assert_allclose(contribs_csc_array, contribs_dense, rtol=1, atol=1e-12)
    else:
        np.testing.assert_allclose(contribs_csc_array, contribs_dense)
1916
1917


1918
@pytest.mark.skipif(psutil.virtual_memory().available / 1024 / 1024 / 1024 < 3, reason="not enough RAM")
1919
def test_int32_max_sparse_contribs():
1920
    params = {"objective": "binary"}
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
    train_features = np.random.rand(100, 1000)
    train_targets = [0] * 50 + [1] * 50
    lgb_train = lgb.Dataset(train_features, train_targets)
    gbm = lgb.train(params, lgb_train, num_boost_round=2)
    csr_input_shape = (3000000, 1000)
    test_features = csr_matrix(csr_input_shape)
    for i in range(0, csr_input_shape[0], csr_input_shape[0] // 6):
        for j in range(0, 1000, 100):
            test_features[i, j] = random.random()
    y_pred_csr = gbm.predict(test_features, pred_contrib=True)
    # Note there is an extra column added to the output for the expected value
    csr_output_shape = (csr_input_shape[0], csr_input_shape[1] + 1)
    assert y_pred_csr.shape == csr_output_shape
    y_pred_csc = gbm.predict(test_features.tocsc(), pred_contrib=True)
    # Note output CSC shape should be same as CSR output shape
    assert y_pred_csc.shape == csr_output_shape


def test_sliced_data():
    def train_and_get_predictions(features, labels):
        dataset = lgb.Dataset(features, label=labels)
        lgb_params = {
1943
1944
1945
            "application": "binary",
            "verbose": -1,
            "min_data": 5,
1946
        }
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
        gbm = lgb.train(
            params=lgb_params,
            train_set=dataset,
            num_boost_round=10,
        )
        return gbm.predict(features)

    num_samples = 100
    features = np.random.rand(num_samples, 5)
    positive_samples = int(num_samples * 0.25)
1957
1958
1959
    labels = np.append(
        np.ones(positive_samples, dtype=np.float32), np.zeros(num_samples - positive_samples, dtype=np.float32)
    )
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
    # test sliced labels
    origin_pred = train_and_get_predictions(features, labels)
    stacked_labels = np.column_stack((labels, np.ones(num_samples, dtype=np.float32)))
    sliced_labels = stacked_labels[:, 0]
    sliced_pred = train_and_get_predictions(features, sliced_labels)
    np.testing.assert_allclose(origin_pred, sliced_pred)
    # append some columns
    stacked_features = np.column_stack((np.ones(num_samples, dtype=np.float32), features))
    stacked_features = np.column_stack((np.ones(num_samples, dtype=np.float32), stacked_features))
    stacked_features = np.column_stack((stacked_features, np.ones(num_samples, dtype=np.float32)))
    stacked_features = np.column_stack((stacked_features, np.ones(num_samples, dtype=np.float32)))
    # append some rows
    stacked_features = np.concatenate((np.ones(9, dtype=np.float32).reshape((1, 9)), stacked_features), axis=0)
    stacked_features = np.concatenate((np.ones(9, dtype=np.float32).reshape((1, 9)), stacked_features), axis=0)
    stacked_features = np.concatenate((stacked_features, np.ones(9, dtype=np.float32).reshape((1, 9))), axis=0)
    stacked_features = np.concatenate((stacked_features, np.ones(9, dtype=np.float32).reshape((1, 9))), axis=0)
    # test sliced 2d matrix
    sliced_features = stacked_features[2:102, 2:7]
    assert np.all(sliced_features == features)
    sliced_pred = train_and_get_predictions(sliced_features, sliced_labels)
    np.testing.assert_allclose(origin_pred, sliced_pred)
    # test sliced CSR
    stacked_csr = csr_matrix(stacked_features)
    sliced_csr = stacked_csr[2:102, 2:7]
    assert np.all(sliced_csr == features)
    sliced_pred = train_and_get_predictions(sliced_csr, sliced_labels)
    np.testing.assert_allclose(origin_pred, sliced_pred)


def test_init_with_subset():
    data = np.random.random((50, 2))
    y = [1] * 25 + [0] * 25
    lgb_train = lgb.Dataset(data, y, free_raw_data=False)
    subset_index_1 = np.random.choice(np.arange(50), 30, replace=False)
    subset_data_1 = lgb_train.subset(subset_index_1)
    subset_index_2 = np.random.choice(np.arange(50), 20, replace=False)
    subset_data_2 = lgb_train.subset(subset_index_2)
1997
1998
1999
    params = {"objective": "binary", "verbose": -1}
    init_gbm = lgb.train(params=params, train_set=subset_data_1, num_boost_round=10, keep_training_booster=True)
    lgb.train(params=params, train_set=subset_data_2, num_boost_round=10, init_model=init_gbm)
2000
2001
2002
2003
    assert lgb_train.get_data().shape[0] == 50
    assert subset_data_1.get_data().shape[0] == 30
    assert subset_data_2.get_data().shape[0] == 20
    lgb_train.save_binary("lgb_train_data.bin")
2004
    lgb_train_from_file = lgb.Dataset("lgb_train_data.bin", free_raw_data=False)
2005
2006
    subset_data_3 = lgb_train_from_file.subset(subset_index_1)
    subset_data_4 = lgb_train_from_file.subset(subset_index_2)
2007
    init_gbm_2 = lgb.train(params=params, train_set=subset_data_3, num_boost_round=10, keep_training_booster=True)
2008
    with np.testing.assert_raises_regex(lgb.basic.LightGBMError, "Unknown format of training data"):
2009
        lgb.train(params=params, train_set=subset_data_4, num_boost_round=10, init_model=init_gbm_2)
2010
2011
2012
2013
2014
    assert lgb_train_from_file.get_data() == "lgb_train_data.bin"
    assert subset_data_3.get_data() == "lgb_train_data.bin"
    assert subset_data_4.get_data() == "lgb_train_data.bin"


2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
def test_training_on_constructed_subset_without_params():
    X = np.random.random((100, 10))
    y = np.random.random(100)
    lgb_data = lgb.Dataset(X, y)
    subset_indices = [1, 2, 3, 4]
    subset = lgb_data.subset(subset_indices).construct()
    bst = lgb.train({}, subset, num_boost_round=1)
    assert subset.get_params() == {}
    assert subset.num_data() == len(subset_indices)
    assert bst.current_iteration() == 1


2027
2028
2029
2030
2031
2032
def generate_trainset_for_monotone_constraints_tests(x3_to_category=True):
    number_of_dpoints = 3000
    x1_positively_correlated_with_y = np.random.random(size=number_of_dpoints)
    x2_negatively_correlated_with_y = np.random.random(size=number_of_dpoints)
    x3_negatively_correlated_with_y = np.random.random(size=number_of_dpoints)
    x = np.column_stack(
2033
2034
        (
            x1_positively_correlated_with_y,
2035
            x2_negatively_correlated_with_y,
2036
2037
2038
            categorize(x3_negatively_correlated_with_y) if x3_to_category else x3_negatively_correlated_with_y,
        )
    )
2039
2040

    zs = np.random.normal(loc=0.0, scale=0.01, size=number_of_dpoints)
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
    scales = 10.0 * (np.random.random(6) + 0.5)
    y = (
        scales[0] * x1_positively_correlated_with_y
        + np.sin(scales[1] * np.pi * x1_positively_correlated_with_y)
        - scales[2] * x2_negatively_correlated_with_y
        - np.cos(scales[3] * np.pi * x2_negatively_correlated_with_y)
        - scales[4] * x3_negatively_correlated_with_y
        - np.cos(scales[5] * np.pi * x3_negatively_correlated_with_y)
        + zs
    )
2051
2052
2053
    categorical_features = []
    if x3_to_category:
        categorical_features = [2]
2054
    return lgb.Dataset(x, label=y, categorical_feature=categorical_features, free_raw_data=False)
2055
2056


2057
@pytest.mark.skipif(getenv("TASK", "") == "cuda", reason="Monotone constraints are not yet supported by CUDA version")
2058
2059
@pytest.mark.parametrize("test_with_categorical_variable", [True, False])
def test_monotone_constraints(test_with_categorical_variable):
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
    def is_increasing(y):
        return (np.diff(y) >= 0.0).all()

    def is_decreasing(y):
        return (np.diff(y) <= 0.0).all()

    def is_non_monotone(y):
        return (np.diff(y) < 0.0).any() and (np.diff(y) > 0.0).any()

    def is_correctly_constrained(learner, x3_to_category=True):
        iterations = 10
        n = 1000
        variable_x = np.linspace(0, 1, n).reshape((n, 1))
        fixed_xs_values = np.linspace(0, 1, n)
        for i in range(iterations):
            fixed_x = fixed_xs_values[i] * np.ones((n, 1))
            monotonically_increasing_x = np.column_stack((variable_x, fixed_x, fixed_x))
            monotonically_increasing_y = learner.predict(monotonically_increasing_x)
            monotonically_decreasing_x = np.column_stack((fixed_x, variable_x, fixed_x))
            monotonically_decreasing_y = learner.predict(monotonically_decreasing_x)
2080
2081
2082
2083
2084
2085
2086
            non_monotone_x = np.column_stack(
                (
                    fixed_x,
                    fixed_x,
                    categorize(variable_x) if x3_to_category else variable_x,
                )
            )
2087
            non_monotone_y = learner.predict(non_monotone_x)
2088
2089
2090
2091
2092
            if not (
                is_increasing(monotonically_increasing_y)
                and is_decreasing(monotonically_decreasing_y)
                and is_non_monotone(non_monotone_y)
            ):
2093
                return False
2094
        return True
2095

2096
2097
2098
2099
2100
2101
2102
2103
    def are_interactions_enforced(gbm, feature_sets):
        def parse_tree_features(gbm):
            # trees start at position 1.
            tree_str = gbm.model_to_string().split("Tree")[1:]
            feature_sets = []
            for tree in tree_str:
                # split_features are in 4th line.
                features = tree.splitlines()[3].split("=")[1].split(" ")
2104
                features = {f"Column_{f}" for f in features}
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
                feature_sets.append(features)
            return np.array(feature_sets)

        def has_interaction(treef):
            n = 0
            for fs in feature_sets:
                if len(treef.intersection(fs)) > 0:
                    n += 1
            return n > 1

        tree_features = parse_tree_features(gbm)
2116
        has_interaction_flag = np.array([has_interaction(treef) for treef in tree_features])
2117
2118
2119

        return not has_interaction_flag.any()

2120
    trainset = generate_trainset_for_monotone_constraints_tests(test_with_categorical_variable)
2121
    for test_with_interaction_constraints in [True, False]:
2122
2123
2124
2125
        error_msg = (
            "Model not correctly constrained "
            f"(test_with_interaction_constraints={test_with_interaction_constraints})"
        )
2126
        for monotone_constraints_method in ["basic", "intermediate", "advanced"]:
2127
            params = {
2128
2129
2130
                "min_data": 20,
                "num_leaves": 20,
                "monotone_constraints": [1, -1, 0],
2131
                "monotone_constraints_method": monotone_constraints_method,
2132
                "use_missing": False,
2133
            }
2134
2135
            if test_with_interaction_constraints:
                params["interaction_constraints"] = [[0], [1], [2]]
2136
            constrained_model = lgb.train(params, trainset)
2137
            assert is_correctly_constrained(constrained_model, test_with_categorical_variable), error_msg
2138
2139
2140
            if test_with_interaction_constraints:
                feature_sets = [["Column_0"], ["Column_1"], "Column_2"]
                assert are_interactions_enforced(constrained_model, feature_sets)
2141
2142


2143
@pytest.mark.skipif(getenv("TASK", "") == "cuda", reason="Monotone constraints are not yet supported by CUDA version")
2144
2145
2146
2147
2148
2149
2150
2151
def test_monotone_penalty():
    def are_first_splits_non_monotone(tree, n, monotone_constraints):
        if n <= 0:
            return True
        if "leaf_value" in tree:
            return True
        if monotone_constraints[tree["split_feature"]] != 0:
            return False
2152
2153
2154
        return are_first_splits_non_monotone(
            tree["left_child"], n - 1, monotone_constraints
        ) and are_first_splits_non_monotone(tree["right_child"], n - 1, monotone_constraints)
2155
2156
2157
2158
2159
2160

    def are_there_monotone_splits(tree, monotone_constraints):
        if "leaf_value" in tree:
            return False
        if monotone_constraints[tree["split_feature"]] != 0:
            return True
2161
2162
2163
        return are_there_monotone_splits(tree["left_child"], monotone_constraints) or are_there_monotone_splits(
            tree["right_child"], monotone_constraints
        )
2164
2165
2166
2167
2168
2169

    max_depth = 5
    monotone_constraints = [1, -1, 0]
    penalization_parameter = 2.0
    trainset = generate_trainset_for_monotone_constraints_tests(x3_to_category=False)
    for monotone_constraints_method in ["basic", "intermediate", "advanced"]:
2170
        params = {
2171
2172
2173
            "max_depth": max_depth,
            "monotone_constraints": monotone_constraints,
            "monotone_penalty": penalization_parameter,
2174
            "monotone_constraints_method": monotone_constraints_method,
2175
        }
2176
2177
2178
        constrained_model = lgb.train(params, trainset, 10)
        dumped_model = constrained_model.dump_model()["tree_info"]
        for tree in dumped_model:
2179
2180
2181
            assert are_first_splits_non_monotone(
                tree["tree_structure"], int(penalization_parameter), monotone_constraints
            )
2182
2183
2184
2185
            assert are_there_monotone_splits(tree["tree_structure"], monotone_constraints)


# test if a penalty as high as the depth indeed prohibits all monotone splits
2186
@pytest.mark.skipif(getenv("TASK", "") == "cuda", reason="Monotone constraints are not yet supported by CUDA version")
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
def test_monotone_penalty_max():
    max_depth = 5
    monotone_constraints = [1, -1, 0]
    penalization_parameter = max_depth
    trainset_constrained_model = generate_trainset_for_monotone_constraints_tests(x3_to_category=False)
    x = trainset_constrained_model.data
    y = trainset_constrained_model.label
    x3_negatively_correlated_with_y = x[:, 2]
    trainset_unconstrained_model = lgb.Dataset(x3_negatively_correlated_with_y.reshape(-1, 1), label=y)
    params_constrained_model = {
2197
2198
        "monotone_constraints": monotone_constraints,
        "monotone_penalty": penalization_parameter,
2199
2200
2201
2202
2203
2204
2205
2206
2207
        "max_depth": max_depth,
        "gpu_use_dp": True,
    }
    params_unconstrained_model = {
        "max_depth": max_depth,
        "gpu_use_dp": True,
    }

    unconstrained_model = lgb.train(params_unconstrained_model, trainset_unconstrained_model, 10)
2208
    unconstrained_model_predictions = unconstrained_model.predict(x3_negatively_correlated_with_y.reshape(-1, 1))
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225

    for monotone_constraints_method in ["basic", "intermediate", "advanced"]:
        params_constrained_model["monotone_constraints_method"] = monotone_constraints_method
        # The penalization is so high that the first 2 features should not be used here
        constrained_model = lgb.train(params_constrained_model, trainset_constrained_model, 10)

        # Check that a very high penalization is the same as not using the features at all
        np.testing.assert_array_equal(constrained_model.predict(x), unconstrained_model_predictions)


def test_max_bin_by_feature():
    col1 = np.arange(0, 100)[:, np.newaxis]
    col2 = np.zeros((100, 1))
    col2[20:] = 1
    X = np.concatenate([col1, col2], axis=1)
    y = np.arange(0, 100)
    params = {
2226
2227
2228
2229
2230
2231
2232
        "objective": "regression_l2",
        "verbose": -1,
        "num_leaves": 100,
        "min_data_in_leaf": 1,
        "min_sum_hessian_in_leaf": 0,
        "min_data_in_bin": 1,
        "max_bin_by_feature": [100, 2],
2233
2234
2235
2236
    }
    lgb_data = lgb.Dataset(X, label=y)
    est = lgb.train(params, lgb_data, num_boost_round=1)
    assert len(np.unique(est.predict(X))) == 100
2237
    params["max_bin_by_feature"] = [2, 100]
2238
2239
2240
2241
2242
2243
2244
2245
    lgb_data = lgb.Dataset(X, label=y)
    est = lgb.train(params, lgb_data, num_boost_round=1)
    assert len(np.unique(est.predict(X))) == 3


def test_small_max_bin():
    np.random.seed(0)
    y = np.random.choice([0, 1], 100)
2246
    x = np.ones((100, 1))
2247
2248
    x[:30, 0] = -1
    x[60:, 0] = 2
2249
    params = {"objective": "binary", "seed": 0, "min_data_in_leaf": 1, "verbose": -1, "max_bin": 2}
2250
2251
2252
    lgb_x = lgb.Dataset(x, label=y)
    lgb.train(params, lgb_x, num_boost_round=5)
    x[0, 0] = np.nan
2253
    params["max_bin"] = 3
2254
2255
2256
2257
2258
2259
2260
2261
    lgb_x = lgb.Dataset(x, label=y)
    lgb.train(params, lgb_x, num_boost_round=5)
    np.random.seed()  # reset seed


def test_refit():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
2262
    params = {"objective": "binary", "metric": "binary_logloss", "verbose": -1, "min_data": 10}
2263
2264
2265
2266
2267
2268
2269
2270
    lgb_train = lgb.Dataset(X_train, y_train)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)
    err_pred = log_loss(y_test, gbm.predict(X_test))
    new_gbm = gbm.refit(X_test, y_test)
    new_err_pred = log_loss(y_test, new_gbm.predict(X_test))
    assert err_pred > new_err_pred


2271
2272
2273
2274
def test_refit_dataset_params():
    # check refit accepts dataset_params
    X, y = load_breast_cancer(return_X_y=True)
    lgb_train = lgb.Dataset(X, y, init_score=np.zeros(y.size))
2275
    train_params = {"objective": "binary", "verbose": -1, "seed": 123}
2276
2277
2278
2279
    gbm = lgb.train(train_params, lgb_train, num_boost_round=10)
    non_weight_err_pred = log_loss(y, gbm.predict(X))
    refit_weight = np.random.rand(y.shape[0])
    dataset_params = {
2280
2281
2282
        "max_bin": 260,
        "min_data_in_bin": 5,
        "data_random_seed": 123,
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
    }
    new_gbm = gbm.refit(
        data=X,
        label=y,
        weight=refit_weight,
        dataset_params=dataset_params,
        decay_rate=0.0,
    )
    weight_err_pred = log_loss(y, new_gbm.predict(X))
    train_set_params = new_gbm.train_set.get_params()
    stored_weights = new_gbm.train_set.get_weight()
    assert weight_err_pred != non_weight_err_pred
    assert train_set_params["max_bin"] == 260
    assert train_set_params["min_data_in_bin"] == 5
    assert train_set_params["data_random_seed"] == 123
    np.testing.assert_allclose(stored_weights, refit_weight)


2301
@pytest.mark.parametrize("boosting_type", ["rf", "dart"])
2302
2303
2304
def test_mape_for_specific_boosting_types(boosting_type):
    X, y = make_synthetic_regression()
    y = abs(y)
2305
    params = {
2306
2307
2308
2309
2310
2311
2312
        "boosting_type": boosting_type,
        "objective": "mape",
        "verbose": -1,
        "bagging_freq": 1,
        "bagging_fraction": 0.8,
        "feature_fraction": 0.8,
        "boost_from_average": True,
2313
2314
2315
2316
2317
    }
    lgb_train = lgb.Dataset(X, y)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)
    pred = gbm.predict(X)
    pred_mean = pred.mean()
2318
2319
2320
    # the following checks that dart and rf with mape can predict outside the 0-1 range
    # https://github.com/microsoft/LightGBM/issues/1579
    assert pred_mean > 8
2321
2322
2323
2324
2325
2326


def check_constant_features(y_true, expected_pred, more_params):
    X_train = np.ones((len(y_true), 1))
    y_train = np.array(y_true)
    params = {
2327
2328
2329
2330
2331
2332
2333
2334
        "objective": "regression",
        "num_class": 1,
        "verbose": -1,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "boost_from_average": True,
2335
2336
2337
2338
2339
2340
2341
2342
2343
    }
    params.update(more_params)
    lgb_train = lgb.Dataset(X_train, y_train, params=params)
    gbm = lgb.train(params, lgb_train, num_boost_round=2)
    pred = gbm.predict(X_train)
    assert np.allclose(pred, expected_pred)


def test_constant_features_regression():
2344
    params = {"objective": "regression"}
2345
2346
2347
2348
2349
2350
    check_constant_features([0.0, 10.0, 0.0, 10.0], 5.0, params)
    check_constant_features([0.0, 1.0, 2.0, 3.0], 1.5, params)
    check_constant_features([-1.0, 1.0, -2.0, 2.0], 0.0, params)


def test_constant_features_binary():
2351
    params = {"objective": "binary"}
2352
2353
2354
2355
2356
    check_constant_features([0.0, 10.0, 0.0, 10.0], 0.5, params)
    check_constant_features([0.0, 1.0, 2.0, 3.0], 0.75, params)


def test_constant_features_multiclass():
2357
    params = {"objective": "multiclass", "num_class": 3}
2358
2359
2360
2361
2362
    check_constant_features([0.0, 1.0, 2.0, 0.0], [0.5, 0.25, 0.25], params)
    check_constant_features([0.0, 1.0, 2.0, 1.0], [0.25, 0.5, 0.25], params)


def test_constant_features_multiclassova():
2363
    params = {"objective": "multiclassova", "num_class": 3}
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
    check_constant_features([0.0, 1.0, 2.0, 0.0], [0.5, 0.25, 0.25], params)
    check_constant_features([0.0, 1.0, 2.0, 1.0], [0.25, 0.5, 0.25], params)


def test_fpreproc():
    def preprocess_data(dtrain, dtest, params):
        train_data = dtrain.construct().get_data()
        test_data = dtest.construct().get_data()
        train_data[:, 0] += 1
        test_data[:, 0] += 1
        dtrain.label[-5:] = 3
        dtest.label[-5:] = 3
        dtrain = lgb.Dataset(train_data, dtrain.label)
        dtest = lgb.Dataset(test_data, dtest.label, reference=dtrain)
2378
        params["num_class"] = 4
2379
2380
2381
2382
        return dtrain, dtest, params

    X, y = load_iris(return_X_y=True)
    dataset = lgb.Dataset(X, y, free_raw_data=False)
2383
    params = {"objective": "multiclass", "num_class": 3, "verbose": -1}
2384
    results = lgb.cv(params, dataset, num_boost_round=10, fpreproc=preprocess_data)
2385
2386
    assert "valid multi_logloss-mean" in results
    assert len(results["valid multi_logloss-mean"]) == 10
2387
2388
2389
2390
2391


def test_metrics():
    X, y = load_digits(n_class=2, return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
2392
2393
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_valid = lgb.Dataset(X_test, y_test, reference=lgb_train)
2394
2395

    evals_result = {}
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
    params_dummy_obj_verbose = {"verbose": -1, "objective": dummy_obj}
    params_obj_verbose = {"objective": "binary", "verbose": -1}
    params_obj_metric_log_verbose = {"objective": "binary", "metric": "binary_logloss", "verbose": -1}
    params_obj_metric_err_verbose = {"objective": "binary", "metric": "binary_error", "verbose": -1}
    params_obj_metric_inv_verbose = {"objective": "binary", "metric": "invalid_metric", "verbose": -1}
    params_obj_metric_quant_verbose = {"objective": "regression", "metric": "quantile", "verbose": 2}
    params_obj_metric_multi_verbose = {
        "objective": "binary",
        "metric": ["binary_logloss", "binary_error"],
        "verbose": -1,
    }
    params_obj_metric_none_verbose = {"objective": "binary", "metric": "None", "verbose": -1}
    params_dummy_obj_metric_log_verbose = {"objective": dummy_obj, "metric": "binary_logloss", "verbose": -1}
    params_dummy_obj_metric_err_verbose = {"objective": dummy_obj, "metric": "binary_error", "verbose": -1}
    params_dummy_obj_metric_inv_verbose = {"objective": dummy_obj, "metric_types": "invalid_metric", "verbose": -1}
    params_dummy_obj_metric_multi_verbose = {
        "objective": dummy_obj,
        "metric": ["binary_logloss", "binary_error"],
        "verbose": -1,
    }
    params_dummy_obj_metric_none_verbose = {"objective": dummy_obj, "metric": "None", "verbose": -1}
2417
2418

    def get_cv_result(params=params_obj_verbose, **kwargs):
2419
        return lgb.cv(params, lgb_train, num_boost_round=2, **kwargs)
2420
2421

    def train_booster(params=params_obj_verbose, **kwargs):
2422
2423
2424
2425
2426
2427
        lgb.train(
            params,
            lgb_train,
            num_boost_round=2,
            valid_sets=[lgb_valid],
            callbacks=[lgb.record_evaluation(evals_result)],
2428
            **kwargs,
2429
        )
2430

2431
    # no custom objective, no feval
2432
2433
2434
    # default metric
    res = get_cv_result()
    assert len(res) == 2
2435
    assert "valid binary_logloss-mean" in res
2436
2437
2438
2439

    # non-default metric in params
    res = get_cv_result(params=params_obj_metric_err_verbose)
    assert len(res) == 2
2440
    assert "valid binary_error-mean" in res
2441
2442

    # default metric in args
2443
    res = get_cv_result(metrics="binary_logloss")
2444
    assert len(res) == 2
2445
    assert "valid binary_logloss-mean" in res
2446
2447

    # non-default metric in args
2448
    res = get_cv_result(metrics="binary_error")
2449
    assert len(res) == 2
2450
    assert "valid binary_error-mean" in res
2451
2452

    # metric in args overwrites one in params
2453
    res = get_cv_result(params=params_obj_metric_inv_verbose, metrics="binary_error")
2454
    assert len(res) == 2
2455
    assert "valid binary_error-mean" in res
2456

2457
2458
2459
    # metric in args overwrites one in params
    res = get_cv_result(params=params_obj_metric_quant_verbose)
    assert len(res) == 2
2460
    assert "valid quantile-mean" in res
2461

2462
2463
2464
    # multiple metrics in params
    res = get_cv_result(params=params_obj_metric_multi_verbose)
    assert len(res) == 4
2465
2466
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
2467
2468

    # multiple metrics in args
2469
    res = get_cv_result(metrics=["binary_logloss", "binary_error"])
2470
    assert len(res) == 4
2471
2472
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
2473
2474

    # remove default metric by 'None' in list
2475
    res = get_cv_result(metrics=["None"])
2476
2477
2478
    assert len(res) == 0

    # remove default metric by 'None' aliases
2479
    for na_alias in ("None", "na", "null", "custom"):
2480
2481
2482
        res = get_cv_result(metrics=na_alias)
        assert len(res) == 0

2483
    # custom objective, no feval
2484
    # no default metric
2485
    res = get_cv_result(params=params_dummy_obj_verbose)
2486
2487
2488
    assert len(res) == 0

    # metric in params
2489
    res = get_cv_result(params=params_dummy_obj_metric_err_verbose)
2490
    assert len(res) == 2
2491
    assert "valid binary_error-mean" in res
2492
2493

    # metric in args
2494
    res = get_cv_result(params=params_dummy_obj_verbose, metrics="binary_error")
2495
    assert len(res) == 2
2496
    assert "valid binary_error-mean" in res
2497
2498

    # metric in args overwrites its' alias in params
2499
    res = get_cv_result(params=params_dummy_obj_metric_inv_verbose, metrics="binary_error")
2500
    assert len(res) == 2
2501
    assert "valid binary_error-mean" in res
2502
2503

    # multiple metrics in params
2504
    res = get_cv_result(params=params_dummy_obj_metric_multi_verbose)
2505
    assert len(res) == 4
2506
2507
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
2508
2509

    # multiple metrics in args
2510
    res = get_cv_result(params=params_dummy_obj_verbose, metrics=["binary_logloss", "binary_error"])
2511
    assert len(res) == 4
2512
2513
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
2514

2515
    # no custom objective, feval
2516
2517
2518
    # default metric with custom one
    res = get_cv_result(feval=constant_metric)
    assert len(res) == 4
2519
2520
    assert "valid binary_logloss-mean" in res
    assert "valid error-mean" in res
2521
2522
2523
2524

    # non-default metric in params with custom one
    res = get_cv_result(params=params_obj_metric_err_verbose, feval=constant_metric)
    assert len(res) == 4
2525
2526
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2527
2528

    # default metric in args with custom one
2529
    res = get_cv_result(metrics="binary_logloss", feval=constant_metric)
2530
    assert len(res) == 4
2531
2532
    assert "valid binary_logloss-mean" in res
    assert "valid error-mean" in res
2533
2534

    # non-default metric in args with custom one
2535
    res = get_cv_result(metrics="binary_error", feval=constant_metric)
2536
    assert len(res) == 4
2537
2538
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2539
2540

    # metric in args overwrites one in params, custom one is evaluated too
2541
    res = get_cv_result(params=params_obj_metric_inv_verbose, metrics="binary_error", feval=constant_metric)
2542
    assert len(res) == 4
2543
2544
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2545
2546
2547
2548

    # multiple metrics in params with custom one
    res = get_cv_result(params=params_obj_metric_multi_verbose, feval=constant_metric)
    assert len(res) == 6
2549
2550
2551
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2552
2553

    # multiple metrics in args with custom one
2554
    res = get_cv_result(metrics=["binary_logloss", "binary_error"], feval=constant_metric)
2555
    assert len(res) == 6
2556
2557
2558
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2559
2560

    # custom metric is evaluated despite 'None' is passed
2561
    res = get_cv_result(metrics=["None"], feval=constant_metric)
2562
    assert len(res) == 2
2563
    assert "valid error-mean" in res
2564

2565
    # custom objective, feval
2566
    # no default metric, only custom one
2567
    res = get_cv_result(params=params_dummy_obj_verbose, feval=constant_metric)
2568
    assert len(res) == 2
2569
    assert "valid error-mean" in res
2570
2571

    # metric in params with custom one
2572
    res = get_cv_result(params=params_dummy_obj_metric_err_verbose, feval=constant_metric)
2573
    assert len(res) == 4
2574
2575
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2576
2577

    # metric in args with custom one
2578
    res = get_cv_result(params=params_dummy_obj_verbose, feval=constant_metric, metrics="binary_error")
2579
    assert len(res) == 4
2580
2581
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2582
2583

    # metric in args overwrites one in params, custom one is evaluated too
2584
    res = get_cv_result(params=params_dummy_obj_metric_inv_verbose, feval=constant_metric, metrics="binary_error")
2585
    assert len(res) == 4
2586
2587
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2588
2589

    # multiple metrics in params with custom one
2590
    res = get_cv_result(params=params_dummy_obj_metric_multi_verbose, feval=constant_metric)
2591
    assert len(res) == 6
2592
2593
2594
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2595
2596

    # multiple metrics in args with custom one
2597
2598
2599
    res = get_cv_result(
        params=params_dummy_obj_verbose, feval=constant_metric, metrics=["binary_logloss", "binary_error"]
    )
2600
    assert len(res) == 6
2601
2602
2603
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2604
2605

    # custom metric is evaluated despite 'None' is passed
2606
    res = get_cv_result(params=params_dummy_obj_metric_none_verbose, feval=constant_metric)
2607
    assert len(res) == 2
2608
    assert "valid error-mean" in res
2609

2610
    # no custom objective, no feval
2611
2612
    # default metric
    train_booster()
2613
2614
    assert len(evals_result["valid_0"]) == 1
    assert "binary_logloss" in evals_result["valid_0"]
2615
2616
2617

    # default metric in params
    train_booster(params=params_obj_metric_log_verbose)
2618
2619
    assert len(evals_result["valid_0"]) == 1
    assert "binary_logloss" in evals_result["valid_0"]
2620
2621
2622

    # non-default metric in params
    train_booster(params=params_obj_metric_err_verbose)
2623
2624
    assert len(evals_result["valid_0"]) == 1
    assert "binary_error" in evals_result["valid_0"]
2625
2626
2627

    # multiple metrics in params
    train_booster(params=params_obj_metric_multi_verbose)
2628
2629
2630
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "binary_error" in evals_result["valid_0"]
2631
2632

    # remove default metric by 'None' aliases
2633
2634
    for na_alias in ("None", "na", "null", "custom"):
        params = {"objective": "binary", "metric": na_alias, "verbose": -1}
2635
2636
2637
        train_booster(params=params)
        assert len(evals_result) == 0

2638
    # custom objective, no feval
2639
    # no default metric
2640
    train_booster(params=params_dummy_obj_verbose)
2641
2642
2643
    assert len(evals_result) == 0

    # metric in params
2644
    train_booster(params=params_dummy_obj_metric_log_verbose)
2645
2646
    assert len(evals_result["valid_0"]) == 1
    assert "binary_logloss" in evals_result["valid_0"]
2647
2648

    # multiple metrics in params
2649
    train_booster(params=params_dummy_obj_metric_multi_verbose)
2650
2651
2652
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "binary_error" in evals_result["valid_0"]
2653

2654
    # no custom objective, feval
2655
2656
    # default metric with custom one
    train_booster(feval=constant_metric)
2657
2658
2659
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2660
2661
2662

    # default metric in params with custom one
    train_booster(params=params_obj_metric_log_verbose, feval=constant_metric)
2663
2664
2665
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2666
2667
2668

    # non-default metric in params with custom one
    train_booster(params=params_obj_metric_err_verbose, feval=constant_metric)
2669
2670
2671
    assert len(evals_result["valid_0"]) == 2
    assert "binary_error" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2672
2673
2674

    # multiple metrics in params with custom one
    train_booster(params=params_obj_metric_multi_verbose, feval=constant_metric)
2675
2676
2677
2678
    assert len(evals_result["valid_0"]) == 3
    assert "binary_logloss" in evals_result["valid_0"]
    assert "binary_error" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2679
2680
2681
2682

    # custom metric is evaluated despite 'None' is passed
    train_booster(params=params_obj_metric_none_verbose, feval=constant_metric)
    assert len(evals_result) == 1
2683
    assert "error" in evals_result["valid_0"]
2684

2685
    # custom objective, feval
2686
    # no default metric, only custom one
2687
    train_booster(params=params_dummy_obj_verbose, feval=constant_metric)
2688
2689
    assert len(evals_result["valid_0"]) == 1
    assert "error" in evals_result["valid_0"]
2690
2691

    # metric in params with custom one
2692
    train_booster(params=params_dummy_obj_metric_log_verbose, feval=constant_metric)
2693
2694
2695
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2696
2697

    # multiple metrics in params with custom one
2698
    train_booster(params=params_dummy_obj_metric_multi_verbose, feval=constant_metric)
2699
2700
2701
2702
    assert len(evals_result["valid_0"]) == 3
    assert "binary_logloss" in evals_result["valid_0"]
    assert "binary_error" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2703
2704

    # custom metric is evaluated despite 'None' is passed
2705
    train_booster(params=params_dummy_obj_metric_none_verbose, feval=constant_metric)
2706
    assert len(evals_result) == 1
2707
    assert "error" in evals_result["valid_0"]
2708
2709

    X, y = load_digits(n_class=3, return_X_y=True)
2710
    lgb_train = lgb.Dataset(X, y)
2711

2712
    obj_multi_aliases = ["multiclass", "softmax", "multiclassova", "multiclass_ova", "ova", "ovr"]
2713
    for obj_multi_alias in obj_multi_aliases:
2714
        # Custom objective replaces multiclass
2715
2716
2717
2718
2719
        params_obj_class_3_verbose = {"objective": obj_multi_alias, "num_class": 3, "verbose": -1}
        params_dummy_obj_class_3_verbose = {"objective": dummy_obj, "num_class": 3, "verbose": -1}
        params_dummy_obj_class_1_verbose = {"objective": dummy_obj, "num_class": 1, "verbose": -1}
        params_obj_verbose = {"objective": obj_multi_alias, "verbose": -1}
        params_dummy_obj_verbose = {"objective": dummy_obj, "verbose": -1}
2720
2721
2722
        # multiclass default metric
        res = get_cv_result(params_obj_class_3_verbose)
        assert len(res) == 2
2723
        assert "valid multi_logloss-mean" in res
2724
2725
2726
        # multiclass default metric with custom one
        res = get_cv_result(params_obj_class_3_verbose, feval=constant_metric)
        assert len(res) == 4
2727
2728
        assert "valid multi_logloss-mean" in res
        assert "valid error-mean" in res
2729
        # multiclass metric alias with custom one for custom objective
2730
        res = get_cv_result(params_dummy_obj_class_3_verbose, feval=constant_metric)
2731
        assert len(res) == 2
2732
        assert "valid error-mean" in res
2733
        # no metric for invalid class_num
2734
        res = get_cv_result(params_dummy_obj_class_1_verbose)
2735
2736
        assert len(res) == 0
        # custom metric for invalid class_num
2737
        res = get_cv_result(params_dummy_obj_class_1_verbose, feval=constant_metric)
2738
        assert len(res) == 2
2739
        assert "valid error-mean" in res
2740
2741
        # multiclass metric alias with custom one with invalid class_num
        with pytest.raises(lgb.basic.LightGBMError):
2742
            get_cv_result(params_dummy_obj_class_1_verbose, metrics=obj_multi_alias, feval=constant_metric)
2743
2744
2745
        # multiclass default metric without num_class
        with pytest.raises(lgb.basic.LightGBMError):
            get_cv_result(params_obj_verbose)
2746
        for metric_multi_alias in obj_multi_aliases + ["multi_logloss"]:
2747
2748
2749
            # multiclass metric alias
            res = get_cv_result(params_obj_class_3_verbose, metrics=metric_multi_alias)
            assert len(res) == 2
2750
            assert "valid multi_logloss-mean" in res
2751
        # multiclass metric
2752
        res = get_cv_result(params_obj_class_3_verbose, metrics="multi_error")
2753
        assert len(res) == 2
2754
        assert "valid multi_error-mean" in res
2755
2756
        # non-valid metric for multiclass objective
        with pytest.raises(lgb.basic.LightGBMError):
2757
2758
            get_cv_result(params_obj_class_3_verbose, metrics="binary_logloss")
    params_class_3_verbose = {"num_class": 3, "verbose": -1}
2759
2760
2761
2762
    # non-default num_class for default objective
    with pytest.raises(lgb.basic.LightGBMError):
        get_cv_result(params_class_3_verbose)
    # no metric with non-default num_class for custom objective
2763
    res = get_cv_result(params_dummy_obj_class_3_verbose)
2764
    assert len(res) == 0
2765
    for metric_multi_alias in obj_multi_aliases + ["multi_logloss"]:
2766
        # multiclass metric alias for custom objective
2767
        res = get_cv_result(params_dummy_obj_class_3_verbose, metrics=metric_multi_alias)
2768
        assert len(res) == 2
2769
        assert "valid multi_logloss-mean" in res
2770
    # multiclass metric for custom objective
2771
    res = get_cv_result(params_dummy_obj_class_3_verbose, metrics="multi_error")
2772
    assert len(res) == 2
2773
    assert "valid multi_error-mean" in res
2774
2775
    # binary metric with non-default num_class for custom objective
    with pytest.raises(lgb.basic.LightGBMError):
2776
        get_cv_result(params_dummy_obj_class_3_verbose, metrics="binary_error")
2777
2778
2779
2780
2781


def test_multiple_feval_train():
    X, y = load_breast_cancer(return_X_y=True)

2782
    params = {"verbose": -1, "objective": "binary", "metric": "binary_logloss"}
2783
2784
2785

    X_train, X_validation, y_train, y_validation = train_test_split(X, y, test_size=0.2)

2786
2787
    train_dataset = lgb.Dataset(data=X_train, label=y_train)
    validation_dataset = lgb.Dataset(data=X_validation, label=y_validation, reference=train_dataset)
2788
2789
2790
2791
2792
2793
2794
    evals_result = {}
    lgb.train(
        params=params,
        train_set=train_dataset,
        valid_sets=validation_dataset,
        num_boost_round=5,
        feval=[constant_metric, decreasing_metric],
2795
        callbacks=[lgb.record_evaluation(evals_result)],
2796
    )
2797

2798
2799
2800
2801
    assert len(evals_result["valid_0"]) == 3
    assert "binary_logloss" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
    assert "decreasing_metric" in evals_result["valid_0"]
2802
2803


2804
2805
def test_objective_callable_train_binary_classification():
    X, y = load_breast_cancer(return_X_y=True)
2806
    params = {"verbose": -1, "objective": logloss_obj, "learning_rate": 0.01}
2807
    train_dataset = lgb.Dataset(X, y)
2808
    booster = lgb.train(params=params, train_set=train_dataset, num_boost_round=20)
2809
2810
2811
    y_pred = logistic_sigmoid(booster.predict(X))
    logloss_error = log_loss(y, y_pred)
    rocauc_error = roc_auc_score(y, y_pred)
2812
    assert booster.params["objective"] == "none"
2813
2814
    assert logloss_error == pytest.approx(0.547907)
    assert rocauc_error == pytest.approx(0.995944)
2815
2816
2817
2818


def test_objective_callable_train_regression():
    X, y = make_synthetic_regression()
2819
    params = {"verbose": -1, "objective": mse_obj}
2820
    lgb_train = lgb.Dataset(X, y)
2821
    booster = lgb.train(params, lgb_train, num_boost_round=20)
2822
2823
    y_pred = booster.predict(X)
    mse_error = mean_squared_error(y, y_pred)
2824
    assert booster.params["objective"] == "none"
2825
    assert mse_error == pytest.approx(286.724194)
2826
2827
2828
2829


def test_objective_callable_cv_binary_classification():
    X, y = load_breast_cancer(return_X_y=True)
2830
    params = {"verbose": -1, "objective": logloss_obj, "learning_rate": 0.01}
2831
    train_dataset = lgb.Dataset(X, y)
2832
2833
2834
2835
    cv_res = lgb.cv(params, train_dataset, num_boost_round=20, nfold=3, return_cvbooster=True)
    cv_booster = cv_res["cvbooster"].boosters
    cv_logloss_errors = [log_loss(y, logistic_sigmoid(cb.predict(X))) < 0.56 for cb in cv_booster]
    cv_objs = [cb.params["objective"] == "none" for cb in cv_booster]
2836
2837
2838
2839
2840
2841
2842
    assert all(cv_objs)
    assert all(cv_logloss_errors)


def test_objective_callable_cv_regression():
    X, y = make_synthetic_regression()
    lgb_train = lgb.Dataset(X, y)
2843
2844
2845
2846
2847
    params = {"verbose": -1, "objective": mse_obj}
    cv_res = lgb.cv(params, lgb_train, num_boost_round=20, nfold=3, stratified=False, return_cvbooster=True)
    cv_booster = cv_res["cvbooster"].boosters
    cv_mse_errors = [mean_squared_error(y, cb.predict(X)) < 463 for cb in cv_booster]
    cv_objs = [cb.params["objective"] == "none" for cb in cv_booster]
2848
2849
2850
2851
    assert all(cv_objs)
    assert all(cv_mse_errors)


2852
2853
2854
def test_multiple_feval_cv():
    X, y = load_breast_cancer(return_X_y=True)

2855
    params = {"verbose": -1, "objective": "binary", "metric": "binary_logloss"}
2856

2857
    train_dataset = lgb.Dataset(data=X, label=y)
2858
2859

    cv_results = lgb.cv(
2860
2861
        params=params, train_set=train_dataset, num_boost_round=5, feval=[constant_metric, decreasing_metric]
    )
2862
2863
2864

    # Expect three metrics but mean and stdv for each metric
    assert len(cv_results) == 6
2865
2866
2867
2868
2869
2870
    assert "valid binary_logloss-mean" in cv_results
    assert "valid error-mean" in cv_results
    assert "valid decreasing_metric-mean" in cv_results
    assert "valid binary_logloss-stdv" in cv_results
    assert "valid error-stdv" in cv_results
    assert "valid decreasing_metric-stdv" in cv_results
2871
2872


2873
2874
2875
2876
2877
2878
def test_default_objective_and_metric():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
    train_dataset = lgb.Dataset(data=X_train, label=y_train)
    validation_dataset = lgb.Dataset(data=X_test, label=y_test, reference=train_dataset)
    evals_result = {}
2879
    params = {"verbose": -1}
2880
2881
2882
2883
2884
    lgb.train(
        params=params,
        train_set=train_dataset,
        valid_sets=validation_dataset,
        num_boost_round=5,
2885
        callbacks=[lgb.record_evaluation(evals_result)],
2886
2887
    )

2888
2889
2890
2891
    assert "valid_0" in evals_result
    assert len(evals_result["valid_0"]) == 1
    assert "l2" in evals_result["valid_0"]
    assert len(evals_result["valid_0"]["l2"]) == 5
2892
2893


2894
@pytest.mark.parametrize("use_weight", [True, False])
2895
def test_multiclass_custom_objective(use_weight):
2896
2897
    def custom_obj(y_pred, ds):
        y_true = ds.get_label()
2898
2899
2900
        weight = ds.get_weight()
        grad, hess = sklearn_multiclass_custom_objective(y_true, y_pred, weight)
        return grad, hess
2901
2902
2903

    centers = [[-4, -4], [4, 4], [-4, 4]]
    X, y = make_blobs(n_samples=1_000, centers=centers, random_state=42)
2904
    weight = np.full_like(y, 2)
2905
    ds = lgb.Dataset(X, y)
2906
2907
    if use_weight:
        ds.set_weight(weight)
2908
    params = {"objective": "multiclass", "num_class": 3, "num_leaves": 7}
2909
2910
2911
    builtin_obj_bst = lgb.train(params, ds, num_boost_round=10)
    builtin_obj_preds = builtin_obj_bst.predict(X)

2912
    params["objective"] = custom_obj
2913
    custom_obj_bst = lgb.train(params, ds, num_boost_round=10)
2914
2915
2916
2917
2918
    custom_obj_preds = softmax(custom_obj_bst.predict(X))

    np.testing.assert_allclose(builtin_obj_preds, custom_obj_preds, rtol=0.01)


2919
@pytest.mark.parametrize("use_weight", [True, False])
2920
def test_multiclass_custom_eval(use_weight):
2921
2922
    def custom_eval(y_pred, ds):
        y_true = ds.get_label()
2923
2924
        weight = ds.get_weight()  # weight is None when not set
        loss = log_loss(y_true, y_pred, sample_weight=weight)
2925
        return "custom_logloss", loss, False
2926
2927
2928

    centers = [[-4, -4], [4, 4], [-4, 4]]
    X, y = make_blobs(n_samples=1_000, centers=centers, random_state=42)
2929
2930
2931
2932
    weight = np.full_like(y, 2)
    X_train, X_valid, y_train, y_valid, weight_train, weight_valid = train_test_split(
        X, y, weight, test_size=0.2, random_state=0
    )
2933
2934
    train_ds = lgb.Dataset(X_train, y_train)
    valid_ds = lgb.Dataset(X_valid, y_valid, reference=train_ds)
2935
2936
2937
    if use_weight:
        train_ds.set_weight(weight_train)
        valid_ds.set_weight(weight_valid)
2938
    params = {"objective": "multiclass", "num_class": 3, "num_leaves": 7}
2939
2940
2941
2942
2943
2944
    eval_result = {}
    bst = lgb.train(
        params,
        train_ds,
        num_boost_round=10,
        valid_sets=[train_ds, valid_ds],
2945
        valid_names=["train", "valid"],
2946
2947
2948
2949
2950
        feval=custom_eval,
        callbacks=[lgb.record_evaluation(eval_result)],
        keep_training_booster=True,
    )

2951
2952
    for key, ds in zip(["train", "valid"], [train_ds, valid_ds]):
        np.testing.assert_allclose(eval_result[key]["multi_logloss"], eval_result[key]["custom_logloss"])
2953
        _, metric, value, _ = bst.eval(ds, key, feval=custom_eval)[1]  # first element is multi_logloss
2954
        assert metric == "custom_logloss"
2955
2956
2957
        np.testing.assert_allclose(value, eval_result[key][metric][-1])


2958
@pytest.mark.skipif(psutil.virtual_memory().available / 1024 / 1024 / 1024 < 3, reason="not enough RAM")
2959
def test_model_size():
2960
    X, y = make_synthetic_regression()
2961
    data = lgb.Dataset(X, y)
2962
    bst = lgb.train({"verbose": -1}, data, num_boost_round=2)
2963
2964
    y_pred = bst.predict(X)
    model_str = bst.model_to_string()
2965
    one_tree = model_str[model_str.find("Tree=1") : model_str.find("end of trees")]
2966
    one_tree_size = len(one_tree)
2967
    one_tree = one_tree.replace("Tree=1", "Tree={}")
2968
2969
2970
    multiplier = 100
    total_trees = multiplier + 2
    try:
2971
2972
        before_tree_sizes = model_str[: model_str.find("tree_sizes")]
        trees = model_str[model_str.find("Tree=0") : model_str.find("end of trees")]
2973
        more_trees = (one_tree * multiplier).format(*range(2, total_trees))
2974
        after_trees = model_str[model_str.find("end of trees") :]
2975
2976
        num_end_spaces = 2**31 - one_tree_size * total_trees
        new_model_str = f"{before_tree_sizes}\n\n{trees}{more_trees}{after_trees}{'':{num_end_spaces}}"
2977
        assert len(new_model_str) > 2**31
2978
        bst.model_from_string(new_model_str)
2979
2980
2981
2982
        assert bst.num_trees() == total_trees
        y_pred_new = bst.predict(X, num_iteration=2)
        np.testing.assert_allclose(y_pred, y_pred_new)
    except MemoryError:
2983
        pytest.skipTest("not enough RAM")
2984
2985


2986
2987
2988
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Skip due to differences in implementation details of CUDA version"
)
2989
def test_get_split_value_histogram():
2990
2991
2992
2993
    X, y = make_synthetic_regression()
    X = np.repeat(X, 3, axis=0)
    y = np.repeat(y, 3, axis=0)
    X[:, 2] = np.random.default_rng(0).integers(0, 20, size=X.shape[0])
2994
    lgb_train = lgb.Dataset(X, y, categorical_feature=[2])
2995
    gbm = lgb.train({"verbose": -1}, lgb_train, num_boost_round=20)
2996
    # test XGBoost-style return value
2997
    params = {"feature": 0, "xgboost_style": True}
2998
2999
    assert gbm.get_split_value_histogram(**params).shape == (12, 2)
    assert gbm.get_split_value_histogram(bins=999, **params).shape == (12, 2)
3000
3001
3002
3003
    assert gbm.get_split_value_histogram(bins=-1, **params).shape == (1, 2)
    assert gbm.get_split_value_histogram(bins=0, **params).shape == (1, 2)
    assert gbm.get_split_value_histogram(bins=1, **params).shape == (1, 2)
    assert gbm.get_split_value_histogram(bins=2, **params).shape == (2, 2)
3004
3005
    assert gbm.get_split_value_histogram(bins=6, **params).shape == (6, 2)
    assert gbm.get_split_value_histogram(bins=7, **params).shape == (7, 2)
3006
3007
3008
    if lgb.compat.PANDAS_INSTALLED:
        np.testing.assert_allclose(
            gbm.get_split_value_histogram(0, xgboost_style=True).values,
3009
            gbm.get_split_value_histogram(gbm.feature_name()[0], xgboost_style=True).values,
3010
3011
3012
        )
        np.testing.assert_allclose(
            gbm.get_split_value_histogram(X.shape[-1] - 1, xgboost_style=True).values,
3013
            gbm.get_split_value_histogram(gbm.feature_name()[X.shape[-1] - 1], xgboost_style=True).values,
3014
3015
3016
3017
        )
    else:
        np.testing.assert_allclose(
            gbm.get_split_value_histogram(0, xgboost_style=True),
3018
            gbm.get_split_value_histogram(gbm.feature_name()[0], xgboost_style=True),
3019
3020
3021
        )
        np.testing.assert_allclose(
            gbm.get_split_value_histogram(X.shape[-1] - 1, xgboost_style=True),
3022
            gbm.get_split_value_histogram(gbm.feature_name()[X.shape[-1] - 1], xgboost_style=True),
3023
3024
3025
        )
    # test numpy-style return value
    hist, bins = gbm.get_split_value_histogram(0)
3026
3027
    assert len(hist) == 20
    assert len(bins) == 21
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
    hist, bins = gbm.get_split_value_histogram(0, bins=999)
    assert len(hist) == 999
    assert len(bins) == 1000
    with pytest.raises(ValueError):
        gbm.get_split_value_histogram(0, bins=-1)
    with pytest.raises(ValueError):
        gbm.get_split_value_histogram(0, bins=0)
    hist, bins = gbm.get_split_value_histogram(0, bins=1)
    assert len(hist) == 1
    assert len(bins) == 2
    hist, bins = gbm.get_split_value_histogram(0, bins=2)
    assert len(hist) == 2
    assert len(bins) == 3
    hist, bins = gbm.get_split_value_histogram(0, bins=6)
    assert len(hist) == 6
    assert len(bins) == 7
    hist, bins = gbm.get_split_value_histogram(0, bins=7)
    assert len(hist) == 7
    assert len(bins) == 8
    hist_idx, bins_idx = gbm.get_split_value_histogram(0)
    hist_name, bins_name = gbm.get_split_value_histogram(gbm.feature_name()[0])
    np.testing.assert_array_equal(hist_idx, hist_name)
    np.testing.assert_allclose(bins_idx, bins_name)
    hist_idx, bins_idx = gbm.get_split_value_histogram(X.shape[-1] - 1)
    hist_name, bins_name = gbm.get_split_value_histogram(gbm.feature_name()[X.shape[-1] - 1])
    np.testing.assert_array_equal(hist_idx, hist_name)
    np.testing.assert_allclose(bins_idx, bins_name)
    # test bins string type
3056
3057
    hist_vals, bin_edges = gbm.get_split_value_histogram(0, bins="auto")
    hist = gbm.get_split_value_histogram(0, bins="auto", xgboost_style=True)
3058
3059
    if lgb.compat.PANDAS_INSTALLED:
        mask = hist_vals > 0
3060
3061
        np.testing.assert_array_equal(hist_vals[mask], hist["Count"].values)
        np.testing.assert_allclose(bin_edges[1:][mask], hist["SplitValue"].values)
3062
3063
3064
3065
    else:
        mask = hist_vals > 0
        np.testing.assert_array_equal(hist_vals[mask], hist[:, 1])
        np.testing.assert_allclose(bin_edges[1:][mask], hist[:, 0])
3066
3067
3068
    # test histogram is disabled for categorical features
    with pytest.raises(lgb.basic.LightGBMError):
        gbm.get_split_value_histogram(2)
3069
3070


3071
3072
3073
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Skip due to differences in implementation details of CUDA version"
)
3074
def test_early_stopping_for_only_first_metric():
3075
    def metrics_combination_train_regression(valid_sets, metric_list, assumed_iteration, first_metric_only, feval=None):
3076
        params = {
3077
3078
3079
3080
3081
3082
            "objective": "regression",
            "learning_rate": 1.1,
            "num_leaves": 10,
            "metric": metric_list,
            "verbose": -1,
            "seed": 123,
3083
        }
3084
3085
3086
3087
3088
3089
        gbm = lgb.train(
            params,
            lgb_train,
            num_boost_round=25,
            valid_sets=valid_sets,
            feval=feval,
3090
            callbacks=[lgb.early_stopping(stopping_rounds=5, first_metric_only=first_metric_only)],
3091
        )
3092
        assert assumed_iteration == gbm.best_iteration
3093

3094
3095
3096
    def metrics_combination_cv_regression(
        metric_list, assumed_iteration, first_metric_only, eval_train_metric, feval=None
    ):
3097
        params = {
3098
3099
3100
3101
3102
3103
3104
            "objective": "regression",
            "learning_rate": 0.9,
            "num_leaves": 10,
            "metric": metric_list,
            "verbose": -1,
            "seed": 123,
            "gpu_use_dp": True,
3105
        }
3106
3107
3108
3109
3110
3111
3112
        ret = lgb.cv(
            params,
            train_set=lgb_train,
            num_boost_round=25,
            stratified=False,
            feval=feval,
            callbacks=[lgb.early_stopping(stopping_rounds=5, first_metric_only=first_metric_only)],
3113
            eval_train_metric=eval_train_metric,
3114
        )
3115
3116
        assert assumed_iteration == len(ret[list(ret.keys())[0]])

3117
    X, y = make_synthetic_regression()
3118
3119
3120
3121
3122
3123
3124
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
    X_test1, X_test2, y_test1, y_test2 = train_test_split(X_test, y_test, test_size=0.5, random_state=73)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_valid1 = lgb.Dataset(X_test1, y_test1, reference=lgb_train)
    lgb_valid2 = lgb.Dataset(X_test2, y_test2, reference=lgb_train)

    iter_valid1_l1 = 3
3125
3126
    iter_valid1_l2 = 3
    iter_valid2_l1 = 3
3127
    iter_valid2_l2 = 15
3128
    assert len({iter_valid1_l1, iter_valid1_l2, iter_valid2_l1, iter_valid2_l2}) == 2
3129
3130
3131
3132
    iter_min_l1 = min([iter_valid1_l1, iter_valid2_l1])
    iter_min_l2 = min([iter_valid1_l2, iter_valid2_l2])
    iter_min_valid1 = min([iter_valid1_l1, iter_valid1_l2])

3133
3134
    iter_cv_l1 = 15
    iter_cv_l2 = 13
3135
    assert len({iter_cv_l1, iter_cv_l2}) == 2
3136
3137
3138
3139
3140
3141
3142
    iter_cv_min = min([iter_cv_l1, iter_cv_l2])

    # test for lgb.train
    metrics_combination_train_regression(lgb_valid1, [], iter_valid1_l2, False)
    metrics_combination_train_regression(lgb_valid1, [], iter_valid1_l2, True)
    metrics_combination_train_regression(lgb_valid1, None, iter_valid1_l2, False)
    metrics_combination_train_regression(lgb_valid1, None, iter_valid1_l2, True)
3143
3144
3145
3146
3147
3148
    metrics_combination_train_regression(lgb_valid1, "l2", iter_valid1_l2, True)
    metrics_combination_train_regression(lgb_valid1, "l1", iter_valid1_l1, True)
    metrics_combination_train_regression(lgb_valid1, ["l2", "l1"], iter_valid1_l2, True)
    metrics_combination_train_regression(lgb_valid1, ["l1", "l2"], iter_valid1_l1, True)
    metrics_combination_train_regression(lgb_valid1, ["l2", "l1"], iter_min_valid1, False)
    metrics_combination_train_regression(lgb_valid1, ["l1", "l2"], iter_min_valid1, False)
3149
3150

    # test feval for lgb.train
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
    metrics_combination_train_regression(
        lgb_valid1,
        "None",
        1,
        False,
        feval=lambda preds, train_data: [decreasing_metric(preds, train_data), constant_metric(preds, train_data)],
    )
    metrics_combination_train_regression(
        lgb_valid1,
        "None",
        25,
        True,
        feval=lambda preds, train_data: [decreasing_metric(preds, train_data), constant_metric(preds, train_data)],
    )
    metrics_combination_train_regression(
        lgb_valid1,
        "None",
        1,
        True,
        feval=lambda preds, train_data: [constant_metric(preds, train_data), decreasing_metric(preds, train_data)],
    )
3172
3173

    # test with two valid data for lgb.train
3174
3175
3176
3177
    metrics_combination_train_regression([lgb_valid1, lgb_valid2], ["l2", "l1"], iter_min_l2, True)
    metrics_combination_train_regression([lgb_valid2, lgb_valid1], ["l2", "l1"], iter_min_l2, True)
    metrics_combination_train_regression([lgb_valid1, lgb_valid2], ["l1", "l2"], iter_min_l1, True)
    metrics_combination_train_regression([lgb_valid2, lgb_valid1], ["l1", "l2"], iter_min_l1, True)
3178
3179
3180

    # test for lgb.cv
    metrics_combination_cv_regression(None, iter_cv_l2, True, False)
3181
3182
3183
3184
3185
3186
    metrics_combination_cv_regression("l2", iter_cv_l2, True, False)
    metrics_combination_cv_regression("l1", iter_cv_l1, True, False)
    metrics_combination_cv_regression(["l2", "l1"], iter_cv_l2, True, False)
    metrics_combination_cv_regression(["l1", "l2"], iter_cv_l1, True, False)
    metrics_combination_cv_regression(["l2", "l1"], iter_cv_min, False, False)
    metrics_combination_cv_regression(["l1", "l2"], iter_cv_min, False, False)
3187
    metrics_combination_cv_regression(None, iter_cv_l2, True, True)
3188
3189
3190
3191
3192
3193
    metrics_combination_cv_regression("l2", iter_cv_l2, True, True)
    metrics_combination_cv_regression("l1", iter_cv_l1, True, True)
    metrics_combination_cv_regression(["l2", "l1"], iter_cv_l2, True, True)
    metrics_combination_cv_regression(["l1", "l2"], iter_cv_l1, True, True)
    metrics_combination_cv_regression(["l2", "l1"], iter_cv_min, False, True)
    metrics_combination_cv_regression(["l1", "l2"], iter_cv_min, False, True)
3194
3195

    # test feval for lgb.cv
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
    metrics_combination_cv_regression(
        "None",
        1,
        False,
        False,
        feval=lambda preds, train_data: [decreasing_metric(preds, train_data), constant_metric(preds, train_data)],
    )
    metrics_combination_cv_regression(
        "None",
        25,
        True,
        False,
        feval=lambda preds, train_data: [decreasing_metric(preds, train_data), constant_metric(preds, train_data)],
    )
    metrics_combination_cv_regression(
        "None",
        1,
        True,
        False,
        feval=lambda preds, train_data: [constant_metric(preds, train_data), decreasing_metric(preds, train_data)],
    )
3217
3218
3219
3220
3221
3222


def test_node_level_subcol():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
3223
3224
3225
3226
3227
        "objective": "binary",
        "metric": "binary_logloss",
        "feature_fraction_bynode": 0.8,
        "feature_fraction": 1.0,
        "verbose": -1,
3228
3229
3230
3231
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    evals_result = {}
3232
    gbm = lgb.train(
3233
        params, lgb_train, num_boost_round=25, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
3234
    )
3235
3236
    ret = log_loss(y_test, gbm.predict(X_test))
    assert ret < 0.14
3237
3238
    assert evals_result["valid_0"]["binary_logloss"][-1] == pytest.approx(ret)
    params["feature_fraction"] = 0.5
3239
3240
3241
3242
3243
    gbm2 = lgb.train(params, lgb_train, num_boost_round=25)
    ret2 = log_loss(y_test, gbm2.predict(X_test))
    assert ret != ret2


3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
def test_forced_split_feature_indices(tmp_path):
    X, y = make_synthetic_regression()
    forced_split = {
        "feature": 0,
        "threshold": 0.5,
        "left": {"feature": X.shape[1], "threshold": 0.5},
    }
    tmp_split_file = tmp_path / "forced_split.json"
    with open(tmp_split_file, "w") as f:
        f.write(json.dumps(forced_split))
    lgb_train = lgb.Dataset(X, y)
3255
    params = {"objective": "regression", "forcedsplits_filename": tmp_split_file}
3256
    with pytest.raises(lgb.basic.LightGBMError, match="Forced splits file includes feature index"):
3257
        lgb.train(params, lgb_train)
3258
3259


3260
def test_forced_bins():
3261
    x = np.empty((100, 2))
3262
3263
3264
    x[:, 0] = np.arange(0, 1, 0.01)
    x[:, 1] = -np.arange(0, 1, 0.01)
    y = np.arange(0, 1, 0.01)
3265
3266
3267
3268
3269
3270
3271
3272
3273
    forcedbins_filename = Path(__file__).absolute().parents[2] / "examples" / "regression" / "forced_bins.json"
    params = {
        "objective": "regression_l1",
        "max_bin": 5,
        "forcedbins_filename": forcedbins_filename,
        "num_leaves": 2,
        "min_data_in_leaf": 1,
        "verbose": -1,
    }
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
    lgb_x = lgb.Dataset(x, label=y)
    est = lgb.train(params, lgb_x, num_boost_round=20)
    new_x = np.zeros((3, x.shape[1]))
    new_x[:, 0] = [0.31, 0.37, 0.41]
    predicted = est.predict(new_x)
    assert len(np.unique(predicted)) == 3
    new_x[:, 0] = [0, 0, 0]
    new_x[:, 1] = [-0.9, -0.6, -0.3]
    predicted = est.predict(new_x)
    assert len(np.unique(predicted)) == 1
3284
    params["forcedbins_filename"] = ""
3285
3286
3287
3288
    lgb_x = lgb.Dataset(x, label=y)
    est = lgb.train(params, lgb_x, num_boost_round=20)
    predicted = est.predict(new_x)
    assert len(np.unique(predicted)) == 3
3289
3290
    params["forcedbins_filename"] = (
        Path(__file__).absolute().parents[2] / "examples" / "regression" / "forced_bins2.json"
3291
    )
3292
    params["max_bin"] = 11
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
    lgb_x = lgb.Dataset(x[:, :1], label=y)
    est = lgb.train(params, lgb_x, num_boost_round=50)
    predicted = est.predict(x[1:, :1])
    _, counts = np.unique(predicted, return_counts=True)
    assert min(counts) >= 9
    assert max(counts) <= 11


def test_binning_same_sign():
    # test that binning works properly for features with only positive or only negative values
3303
    x = np.empty((99, 2))
3304
3305
3306
    x[:, 0] = np.arange(0.01, 1, 0.01)
    x[:, 1] = -np.arange(0.01, 1, 0.01)
    y = np.arange(0.01, 1, 0.01)
3307
3308
3309
3310
3311
3312
3313
3314
    params = {
        "objective": "regression_l1",
        "max_bin": 5,
        "num_leaves": 2,
        "min_data_in_leaf": 1,
        "verbose": -1,
        "seed": 0,
    }
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
    lgb_x = lgb.Dataset(x, label=y)
    est = lgb.train(params, lgb_x, num_boost_round=20)
    new_x = np.zeros((3, 2))
    new_x[:, 0] = [-1, 0, 1]
    predicted = est.predict(new_x)
    assert predicted[0] == pytest.approx(predicted[1])
    assert predicted[1] != pytest.approx(predicted[2])
    new_x = np.zeros((3, 2))
    new_x[:, 1] = [-1, 0, 1]
    predicted = est.predict(new_x)
    assert predicted[0] != pytest.approx(predicted[1])
    assert predicted[1] == pytest.approx(predicted[2])


def test_dataset_update_params():
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
    default_params = {
        "max_bin": 100,
        "max_bin_by_feature": [20, 10],
        "bin_construct_sample_cnt": 10000,
        "min_data_in_bin": 1,
        "use_missing": False,
        "zero_as_missing": False,
        "categorical_feature": [0],
        "feature_pre_filter": True,
        "pre_partition": False,
        "enable_bundle": True,
        "data_random_seed": 0,
        "is_enable_sparse": True,
        "header": True,
        "two_round": True,
        "label_column": 0,
        "weight_column": 0,
        "group_column": 0,
        "ignore_column": 0,
        "min_data_in_leaf": 10,
        "linear_tree": False,
        "precise_float_parser": True,
        "verbose": -1,
    }
    unchangeable_params = {
        "max_bin": 150,
        "max_bin_by_feature": [30, 5],
        "bin_construct_sample_cnt": 5000,
        "min_data_in_bin": 2,
        "use_missing": True,
        "zero_as_missing": True,
        "categorical_feature": [0, 1],
        "feature_pre_filter": False,
        "pre_partition": True,
        "enable_bundle": False,
        "data_random_seed": 1,
        "is_enable_sparse": False,
        "header": False,
        "two_round": False,
        "label_column": 1,
        "weight_column": 1,
        "group_column": 1,
        "ignore_column": 1,
        "forcedbins_filename": "/some/path/forcedbins.json",
        "min_data_in_leaf": 2,
        "linear_tree": True,
        "precise_float_parser": False,
    }
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
    X = np.random.random((100, 2))
    y = np.random.random(100)

    # decreasing without freeing raw data is allowed
    lgb_data = lgb.Dataset(X, y, params=default_params, free_raw_data=False).construct()
    default_params["min_data_in_leaf"] -= 1
    lgb.train(default_params, lgb_data, num_boost_round=3)

    # decreasing before lazy init is allowed
    lgb_data = lgb.Dataset(X, y, params=default_params)
    default_params["min_data_in_leaf"] -= 1
    lgb.train(default_params, lgb_data, num_boost_round=3)

    # increasing is allowed
    default_params["min_data_in_leaf"] += 2
    lgb.train(default_params, lgb_data, num_boost_round=3)

    # decreasing with disabled filter is allowed
    default_params["feature_pre_filter"] = False
    lgb_data = lgb.Dataset(X, y, params=default_params).construct()
    default_params["min_data_in_leaf"] -= 4
    lgb.train(default_params, lgb_data, num_boost_round=3)

    # decreasing with enabled filter is disallowed;
    # also changes of other params are disallowed
    default_params["feature_pre_filter"] = True
    lgb_data = lgb.Dataset(X, y, params=default_params).construct()
    for key, value in unchangeable_params.items():
        new_params = default_params.copy()
        new_params[key] = value
3408
3409
3410
3411
        if key != "forcedbins_filename":
            param_name = key
        else:
            param_name = "forced bins"
3412
3413
3414
3415
3416
        err_msg = (
            "Reducing `min_data_in_leaf` with `feature_pre_filter=true` may cause *"
            if key == "min_data_in_leaf"
            else f"Cannot change {param_name} *"
        )
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
        with np.testing.assert_raises_regex(lgb.basic.LightGBMError, err_msg):
            lgb.train(new_params, lgb_data, num_boost_round=3)


def test_dataset_params_with_reference():
    default_params = {"max_bin": 100}
    X = np.random.random((100, 2))
    y = np.random.random(100)
    X_val = np.random.random((100, 2))
    y_val = np.random.random(100)
    lgb_train = lgb.Dataset(X, y, params=default_params, free_raw_data=False).construct()
    lgb_val = lgb.Dataset(X_val, y_val, reference=lgb_train, free_raw_data=False).construct()
    assert lgb_train.get_params() == default_params
    assert lgb_val.get_params() == default_params
    lgb.train(default_params, lgb_train, valid_sets=[lgb_val])


def test_extra_trees():
    # check extra trees increases regularization
3436
    X, y = make_synthetic_regression()
3437
    lgb_x = lgb.Dataset(X, label=y)
3438
    params = {"objective": "regression", "num_leaves": 32, "verbose": -1, "extra_trees": False, "seed": 0}
3439
3440
3441
    est = lgb.train(params, lgb_x, num_boost_round=10)
    predicted = est.predict(X)
    err = mean_squared_error(y, predicted)
3442
    params["extra_trees"] = True
3443
3444
3445
3446
3447
3448
3449
3450
    est = lgb.train(params, lgb_x, num_boost_round=10)
    predicted_new = est.predict(X)
    err_new = mean_squared_error(y, predicted_new)
    assert err < err_new


def test_path_smoothing():
    # check path smoothing increases regularization
3451
    X, y = make_synthetic_regression()
3452
    lgb_x = lgb.Dataset(X, label=y)
3453
    params = {"objective": "regression", "num_leaves": 32, "verbose": -1, "seed": 0}
3454
3455
3456
    est = lgb.train(params, lgb_x, num_boost_round=10)
    predicted = est.predict(X)
    err = mean_squared_error(y, predicted)
3457
    params["path_smooth"] = 1
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
    est = lgb.train(params, lgb_x, num_boost_round=10)
    predicted_new = est.predict(X)
    err_new = mean_squared_error(y, predicted_new)
    assert err < err_new


def test_trees_to_dataframe():
    pytest.importorskip("pandas")

    def _imptcs_to_numpy(X, impcts_dict):
3468
3469
        cols = [f"Column_{i}" for i in range(X.shape[1])]
        return [impcts_dict.get(col, 0.0) for col in cols]
3470
3471
3472
3473
3474
3475

    X, y = load_breast_cancer(return_X_y=True)
    data = lgb.Dataset(X, label=y)
    num_trees = 10
    bst = lgb.train({"objective": "binary", "verbose": -1}, data, num_trees)
    tree_df = bst.trees_to_dataframe()
3476
    split_dict = tree_df[~tree_df["split_gain"].isnull()].groupby("split_feature").size().to_dict()
3477

3478
    gains_dict = tree_df.groupby("split_feature")["split_gain"].sum().to_dict()
3479
3480
3481

    tree_split = _imptcs_to_numpy(X, split_dict)
    tree_gains = _imptcs_to_numpy(X, gains_dict)
3482
3483
3484
3485
    mod_split = bst.feature_importance("split")
    mod_gains = bst.feature_importance("gain")
    num_trees_from_df = tree_df["tree_index"].nunique()
    obs_counts_from_df = tree_df.loc[tree_df["node_depth"] == 1, "count"].values
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499

    np.testing.assert_equal(tree_split, mod_split)
    np.testing.assert_allclose(tree_gains, mod_gains)
    assert num_trees_from_df == num_trees
    np.testing.assert_equal(obs_counts_from_df, len(y))

    # test edge case with one leaf
    X = np.ones((10, 2))
    y = np.random.rand(10)
    data = lgb.Dataset(X, label=y)
    bst = lgb.train({"objective": "binary", "verbose": -1}, data, num_trees)
    tree_df = bst.trees_to_dataframe()

    assert len(tree_df) == 1
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
    assert tree_df.loc[0, "tree_index"] == 0
    assert tree_df.loc[0, "node_depth"] == 1
    assert tree_df.loc[0, "node_index"] == "0-L0"
    assert tree_df.loc[0, "value"] is not None
    for col in (
        "left_child",
        "right_child",
        "parent_index",
        "split_feature",
        "split_gain",
        "threshold",
        "decision_type",
        "missing_direction",
        "missing_type",
        "weight",
        "count",
    ):
3517
3518
3519
3520
        assert tree_df.loc[0, col] is None


def test_interaction_constraints():
3521
    X, y = make_synthetic_regression(n_samples=200)
3522
3523
3524
    num_features = X.shape[1]
    train_data = lgb.Dataset(X, label=y)
    # check that constraint containing all features is equivalent to no constraint
3525
    params = {"verbose": -1, "seed": 0}
3526
3527
    est = lgb.train(params, train_data, num_boost_round=10)
    pred1 = est.predict(X)
3528
    est = lgb.train(dict(params, interaction_constraints=[list(range(num_features))]), train_data, num_boost_round=10)
3529
3530
3531
    pred2 = est.predict(X)
    np.testing.assert_allclose(pred1, pred2)
    # check that constraint partitioning the features reduces train accuracy
3532
    est = lgb.train(dict(params, interaction_constraints=[[0, 2], [1, 3]]), train_data, num_boost_round=10)
3533
3534
3535
    pred3 = est.predict(X)
    assert mean_squared_error(y, pred1) < mean_squared_error(y, pred3)
    # check that constraints consisting of single features reduce accuracy further
3536
3537
3538
    est = lgb.train(
        dict(params, interaction_constraints=[[i] for i in range(num_features)]), train_data, num_boost_round=10
    )
3539
3540
3541
3542
3543
3544
    pred4 = est.predict(X)
    assert mean_squared_error(y, pred3) < mean_squared_error(y, pred4)
    # test that interaction constraints work when not all features are used
    X = np.concatenate([np.zeros((X.shape[0], 1)), X], axis=1)
    num_features = X.shape[1]
    train_data = lgb.Dataset(X, label=y)
3545
3546
3547
3548
3549
    est = lgb.train(
        dict(params, interaction_constraints=[[0] + list(range(2, num_features)), [1] + list(range(2, num_features))]),
        train_data,
        num_boost_round=10,
    )
3550
3551


3552
3553
3554
3555
3556
3557
3558
def test_linear_trees_num_threads():
    # check that number of threads does not affect result
    np.random.seed(0)
    x = np.arange(0, 1000, 0.1)
    y = 2 * x + np.random.normal(0, 0.1, len(x))
    x = x[:, np.newaxis]
    lgb_train = lgb.Dataset(x, label=y)
3559
    params = {"verbose": -1, "objective": "regression", "seed": 0, "linear_tree": True, "num_threads": 2}
3560
3561
3562
3563
3564
3565
3566
3567
    est = lgb.train(params, lgb_train, num_boost_round=100)
    pred1 = est.predict(x)
    params["num_threads"] = 4
    est = lgb.train(params, lgb_train, num_boost_round=100)
    pred2 = est.predict(x)
    np.testing.assert_allclose(pred1, pred2)


3568
3569
3570
3571
3572
3573
3574
def test_linear_trees(tmp_path):
    # check that setting linear_tree=True fits better than ordinary trees when data has linear relationship
    np.random.seed(0)
    x = np.arange(0, 100, 0.1)
    y = 2 * x + np.random.normal(0, 0.1, len(x))
    x = x[:, np.newaxis]
    lgb_train = lgb.Dataset(x, label=y)
3575
    params = {"verbose": -1, "metric": "mse", "seed": 0, "num_leaves": 2}
3576
3577
3578
3579
    est = lgb.train(params, lgb_train, num_boost_round=10)
    pred1 = est.predict(x)
    lgb_train = lgb.Dataset(x, label=y)
    res = {}
3580
    est = lgb.train(
3581
        dict(params, linear_tree=True),
3582
3583
3584
        lgb_train,
        num_boost_round=10,
        valid_sets=[lgb_train],
3585
3586
        valid_names=["train"],
        callbacks=[lgb.record_evaluation(res)],
3587
    )
3588
    pred2 = est.predict(x)
3589
    assert res["train"]["l2"][-1] == pytest.approx(mean_squared_error(y, pred2), abs=1e-1)
3590
3591
3592
3593
3594
3595
3596
3597
    assert mean_squared_error(y, pred2) < mean_squared_error(y, pred1)
    # test again with nans in data
    x[:10] = np.nan
    lgb_train = lgb.Dataset(x, label=y)
    est = lgb.train(params, lgb_train, num_boost_round=10)
    pred1 = est.predict(x)
    lgb_train = lgb.Dataset(x, label=y)
    res = {}
3598
    est = lgb.train(
3599
        dict(params, linear_tree=True),
3600
3601
3602
        lgb_train,
        num_boost_round=10,
        valid_sets=[lgb_train],
3603
3604
        valid_names=["train"],
        callbacks=[lgb.record_evaluation(res)],
3605
    )
3606
    pred2 = est.predict(x)
3607
    assert res["train"]["l2"][-1] == pytest.approx(mean_squared_error(y, pred2), abs=1e-1)
3608
3609
3610
    assert mean_squared_error(y, pred2) < mean_squared_error(y, pred1)
    # test again with bagging
    res = {}
3611
    est = lgb.train(
3612
        dict(params, linear_tree=True, subsample=0.8, bagging_freq=1),
3613
3614
3615
        lgb_train,
        num_boost_round=10,
        valid_sets=[lgb_train],
3616
3617
        valid_names=["train"],
        callbacks=[lgb.record_evaluation(res)],
3618
    )
3619
    pred = est.predict(x)
3620
    assert res["train"]["l2"][-1] == pytest.approx(mean_squared_error(y, pred), abs=1e-1)
3621
3622
3623
3624
3625
3626
    # test with a feature that has only one non-nan value
    x = np.concatenate([np.ones([x.shape[0], 1]), x], 1)
    x[500:, 1] = np.nan
    y[500:] += 10
    lgb_train = lgb.Dataset(x, label=y)
    res = {}
3627
    est = lgb.train(
3628
        dict(params, linear_tree=True, subsample=0.8, bagging_freq=1),
3629
3630
3631
        lgb_train,
        num_boost_round=10,
        valid_sets=[lgb_train],
3632
3633
        valid_names=["train"],
        callbacks=[lgb.record_evaluation(res)],
3634
    )
3635
    pred = est.predict(x)
3636
    assert res["train"]["l2"][-1] == pytest.approx(mean_squared_error(y, pred), abs=1e-1)
3637
3638
3639
3640
    # test with a categorical feature
    x[:250, 0] = 0
    y[:250] += 10
    lgb_train = lgb.Dataset(x, label=y)
3641
3642
3643
3644
3645
3646
    est = lgb.train(
        dict(params, linear_tree=True, subsample=0.8, bagging_freq=1),
        lgb_train,
        num_boost_round=10,
        categorical_feature=[0],
    )
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
    # test refit: same results on same data
    est2 = est.refit(x, label=y)
    p1 = est.predict(x)
    p2 = est2.predict(x)
    assert np.mean(np.abs(p1 - p2)) < 2

    # test refit with save and load
    temp_model = str(tmp_path / "temp_model.txt")
    est.save_model(temp_model)
    est2 = lgb.Booster(model_file=temp_model)
    est2 = est2.refit(x, label=y)
    p1 = est.predict(x)
    p2 = est2.predict(x)
    assert np.mean(np.abs(p1 - p2)) < 2
    # test refit: different results training on different data
    est3 = est.refit(x[:100, :], label=y[:100])
    p3 = est3.predict(x)
    assert np.mean(np.abs(p2 - p1)) > np.abs(np.max(p3 - p1))
    # test when num_leaves - 1 < num_features and when num_leaves - 1 > num_features
    X_train, _, y_train, _ = train_test_split(*load_breast_cancer(return_X_y=True), test_size=0.1, random_state=2)
3667
    params = {"linear_tree": True, "verbose": -1, "metric": "mse", "seed": 0}
3668
3669
3670
3671
3672
3673
    train_data = lgb.Dataset(X_train, label=y_train, params=dict(params, num_leaves=2))
    est = lgb.train(params, train_data, num_boost_round=10, categorical_feature=[0])
    train_data = lgb.Dataset(X_train, label=y_train, params=dict(params, num_leaves=60))
    est = lgb.train(params, train_data, num_boost_round=10, categorical_feature=[0])


3674
def test_save_and_load_linear(tmp_path):
3675
3676
3677
    X_train, X_test, y_train, y_test = train_test_split(
        *load_breast_cancer(return_X_y=True), test_size=0.1, random_state=2
    )
3678
    X_train = np.concatenate([np.ones((X_train.shape[0], 1)), X_train], 1)
3679
3680
3681
    X_train[: X_train.shape[0] // 2, 0] = 0
    y_train[: X_train.shape[0] // 2] = 1
    params = {"linear_tree": True}
3682
3683
3684
3685
    train_data_1 = lgb.Dataset(X_train, label=y_train, params=params)
    est_1 = lgb.train(params, train_data_1, num_boost_round=10, categorical_feature=[0])
    pred_1 = est_1.predict(X_train)

3686
    tmp_dataset = str(tmp_path / "temp_dataset.bin")
3687
3688
3689
3690
3691
3692
    train_data_1.save_binary(tmp_dataset)
    train_data_2 = lgb.Dataset(tmp_dataset)
    est_2 = lgb.train(params, train_data_2, num_boost_round=10)
    pred_2 = est_2.predict(X_train)
    np.testing.assert_allclose(pred_1, pred_2)

3693
    model_file = str(tmp_path / "model.txt")
3694
3695
3696
3697
3698
3699
    est_2.save_model(model_file)
    est_3 = lgb.Booster(model_file=model_file)
    pred_3 = est_3.predict(X_train)
    np.testing.assert_allclose(pred_2, pred_3)


3700
3701
3702
def test_linear_single_leaf():
    X_train, y_train = load_breast_cancer(return_X_y=True)
    train_data = lgb.Dataset(X_train, label=y_train)
3703
    params = {"objective": "binary", "linear_tree": True, "min_sum_hessian": 5000}
3704
3705
3706
3707
3708
    bst = lgb.train(params, train_data, num_boost_round=5)
    y_pred = bst.predict(X_train)
    assert log_loss(y_train, y_pred) < 0.661


3709
3710
3711
3712
3713
def test_predict_with_start_iteration():
    def inner_test(X, y, params, early_stopping_rounds):
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
        train_data = lgb.Dataset(X_train, label=y_train)
        valid_data = lgb.Dataset(X_test, label=y_test)
3714
        callbacks = [lgb.early_stopping(early_stopping_rounds)] if early_stopping_rounds is not None else []
3715
        booster = lgb.train(params, train_data, num_boost_round=50, valid_sets=[valid_data], callbacks=callbacks)
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732

        # test that the predict once with all iterations equals summed results with start_iteration and num_iteration
        all_pred = booster.predict(X, raw_score=True)
        all_pred_contrib = booster.predict(X, pred_contrib=True)
        steps = [10, 12]
        for step in steps:
            pred = np.zeros_like(all_pred)
            pred_contrib = np.zeros_like(all_pred_contrib)
            for start_iter in range(0, 50, step):
                pred += booster.predict(X, start_iteration=start_iter, num_iteration=step, raw_score=True)
                pred_contrib += booster.predict(X, start_iteration=start_iter, num_iteration=step, pred_contrib=True)
            np.testing.assert_allclose(all_pred, pred)
            np.testing.assert_allclose(all_pred_contrib, pred_contrib)
        # test the case where start_iteration <= 0, and num_iteration is None
        pred1 = booster.predict(X, start_iteration=-1)
        pred2 = booster.predict(X, num_iteration=booster.best_iteration)
        np.testing.assert_allclose(pred1, pred2)
3733

3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
        # test the case where start_iteration > 0, and num_iteration <= 0
        pred4 = booster.predict(X, start_iteration=10, num_iteration=-1)
        pred5 = booster.predict(X, start_iteration=10, num_iteration=90)
        pred6 = booster.predict(X, start_iteration=10, num_iteration=0)
        np.testing.assert_allclose(pred4, pred5)
        np.testing.assert_allclose(pred4, pred6)

        # test the case where start_iteration > 0, and num_iteration <= 0, with pred_leaf=True
        pred4 = booster.predict(X, start_iteration=10, num_iteration=-1, pred_leaf=True)
        pred5 = booster.predict(X, start_iteration=10, num_iteration=40, pred_leaf=True)
        pred6 = booster.predict(X, start_iteration=10, num_iteration=0, pred_leaf=True)
        np.testing.assert_allclose(pred4, pred5)
        np.testing.assert_allclose(pred4, pred6)

        # test the case where start_iteration > 0, and num_iteration <= 0, with pred_contrib=True
        pred4 = booster.predict(X, start_iteration=10, num_iteration=-1, pred_contrib=True)
        pred5 = booster.predict(X, start_iteration=10, num_iteration=40, pred_contrib=True)
        pred6 = booster.predict(X, start_iteration=10, num_iteration=0, pred_contrib=True)
        np.testing.assert_allclose(pred4, pred5)
        np.testing.assert_allclose(pred4, pred6)

    # test for regression
3756
    X, y = make_synthetic_regression()
3757
    params = {"objective": "regression", "verbose": -1, "metric": "l2", "learning_rate": 0.5}
3758
3759
3760
3761
3762
3763
3764
    # test both with and without early stopping
    inner_test(X, y, params, early_stopping_rounds=1)
    inner_test(X, y, params, early_stopping_rounds=5)
    inner_test(X, y, params, early_stopping_rounds=None)

    # test for multi-class
    X, y = load_iris(return_X_y=True)
3765
    params = {"objective": "multiclass", "num_class": 3, "verbose": -1, "metric": "multi_error"}
3766
3767
3768
3769
3770
3771
3772
    # test both with and without early stopping
    inner_test(X, y, params, early_stopping_rounds=1)
    inner_test(X, y, params, early_stopping_rounds=5)
    inner_test(X, y, params, early_stopping_rounds=None)

    # test for binary
    X, y = load_breast_cancer(return_X_y=True)
3773
    params = {"objective": "binary", "verbose": -1, "metric": "auc"}
3774
3775
3776
3777
3778
3779
3780
3781
3782
    # test both with and without early stopping
    inner_test(X, y, params, early_stopping_rounds=1)
    inner_test(X, y, params, early_stopping_rounds=5)
    inner_test(X, y, params, early_stopping_rounds=None)


def test_average_precision_metric():
    # test against sklearn average precision metric
    X, y = load_breast_cancer(return_X_y=True)
3783
    params = {"objective": "binary", "metric": "average_precision", "verbose": -1}
3784
3785
    res = {}
    lgb_X = lgb.Dataset(X, label=y)
3786
3787
    est = lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(res)])
    ap = res["training"]["average_precision"][-1]
3788
3789
3790
3791
3792
3793
3794
    pred = est.predict(X)
    sklearn_ap = average_precision_score(y, pred)
    assert ap == pytest.approx(sklearn_ap)
    # test that average precision is 1 where model predicts perfectly
    y = y.copy()
    y[:] = 1
    lgb_X = lgb.Dataset(X, label=y)
3795
3796
    lgb.train(params, lgb_X, num_boost_round=1, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(res)])
    assert res["training"]["average_precision"][-1] == pytest.approx(1)
3797
3798
3799
3800
3801
3802


def test_reset_params_works_with_metric_num_class_and_boosting():
    X, y = load_breast_cancer(return_X_y=True)
    dataset_params = {"max_bin": 150}
    booster_params = {
3803
3804
3805
3806
3807
3808
        "objective": "multiclass",
        "max_depth": 4,
        "bagging_fraction": 0.8,
        "metric": ["multi_logloss", "multi_error"],
        "boosting": "gbdt",
        "num_class": 5,
3809
3810
    }
    dtrain = lgb.Dataset(X, y, params=dataset_params)
3811
    bst = lgb.Booster(params=booster_params, train_set=dtrain)
3812
3813
3814
3815

    expected_params = dict(dataset_params, **booster_params)
    assert bst.params == expected_params

3816
    booster_params["bagging_fraction"] += 0.1
3817
3818
3819
3820
3821
    new_bst = bst.reset_parameter(booster_params)

    expected_params = dict(dataset_params, **booster_params)
    assert bst.params == expected_params
    assert new_bst.params == expected_params
3822
3823
3824
3825
3826


def test_dump_model():
    X, y = load_breast_cancer(return_X_y=True)
    train_data = lgb.Dataset(X, label=y)
3827
    params = {"objective": "binary", "verbose": -1}
3828
3829
3830
3831
3832
3833
3834
    bst = lgb.train(params, train_data, num_boost_round=5)
    dumped_model_str = str(bst.dump_model(5, 0))
    assert "leaf_features" not in dumped_model_str
    assert "leaf_coeff" not in dumped_model_str
    assert "leaf_const" not in dumped_model_str
    assert "leaf_value" in dumped_model_str
    assert "leaf_count" in dumped_model_str
3835
    params["linear_tree"] = True
3836
3837
3838
3839
3840
3841
3842
3843
    train_data = lgb.Dataset(X, label=y)
    bst = lgb.train(params, train_data, num_boost_round=5)
    dumped_model_str = str(bst.dump_model(5, 0))
    assert "leaf_features" in dumped_model_str
    assert "leaf_coeff" in dumped_model_str
    assert "leaf_const" in dumped_model_str
    assert "leaf_value" in dumped_model_str
    assert "leaf_count" in dumped_model_str
3844
3845
3846
3847


def test_dump_model_hook():
    def hook(obj):
3848
3849
3850
        if "leaf_value" in obj:
            obj["LV"] = obj["leaf_value"]
            del obj["leaf_value"]
3851
3852
3853
3854
        return obj

    X, y = load_breast_cancer(return_X_y=True)
    train_data = lgb.Dataset(X, label=y)
3855
    params = {"objective": "binary", "verbose": -1}
3856
3857
3858
3859
    bst = lgb.train(params, train_data, num_boost_round=5)
    dumped_model_str = str(bst.dump_model(5, 0, object_hook=hook))
    assert "leaf_value" not in dumped_model_str
    assert "LV" in dumped_model_str
3860
3861


3862
@pytest.mark.skipif(getenv("TASK", "") == "cuda", reason="Forced splits are not yet supported by CUDA version")
3863
def test_force_split_with_feature_fraction(tmp_path):
3864
    X, y = make_synthetic_regression()
3865
3866
3867
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)

3868
    forced_split = {"feature": 0, "threshold": 0.5, "right": {"feature": 2, "threshold": 10.0}}
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878

    tmp_split_file = tmp_path / "forced_split.json"
    with open(tmp_split_file, "w") as f:
        f.write(json.dumps(forced_split))

    params = {
        "objective": "regression",
        "feature_fraction": 0.6,
        "force_col_wise": True,
        "feature_fraction_seed": 1,
3879
        "forcedsplits_filename": tmp_split_file,
3880
3881
3882
3883
    }

    gbm = lgb.train(params, lgb_train)
    ret = mean_absolute_error(y_test, gbm.predict(X_test))
3884
    assert ret < 15.7
3885
3886
3887
3888
3889

    tree_info = gbm.dump_model()["tree_info"]
    assert len(tree_info) > 1
    for tree in tree_info:
        tree_structure = tree["tree_structure"]
3890
        assert tree_structure["split_feature"] == 0
3891
3892


3893
3894
3895
3896
3897
3898
def test_goss_boosting_and_strategy_equivalent():
    X, y = make_synthetic_regression(n_samples=10_000, n_features=10, n_informative=5, random_state=42)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    base_params = {
3899
3900
3901
3902
3903
3904
3905
        "metric": "l2",
        "verbose": -1,
        "bagging_seed": 0,
        "learning_rate": 0.05,
        "num_threads": 1,
        "force_row_wise": True,
        "gpu_use_dp": True,
3906
    }
3907
    params1 = {**base_params, "boosting": "goss"}
3908
    evals_result1 = {}
3909
3910
3911
3912
    lgb.train(
        params1, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result1)]
    )
    params2 = {**base_params, "data_sample_strategy": "goss"}
3913
    evals_result2 = {}
3914
3915
3916
3917
    lgb.train(
        params2, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result2)]
    )
    assert evals_result1["valid_0"]["l2"] == evals_result2["valid_0"]["l2"]
3918
3919
3920
3921
3922
3923
3924
3925
3926


def test_sample_strategy_with_boosting():
    X, y = make_synthetic_regression(n_samples=10_000, n_features=10, n_informative=5, random_state=42)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)

    base_params = {
3927
3928
3929
3930
3931
        "metric": "l2",
        "verbose": -1,
        "num_threads": 1,
        "force_row_wise": True,
        "gpu_use_dp": True,
3932
3933
    }

3934
    params1 = {**base_params, "boosting": "dart", "data_sample_strategy": "goss"}
3935
    evals_result = {}
3936
3937
3938
3939
    gbm = lgb.train(
        params1, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res1 = evals_result["valid_0"]["l2"][-1]
3940
3941
3942
3943
    test_res1 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res1 == pytest.approx(3149.393862, abs=1.0)
    assert eval_res1 == pytest.approx(test_res1)

3944
    params2 = {**base_params, "boosting": "gbdt", "data_sample_strategy": "goss"}
3945
    evals_result = {}
3946
3947
3948
3949
    gbm = lgb.train(
        params2, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res2 = evals_result["valid_0"]["l2"][-1]
3950
3951
3952
3953
    test_res2 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res2 == pytest.approx(2547.715968, abs=1.0)
    assert eval_res2 == pytest.approx(test_res2)

3954
    params3 = {**base_params, "boosting": "goss", "data_sample_strategy": "goss"}
3955
    evals_result = {}
3956
3957
3958
3959
    gbm = lgb.train(
        params3, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res3 = evals_result["valid_0"]["l2"][-1]
3960
3961
3962
3963
    test_res3 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res3 == pytest.approx(2547.715968, abs=1.0)
    assert eval_res3 == pytest.approx(test_res3)

3964
    params4 = {**base_params, "boosting": "rf", "data_sample_strategy": "goss"}
3965
    evals_result = {}
3966
3967
3968
3969
    gbm = lgb.train(
        params4, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res4 = evals_result["valid_0"]["l2"][-1]
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
    test_res4 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res4 == pytest.approx(2095.538735, abs=1.0)
    assert eval_res4 == pytest.approx(test_res4)

    assert test_res1 != test_res2
    assert eval_res1 != eval_res2
    assert test_res2 == test_res3
    assert eval_res2 == eval_res3
    assert eval_res1 != eval_res4
    assert test_res1 != test_res4
    assert eval_res2 != eval_res4
    assert test_res2 != test_res4

3983
3984
3985
3986
3987
3988
3989
    params5 = {
        **base_params,
        "boosting": "dart",
        "data_sample_strategy": "bagging",
        "bagging_freq": 1,
        "bagging_fraction": 0.5,
    }
3990
    evals_result = {}
3991
3992
3993
3994
    gbm = lgb.train(
        params5, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res5 = evals_result["valid_0"]["l2"][-1]
3995
3996
3997
3998
    test_res5 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res5 == pytest.approx(3134.866931, abs=1.0)
    assert eval_res5 == pytest.approx(test_res5)

3999
4000
4001
4002
4003
4004
4005
    params6 = {
        **base_params,
        "boosting": "gbdt",
        "data_sample_strategy": "bagging",
        "bagging_freq": 1,
        "bagging_fraction": 0.5,
    }
4006
    evals_result = {}
4007
4008
4009
4010
    gbm = lgb.train(
        params6, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res6 = evals_result["valid_0"]["l2"][-1]
4011
4012
4013
4014
4015
4016
    test_res6 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res6 == pytest.approx(2539.792378, abs=1.0)
    assert eval_res6 == pytest.approx(test_res6)
    assert test_res5 != test_res6
    assert eval_res5 != eval_res6

4017
4018
4019
4020
4021
4022
4023
    params7 = {
        **base_params,
        "boosting": "rf",
        "data_sample_strategy": "bagging",
        "bagging_freq": 1,
        "bagging_fraction": 0.5,
    }
4024
    evals_result = {}
4025
4026
4027
4028
    gbm = lgb.train(
        params7, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res7 = evals_result["valid_0"]["l2"][-1]
4029
4030
4031
4032
4033
4034
4035
4036
4037
    test_res7 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res7 == pytest.approx(1518.704481, abs=1.0)
    assert eval_res7 == pytest.approx(test_res7)
    assert test_res5 != test_res7
    assert eval_res5 != eval_res7
    assert test_res6 != test_res7
    assert eval_res6 != eval_res7


4038
4039
4040
4041
4042
def test_record_evaluation_with_train():
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, y)
    eval_result = {}
    callbacks = [lgb.record_evaluation(eval_result)]
4043
    params = {"objective": "l2", "num_leaves": 3}
4044
4045
    num_boost_round = 5
    bst = lgb.train(params, ds, num_boost_round=num_boost_round, valid_sets=[ds], callbacks=callbacks)
4046
    assert list(eval_result.keys()) == ["training"]
4047
4048
4049
4050
4051
    train_mses = []
    for i in range(num_boost_round):
        pred = bst.predict(X, num_iteration=i + 1)
        mse = mean_squared_error(y, pred)
        train_mses.append(mse)
4052
    np.testing.assert_allclose(eval_result["training"]["l2"], train_mses)
4053
4054


4055
@pytest.mark.parametrize("train_metric", [False, True])
4056
4057
4058
4059
4060
def test_record_evaluation_with_cv(train_metric):
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, y)
    eval_result = {}
    callbacks = [lgb.record_evaluation(eval_result)]
4061
4062
4063
4064
4065
4066
    metrics = ["l2", "rmse"]
    params = {"objective": "l2", "num_leaves": 3, "metric": metrics}
    cv_hist = lgb.cv(
        params, ds, num_boost_round=5, stratified=False, callbacks=callbacks, eval_train_metric=train_metric
    )
    expected_datasets = {"valid"}
4067
    if train_metric:
4068
        expected_datasets.add("train")
4069
4070
4071
    assert set(eval_result.keys()) == expected_datasets
    for dataset in expected_datasets:
        for metric in metrics:
4072
4073
4074
            for agg in ("mean", "stdv"):
                key = f"{dataset} {metric}-{agg}"
                np.testing.assert_allclose(cv_hist[key], eval_result[dataset][f"{metric}-{agg}"])
4075
4076
4077


def test_pandas_with_numpy_regular_dtypes():
4078
4079
4080
4081
    pd = pytest.importorskip("pandas")
    uints = ["uint8", "uint16", "uint32", "uint64"]
    ints = ["int8", "int16", "int32", "int64"]
    bool_and_floats = ["bool", "float16", "float32", "float64"]
4082
4083
4084
4085
    rng = np.random.RandomState(42)

    n_samples = 100
    # data as float64
4086
4087
4088
4089
4090
4091
4092
4093
    df = pd.DataFrame(
        {
            "x1": rng.randint(0, 2, n_samples),
            "x2": rng.randint(1, 3, n_samples),
            "x3": 10 * rng.randint(1, 3, n_samples),
            "x4": 100 * rng.randint(1, 3, n_samples),
        }
    )
4094
    df = df.astype(np.float64)
4095
    y = df["x1"] * (df["x2"] + df["x3"] + df["x4"])
4096
    ds = lgb.Dataset(df, y)
4097
    params = {"objective": "l2", "num_leaves": 31, "min_child_samples": 1}
4098
4099
4100
4101
    bst = lgb.train(params, ds, num_boost_round=5)
    preds = bst.predict(df)

    # test all features were used
4102
    assert bst.trees_to_dataframe()["split_feature"].nunique() == df.shape[1]
4103
4104
4105
4106
4107
4108
    # test the score is better than predicting the mean
    baseline = np.full_like(y, y.mean())
    assert mean_squared_error(y, preds) < mean_squared_error(y, baseline)

    # test all predictions are equal using different input dtypes
    for target_dtypes in [uints, ints, bool_and_floats]:
4109
        df2 = df.astype({f"x{i}": dtype for i, dtype in enumerate(target_dtypes, start=1)})
4110
4111
4112
4113
4114
4115
4116
4117
        assert df2.dtypes.tolist() == target_dtypes
        ds2 = lgb.Dataset(df2, y)
        bst2 = lgb.train(params, ds2, num_boost_round=5)
        preds2 = bst2.predict(df2)
        np.testing.assert_allclose(preds, preds2)


def test_pandas_nullable_dtypes():
4118
    pd = pytest.importorskip("pandas")
4119
    rng = np.random.RandomState(0)
4120
4121
4122
4123
4124
4125
4126
4127
    df = pd.DataFrame(
        {
            "x1": rng.randint(1, 3, size=100),
            "x2": np.linspace(-1, 1, 100),
            "x3": pd.arrays.SparseArray(rng.randint(0, 11, size=100)),
            "x4": rng.rand(100) < 0.5,
        }
    )
4128
    # introduce some missing values
4129
4130
4131
    df.loc[1, "x1"] = np.nan
    df.loc[2, "x2"] = np.nan
    df.loc[3, "x4"] = np.nan
4132
    # the previous line turns x3 into object dtype in recent versions of pandas
4133
4134
    df["x4"] = df["x4"].astype(np.float64)
    y = df["x1"] * df["x2"] + df["x3"] * (1 + df["x4"])
4135
4136
4137
    y = y.fillna(0)

    # train with regular dtypes
4138
    params = {"objective": "l2", "num_leaves": 31, "min_child_samples": 1}
4139
4140
4141
4142
4143
4144
    ds = lgb.Dataset(df, y)
    bst = lgb.train(params, ds, num_boost_round=5)
    preds = bst.predict(df)

    # convert to nullable dtypes
    df2 = df.copy()
4145
4146
4147
    df2["x1"] = df2["x1"].astype("Int32")
    df2["x2"] = df2["x2"].astype("Float64")
    df2["x4"] = df2["x4"].astype("boolean")
4148
4149
4150
4151
4152
4153
4154
4155

    # test training succeeds
    ds_nullable_dtypes = lgb.Dataset(df2, y)
    bst_nullable_dtypes = lgb.train(params, ds_nullable_dtypes, num_boost_round=5)
    preds_nullable_dtypes = bst_nullable_dtypes.predict(df2)

    trees_df = bst_nullable_dtypes.trees_to_dataframe()
    # test all features were used
4156
    assert trees_df["split_feature"].nunique() == df.shape[1]
4157
4158
4159
4160
4161
4162
    # test the score is better than predicting the mean
    baseline = np.full_like(y, y.mean())
    assert mean_squared_error(y, preds) < mean_squared_error(y, baseline)

    # test equal predictions
    np.testing.assert_allclose(preds, preds_nullable_dtypes)
4163
4164
4165
4166
4167


def test_boost_from_average_with_single_leaf_trees():
    # test data are taken from bug report
    # https://github.com/microsoft/LightGBM/issues/4708
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
    X = np.array(
        [
            [1021.0589, 1018.9578],
            [1023.85754, 1018.7854],
            [1024.5468, 1018.88513],
            [1019.02954, 1018.88513],
            [1016.79926, 1018.88513],
            [1007.6, 1018.88513],
        ],
        dtype=np.float32,
    )
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
    y = np.array([1023.8, 1024.6, 1024.4, 1023.8, 1022.0, 1014.4], dtype=np.float32)
    params = {
        "extra_trees": True,
        "min_data_in_bin": 1,
        "extra_seed": 7,
        "objective": "regression",
        "verbose": -1,
        "boost_from_average": True,
        "min_data_in_leaf": 1,
    }
    train_set = lgb.Dataset(X, y)
    model = lgb.train(params=params, train_set=train_set, num_boost_round=10)

    preds = model.predict(X)
    mean_preds = np.mean(preds)
    assert y.min() <= mean_preds <= y.max()
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220


def test_cegb_split_buffer_clean():
    # modified from https://github.com/microsoft/LightGBM/issues/3679#issuecomment-938652811
    # and https://github.com/microsoft/LightGBM/pull/5087
    # test that the ``splits_per_leaf_`` of CEGB is cleaned before training a new tree
    # which is done in the fix #5164
    # without the fix:
    #    Check failed: (best_split_info.left_count) > (0)

    R, C = 1000, 100
    seed = 29
    np.random.seed(seed)
    data = np.random.randn(R, C)
    for i in range(1, C):
        data[i] += data[0] * np.random.randn()

    N = int(0.8 * len(data))
    train_data = data[:N]
    test_data = data[N:]
    train_y = np.sum(train_data, axis=1)
    test_y = np.sum(test_data, axis=1)

    train = lgb.Dataset(train_data, train_y, free_raw_data=True)

    params = {
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
        "boosting_type": "gbdt",
        "objective": "regression",
        "max_bin": 255,
        "num_leaves": 31,
        "seed": 0,
        "learning_rate": 0.1,
        "min_data_in_leaf": 0,
        "verbose": -1,
        "min_split_gain": 1000.0,
        "cegb_penalty_feature_coupled": 5 * np.arange(C),
        "cegb_penalty_split": 0.0002,
        "cegb_tradeoff": 10.0,
        "force_col_wise": True,
4234
4235
4236
4237
4238
4239
    }

    model = lgb.train(params, train, num_boost_round=10)
    predicts = model.predict(test_data)
    rmse = np.sqrt(mean_squared_error(test_y, predicts))
    assert rmse < 10.0
4240
4241


4242
4243
4244
4245
def test_verbosity_and_verbose(capsys):
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, y)
    params = {
4246
4247
4248
        "num_leaves": 3,
        "verbose": 1,
        "verbosity": 0,
4249
4250
    }
    lgb.train(params, ds, num_boost_round=1)
4251
    expected_msg = "[LightGBM] [Warning] verbosity is set=0, verbose=1 will be ignored. " "Current value: verbosity=0"
4252
4253
4254
4255
    stdout = capsys.readouterr().out
    assert expected_msg in stdout


4256
4257
@pytest.mark.parametrize("verbosity_param", lgb.basic._ConfigAliases.get("verbosity"))
@pytest.mark.parametrize("verbosity", [-1, 0])
4258
4259
4260
4261
def test_verbosity_can_suppress_alias_warnings(capsys, verbosity_param, verbosity):
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, y)
    params = {
4262
4263
4264
4265
        "num_leaves": 3,
        "subsample": 0.75,
        "bagging_fraction": 0.8,
        "force_col_wise": True,
4266
4267
4268
4269
        verbosity_param: verbosity,
    }
    lgb.train(params, ds, num_boost_round=1)
    expected_msg = (
4270
4271
        "[LightGBM] [Warning] bagging_fraction is set=0.8, subsample=0.75 will be ignored. "
        "Current value: bagging_fraction=0.8"
4272
4273
4274
4275
4276
    )
    stdout = capsys.readouterr().out
    if verbosity >= 0:
        assert expected_msg in stdout
    else:
4277
        assert re.search(r"\[LightGBM\]", stdout) is None
4278
4279


4280
@pytest.mark.skipif(not PANDAS_INSTALLED, reason="pandas is not installed")
4281
4282
def test_validate_features():
    X, y = make_synthetic_regression()
4283
    features = ["x1", "x2", "x3", "x4"]
4284
4285
    df = pd_DataFrame(X, columns=features)
    ds = lgb.Dataset(df, y)
4286
    bst = lgb.train({"num_leaves": 15, "verbose": -1}, ds, num_boost_round=10)
4287
4288
4289
    assert bst.feature_name() == features

    # try to predict with a different feature
4290
    df2 = df.rename(columns={"x3": "z"})
4291
4292
4293
4294
4295
    with pytest.raises(lgb.basic.LightGBMError, match="Expected 'x3' at position 2 but found 'z'"):
        bst.predict(df2, validate_features=True)

    # check that disabling the check doesn't raise the error
    bst.predict(df2, validate_features=False)
4296
4297
4298
4299
4300
4301
4302

    # try to refit with a different feature
    with pytest.raises(lgb.basic.LightGBMError, match="Expected 'x3' at position 2 but found 'z'"):
        bst.refit(df2, y, validate_features=True)

    # check that disabling the check doesn't raise the error
    bst.refit(df2, y, validate_features=False)
4303
4304


4305
4306
4307
4308
4309
4310
4311
def test_train_and_cv_raise_informative_error_for_train_set_of_wrong_type():
    with pytest.raises(TypeError, match=r"train\(\) only accepts Dataset object, train_set has type 'list'\."):
        lgb.train({}, train_set=[])
    with pytest.raises(TypeError, match=r"cv\(\) only accepts Dataset object, train_set has type 'list'\."):
        lgb.cv({}, train_set=[])


4312
@pytest.mark.parametrize("num_boost_round", [-7, -1, 0])
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
def test_train_and_cv_raise_informative_error_for_impossible_num_boost_round(num_boost_round):
    X, y = make_synthetic_regression(n_samples=100)
    error_msg = rf"num_boost_round must be greater than 0\. Got {num_boost_round}\."
    with pytest.raises(ValueError, match=error_msg):
        lgb.train({}, train_set=lgb.Dataset(X, y), num_boost_round=num_boost_round)
    with pytest.raises(ValueError, match=error_msg):
        lgb.cv({}, train_set=lgb.Dataset(X, y), num_boost_round=num_boost_round)


def test_train_raises_informative_error_if_any_valid_sets_are_not_dataset_objects():
    X, y = make_synthetic_regression(n_samples=100)
    X_valid = X * 2.0
4325
4326
4327
    with pytest.raises(
        TypeError, match=r"Every item in valid_sets must be a Dataset object\. Item 1 has type 'tuple'\."
    ):
4328
4329
4330
        lgb.train(
            params={},
            train_set=lgb.Dataset(X, y),
4331
            valid_sets=[lgb.Dataset(X_valid, y), ([1.0], [2.0]), [5.6, 5.7, 5.8]],
4332
4333
4334
        )


4335
4336
def test_train_raises_informative_error_for_params_of_wrong_type():
    X, y = make_synthetic_regression()
4337
    params = {"num_leaves": "too-many"}
4338
    dtrain = lgb.Dataset(X, label=y)
4339
    with pytest.raises(lgb.basic.LightGBMError, match='Parameter num_leaves should be of type int, got "too-many"'):
4340
        lgb.train(params, dtrain)
4341
4342
4343
4344
4345


def test_quantized_training():
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, label=y)
4346
    bst_params = {"num_leaves": 15, "verbose": -1, "seed": 0}
4347
4348
    bst = lgb.train(bst_params, ds, num_boost_round=10)
    rmse = np.sqrt(np.mean((bst.predict(X) - y) ** 2))
4349
4350
4351
4352
4353
4354
4355
    bst_params.update(
        {
            "use_quantized_grad": True,
            "num_grad_quant_bins": 30,
            "quant_train_renew_leaf": True,
        }
    )
4356
4357
4358
    quant_bst = lgb.train(bst_params, ds, num_boost_round=10)
    quant_rmse = np.sqrt(np.mean((quant_bst.predict(X) - y) ** 2))
    assert quant_rmse < rmse + 6.0