test_engine.py 172 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
# coding: utf-8
wxchan's avatar
wxchan committed
2
import copy
3
import itertools
4
import json
wxchan's avatar
wxchan committed
5
import math
6
import pickle
7
import platform
8
import random
9
import re
10
from os import getenv
11
from pathlib import Path
12
from shutil import copyfile
wxchan's avatar
wxchan committed
13
14

import numpy as np
15
import psutil
16
import pytest
17
from scipy.sparse import csr_matrix, isspmatrix_csc, isspmatrix_csr
18
from sklearn.datasets import load_svmlight_file, make_blobs, make_multilabel_classification
19
20
from sklearn.metrics import average_precision_score, log_loss, mean_absolute_error, mean_squared_error, roc_auc_score
from sklearn.model_selection import GroupKFold, TimeSeriesSplit, train_test_split
wxchan's avatar
wxchan committed
21

22
import lightgbm as lgb
23
from lightgbm.compat import PANDAS_INSTALLED, pd_DataFrame, pd_Series
24

25
26
27
28
29
30
31
32
33
34
35
36
37
from .utils import (
    SERIALIZERS,
    dummy_obj,
    load_breast_cancer,
    load_digits,
    load_iris,
    logistic_sigmoid,
    make_synthetic_regression,
    mse_obj,
    pickle_and_unpickle_object,
    sklearn_multiclass_custom_objective,
    softmax,
)
wxchan's avatar
wxchan committed
38

39
40
41
decreasing_generator = itertools.count(0, -1)


42
43
44
45
46
47
def logloss_obj(preds, train_data):
    y_true = train_data.get_label()
    y_pred = logistic_sigmoid(preds)
    grad = y_pred - y_true
    hess = y_pred * (1.0 - y_pred)
    return grad, hess
48
49


wxchan's avatar
wxchan committed
50
51
52
def multi_logloss(y_true, y_pred):
    return np.mean([-math.log(y_pred[i][y]) for i, y in enumerate(y_true)])

wxchan's avatar
wxchan committed
53

Belinda Trotta's avatar
Belinda Trotta committed
54
55
56
57
58
59
60
def top_k_error(y_true, y_pred, k):
    if k == y_pred.shape[1]:
        return 0
    max_rest = np.max(-np.partition(-y_pred, k)[:, k:], axis=1)
    return 1 - np.mean((y_pred[np.arange(len(y_true)), y_true] > max_rest))


61
def constant_metric(preds, train_data):
62
    return ("error", 0.0, False)
63
64
65


def decreasing_metric(preds, train_data):
66
    return ("decreasing_metric", next(decreasing_generator), False)
67
68


69
70
71
72
def categorize(continuous_x):
    return np.digitize(continuous_x, bins=np.arange(0, 1, 0.01))


73
74
75
76
def test_binary():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
77
78
79
80
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
        "num_iteration": 50,  # test num_iteration in dict here
81
82
83
84
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    evals_result = {}
85
    gbm = lgb.train(
86
        params, lgb_train, num_boost_round=20, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
87
    )
88
89
    ret = log_loss(y_test, gbm.predict(X_test))
    assert ret < 0.14
90
91
    assert len(evals_result["valid_0"]["binary_logloss"]) == 50
    assert evals_result["valid_0"]["binary_logloss"][-1] == pytest.approx(ret)
92
93
94
95
96
97


def test_rf():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
98
99
100
101
102
103
104
105
        "boosting_type": "rf",
        "objective": "binary",
        "bagging_freq": 1,
        "bagging_fraction": 0.5,
        "feature_fraction": 0.5,
        "num_leaves": 50,
        "metric": "binary_logloss",
        "verbose": -1,
106
107
108
109
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    evals_result = {}
110
    gbm = lgb.train(
111
        params, lgb_train, num_boost_round=50, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
112
    )
113
114
    ret = log_loss(y_test, gbm.predict(X_test))
    assert ret < 0.19
115
    assert evals_result["valid_0"]["binary_logloss"][-1] == pytest.approx(ret)
116
117


118
@pytest.mark.parametrize("objective", ["regression", "regression_l1", "huber", "fair", "poisson", "quantile"])
119
def test_regression(objective):
120
121
    X, y = make_synthetic_regression()
    y = np.abs(y)
122
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
123
    params = {"objective": objective, "metric": "l2", "verbose": -1}
124
125
126
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    evals_result = {}
127
    gbm = lgb.train(
128
        params, lgb_train, num_boost_round=50, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
129
    )
130
    ret = mean_squared_error(y_test, gbm.predict(X_test))
131
    if objective == "huber":
132
        assert ret < 430
133
    elif objective == "fair":
134
        assert ret < 296
135
    elif objective == "poisson":
136
        assert ret < 193
137
    elif objective == "quantile":
138
        assert ret < 1311
139
    else:
140
        assert ret < 343
141
    assert evals_result["valid_0"]["l2"][-1] == pytest.approx(ret)
142
143
144
145
146
147
148
149
150
151
152
153


def test_missing_value_handle():
    X_train = np.zeros((100, 1))
    y_train = np.zeros(100)
    trues = random.sample(range(100), 20)
    for idx in trues:
        X_train[idx, 0] = np.nan
        y_train[idx] = 1
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

154
    params = {"metric": "l2", "verbose": -1, "boost_from_average": False}
155
    evals_result = {}
156
    gbm = lgb.train(
157
        params, lgb_train, num_boost_round=20, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
158
    )
159
160
    ret = mean_squared_error(y_train, gbm.predict(X_train))
    assert ret < 0.005
161
    assert evals_result["valid_0"]["l2"][-1] == pytest.approx(ret)
162
163
164
165
166
167
168
169
170
171
172
173


def test_missing_value_handle_more_na():
    X_train = np.ones((100, 1))
    y_train = np.ones(100)
    trues = random.sample(range(100), 80)
    for idx in trues:
        X_train[idx, 0] = np.nan
        y_train[idx] = 0
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

174
    params = {"metric": "l2", "verbose": -1, "boost_from_average": False}
175
    evals_result = {}
176
    gbm = lgb.train(
177
        params, lgb_train, num_boost_round=20, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
178
    )
179
180
    ret = mean_squared_error(y_train, gbm.predict(X_train))
    assert ret < 0.005
181
    assert evals_result["valid_0"]["l2"][-1] == pytest.approx(ret)
182
183
184
185
186
187
188
189
190
191
192
193


def test_missing_value_handle_na():
    x = [0, 1, 2, 3, 4, 5, 6, 7, np.nan]
    y = [1, 1, 1, 1, 0, 0, 0, 0, 1]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
194
195
196
197
198
199
200
201
202
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "zero_as_missing": False,
203
204
    }
    evals_result = {}
205
    gbm = lgb.train(
206
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
207
    )
208
209
210
211
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
212
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
213
214
215
216
217
218
219
220
221
222
223
224


def test_missing_value_handle_zero():
    x = [0, 1, 2, 3, 4, 5, 6, 7, np.nan]
    y = [0, 1, 1, 1, 0, 0, 0, 0, 0]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
225
226
227
228
229
230
231
232
233
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "zero_as_missing": True,
234
235
    }
    evals_result = {}
236
    gbm = lgb.train(
237
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
238
    )
239
240
241
242
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
243
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
244
245
246
247
248
249
250
251
252
253
254
255


def test_missing_value_handle_none():
    x = [0, 1, 2, 3, 4, 5, 6, 7, np.nan]
    y = [0, 1, 1, 1, 0, 0, 0, 0, 0]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
256
257
258
259
260
261
262
263
264
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "use_missing": False,
265
266
    }
    evals_result = {}
267
    gbm = lgb.train(
268
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
269
    )
270
271
272
273
274
    pred = gbm.predict(X_train)
    assert pred[0] == pytest.approx(pred[1])
    assert pred[-1] == pytest.approx(pred[0])
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.83
275
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
276
277


278
279
280
281
282
283
284
285
286
287
288
289
290
291
@pytest.mark.parametrize(
    "use_quantized_grad",
    [
        pytest.param(
            True,
            marks=pytest.mark.skipif(
                getenv("TASK", "") == "cuda",
                reason="Skip because quantized training with categorical features is not supported for cuda version",
            ),
        ),
        False,
    ],
)
def test_categorical_handle(use_quantized_grad):
292
293
294
295
296
297
298
299
300
    x = [0, 1, 2, 3, 4, 5, 6, 7]
    y = [0, 1, 0, 1, 0, 1, 0, 1]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
301
302
303
304
305
306
307
308
309
310
311
312
313
314
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "min_data_per_group": 1,
        "cat_smooth": 1,
        "cat_l2": 0,
        "max_cat_to_onehot": 1,
        "zero_as_missing": True,
        "categorical_column": 0,
315
        "use_quantized_grad": use_quantized_grad,
316
317
    }
    evals_result = {}
318
    gbm = lgb.train(
319
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
320
    )
321
322
323
324
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
325
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
326
327


328
329
330
331
332
333
334
335
336
337
338
339
340
341
@pytest.mark.parametrize(
    "use_quantized_grad",
    [
        pytest.param(
            True,
            marks=pytest.mark.skipif(
                getenv("TASK", "") == "cuda",
                reason="Skip because quantized training with categorical features is not supported for cuda version",
            ),
        ),
        False,
    ],
)
def test_categorical_handle_na(use_quantized_grad):
342
343
344
345
346
347
348
349
350
    x = [0, np.nan, 0, np.nan, 0, np.nan]
    y = [0, 1, 0, 1, 0, 1]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
351
352
353
354
355
356
357
358
359
360
361
362
363
364
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "min_data_per_group": 1,
        "cat_smooth": 1,
        "cat_l2": 0,
        "max_cat_to_onehot": 1,
        "zero_as_missing": False,
        "categorical_column": 0,
365
        "use_quantized_grad": use_quantized_grad,
366
367
    }
    evals_result = {}
368
    gbm = lgb.train(
369
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
370
    )
371
372
373
374
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
375
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
376
377


378
379
380
381
382
383
384
385
386
387
388
389
390
391
@pytest.mark.parametrize(
    "use_quantized_grad",
    [
        pytest.param(
            True,
            marks=pytest.mark.skipif(
                getenv("TASK", "") == "cuda",
                reason="Skip because quantized training with categorical features is not supported for cuda version",
            ),
        ),
        False,
    ],
)
def test_categorical_non_zero_inputs(use_quantized_grad):
392
393
394
395
396
397
398
399
400
    x = [1, 1, 1, 1, 1, 1, 2, 2]
    y = [1, 1, 1, 1, 1, 1, 0, 0]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
401
402
403
404
405
406
407
408
409
410
411
412
413
414
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "min_data_per_group": 1,
        "cat_smooth": 1,
        "cat_l2": 0,
        "max_cat_to_onehot": 1,
        "zero_as_missing": False,
        "categorical_column": 0,
415
        "use_quantized_grad": use_quantized_grad,
416
417
    }
    evals_result = {}
418
    gbm = lgb.train(
419
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
420
    )
421
422
423
424
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
425
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
426
427
428
429
430


def test_multiclass():
    X, y = load_digits(n_class=10, return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
431
    params = {"objective": "multiclass", "metric": "multi_logloss", "num_class": 10, "verbose": -1}
432
433
434
    lgb_train = lgb.Dataset(X_train, y_train, params=params)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, params=params)
    evals_result = {}
435
    gbm = lgb.train(
436
        params, lgb_train, num_boost_round=50, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
437
    )
438
439
    ret = multi_logloss(y_test, gbm.predict(X_test))
    assert ret < 0.16
440
    assert evals_result["valid_0"]["multi_logloss"][-1] == pytest.approx(ret)
441
442
443
444
445
446


def test_multiclass_rf():
    X, y = load_digits(n_class=10, return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
447
448
449
450
451
452
453
454
455
456
457
        "boosting_type": "rf",
        "objective": "multiclass",
        "metric": "multi_logloss",
        "bagging_freq": 1,
        "bagging_fraction": 0.6,
        "feature_fraction": 0.6,
        "num_class": 10,
        "num_leaves": 50,
        "min_data": 1,
        "verbose": -1,
        "gpu_use_dp": True,
458
459
460
461
    }
    lgb_train = lgb.Dataset(X_train, y_train, params=params)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, params=params)
    evals_result = {}
462
    gbm = lgb.train(
463
        params, lgb_train, num_boost_round=50, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
464
    )
465
466
    ret = multi_logloss(y_test, gbm.predict(X_test))
    assert ret < 0.23
467
    assert evals_result["valid_0"]["multi_logloss"][-1] == pytest.approx(ret)
468
469
470
471
472


def test_multiclass_prediction_early_stopping():
    X, y = load_digits(n_class=10, return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
473
    params = {"objective": "multiclass", "metric": "multi_logloss", "num_class": 10, "verbose": -1}
474
    lgb_train = lgb.Dataset(X_train, y_train, params=params)
475
    gbm = lgb.train(params, lgb_train, num_boost_round=50)
476

477
    pred_parameter = {"pred_early_stop": True, "pred_early_stop_freq": 5, "pred_early_stop_margin": 1.5}
478
479
480
481
    ret = multi_logloss(y_test, gbm.predict(X_test, **pred_parameter))
    assert ret < 0.8
    assert ret > 0.6  # loss will be higher than when evaluating the full model

482
    pred_parameter["pred_early_stop_margin"] = 5.5
483
484
485
486
487
488
    ret = multi_logloss(y_test, gbm.predict(X_test, **pred_parameter))
    assert ret < 0.2


def test_multi_class_error():
    X, y = load_digits(n_class=10, return_X_y=True)
489
    params = {"objective": "multiclass", "num_classes": 10, "metric": "multi_error", "num_leaves": 4, "verbose": -1}
490
491
492
493
    lgb_data = lgb.Dataset(X, label=y)
    est = lgb.train(params, lgb_data, num_boost_round=10)
    predict_default = est.predict(X)
    results = {}
494
    est = lgb.train(
495
        dict(params, multi_error_top_k=1),
496
497
498
        lgb_data,
        num_boost_round=10,
        valid_sets=[lgb_data],
499
        callbacks=[lgb.record_evaluation(results)],
500
    )
501
502
503
504
505
    predict_1 = est.predict(X)
    # check that default gives same result as k = 1
    np.testing.assert_allclose(predict_1, predict_default)
    # check against independent calculation for k = 1
    err = top_k_error(y, predict_1, 1)
506
    assert results["training"]["multi_error"][-1] == pytest.approx(err)
507
508
    # check against independent calculation for k = 2
    results = {}
509
    est = lgb.train(
510
        dict(params, multi_error_top_k=2),
511
512
513
        lgb_data,
        num_boost_round=10,
        valid_sets=[lgb_data],
514
        callbacks=[lgb.record_evaluation(results)],
515
    )
516
517
    predict_2 = est.predict(X)
    err = top_k_error(y, predict_2, 2)
518
    assert results["training"]["multi_error@2"][-1] == pytest.approx(err)
519
520
    # check against independent calculation for k = 10
    results = {}
521
    est = lgb.train(
522
        dict(params, multi_error_top_k=10),
523
524
525
        lgb_data,
        num_boost_round=10,
        valid_sets=[lgb_data],
526
        callbacks=[lgb.record_evaluation(results)],
527
    )
528
529
    predict_3 = est.predict(X)
    err = top_k_error(y, predict_3, 10)
530
    assert results["training"]["multi_error@10"][-1] == pytest.approx(err)
531
532
533
534
    # check cases where predictions are equal
    X = np.array([[0, 0], [0, 0]])
    y = np.array([0, 1])
    lgb_data = lgb.Dataset(X, label=y)
535
    params["num_classes"] = 2
536
    results = {}
537
538
    lgb.train(params, lgb_data, num_boost_round=10, valid_sets=[lgb_data], callbacks=[lgb.record_evaluation(results)])
    assert results["training"]["multi_error"][-1] == pytest.approx(1)
539
    results = {}
540
    lgb.train(
541
        dict(params, multi_error_top_k=2),
542
543
544
        lgb_data,
        num_boost_round=10,
        valid_sets=[lgb_data],
545
        callbacks=[lgb.record_evaluation(results)],
546
    )
547
    assert results["training"]["multi_error@2"][-1] == pytest.approx(0)
548
549


550
551
552
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Skip due to differences in implementation details of CUDA version"
)
553
def test_auc_mu(rng):
554
555
556
557
558
    # should give same result as binary auc for 2 classes
    X, y = load_digits(n_class=10, return_X_y=True)
    y_new = np.zeros((len(y)))
    y_new[y != 0] = 1
    lgb_X = lgb.Dataset(X, label=y_new)
559
    params = {"objective": "multiclass", "metric": "auc_mu", "verbose": -1, "num_classes": 2, "seed": 0}
560
    results_auc_mu = {}
561
562
    lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_auc_mu)])
    params = {"objective": "binary", "metric": "auc", "verbose": -1, "seed": 0}
563
    results_auc = {}
564
565
    lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_auc)])
    np.testing.assert_allclose(results_auc_mu["training"]["auc_mu"], results_auc["training"]["auc"])
566
567
    # test the case where all predictions are equal
    lgb_X = lgb.Dataset(X[:10], label=y_new[:10])
568
569
570
571
572
573
574
575
    params = {
        "objective": "multiclass",
        "metric": "auc_mu",
        "verbose": -1,
        "num_classes": 2,
        "min_data_in_leaf": 20,
        "seed": 0,
    }
576
    results_auc_mu = {}
577
578
    lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_auc_mu)])
    assert results_auc_mu["training"]["auc_mu"][-1] == pytest.approx(0.5)
579
580
    # test that weighted data gives different auc_mu
    lgb_X = lgb.Dataset(X, label=y)
581
    lgb_X_weighted = lgb.Dataset(X, label=y, weight=np.abs(rng.standard_normal(size=y.shape)))
582
583
584
    results_unweighted = {}
    results_weighted = {}
    params = dict(params, num_classes=10, num_leaves=5)
585
    lgb.train(
586
        params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_unweighted)]
587
588
589
590
591
592
    )
    lgb.train(
        params,
        lgb_X_weighted,
        num_boost_round=10,
        valid_sets=[lgb_X_weighted],
593
        callbacks=[lgb.record_evaluation(results_weighted)],
594
    )
595
596
    assert results_weighted["training"]["auc_mu"][-1] < 1
    assert results_unweighted["training"]["auc_mu"][-1] != results_weighted["training"]["auc_mu"][-1]
597
598
    # test that equal data weights give same auc_mu as unweighted data
    lgb_X_weighted = lgb.Dataset(X, label=y, weight=np.ones(y.shape) * 0.5)
599
600
601
602
603
    lgb.train(
        params,
        lgb_X_weighted,
        num_boost_round=10,
        valid_sets=[lgb_X_weighted],
604
605
606
607
        callbacks=[lgb.record_evaluation(results_weighted)],
    )
    assert results_unweighted["training"]["auc_mu"][-1] == pytest.approx(
        results_weighted["training"]["auc_mu"][-1], abs=1e-5
608
    )
609
610
611
612
    # should give 1 when accuracy = 1
    X = X[:10, :]
    y = y[:10]
    lgb_X = lgb.Dataset(X, label=y)
613
    params = {"objective": "multiclass", "metric": "auc_mu", "num_classes": 10, "min_data_in_leaf": 1, "verbose": -1}
614
    results = {}
615
616
    lgb.train(params, lgb_X, num_boost_round=100, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results)])
    assert results["training"]["auc_mu"][-1] == pytest.approx(1)
617
    # test loading class weights
618
    Xy = np.loadtxt(
619
        str(Path(__file__).absolute().parents[2] / "examples" / "multiclass_classification" / "multiclass.train")
620
    )
621
622
623
    y = Xy[:, 0]
    X = Xy[:, 1:]
    lgb_X = lgb.Dataset(X, label=y)
624
625
626
627
628
629
630
631
    params = {
        "objective": "multiclass",
        "metric": "auc_mu",
        "auc_mu_weights": [0, 2, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0],
        "num_classes": 5,
        "verbose": -1,
        "seed": 0,
    }
632
    results_weight = {}
633
634
    lgb.train(params, lgb_X, num_boost_round=5, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_weight)])
    params["auc_mu_weights"] = []
635
    results_no_weight = {}
636
    lgb.train(
637
        params, lgb_X, num_boost_round=5, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_no_weight)]
638
    )
639
    assert results_weight["training"]["auc_mu"][-1] != results_no_weight["training"]["auc_mu"][-1]
640
641


642
def test_ranking_prediction_early_stopping():
643
644
645
646
647
    rank_example_dir = Path(__file__).absolute().parents[2] / "examples" / "lambdarank"
    X_train, y_train = load_svmlight_file(str(rank_example_dir / "rank.train"))
    q_train = np.loadtxt(str(rank_example_dir / "rank.train.query"))
    X_test, _ = load_svmlight_file(str(rank_example_dir / "rank.test"))
    params = {"objective": "rank_xendcg", "verbose": -1}
648
649
650
    lgb_train = lgb.Dataset(X_train, y_train, group=q_train, params=params)
    gbm = lgb.train(params, lgb_train, num_boost_round=50)

651
    pred_parameter = {"pred_early_stop": True, "pred_early_stop_freq": 5, "pred_early_stop_margin": 1.5}
652
653
654
655
656
657
658
659
    ret_early = gbm.predict(X_test, **pred_parameter)

    pred_parameter["pred_early_stop_margin"] = 5.5
    ret_early_more_strict = gbm.predict(X_test, **pred_parameter)
    with pytest.raises(AssertionError):
        np.testing.assert_allclose(ret_early, ret_early_more_strict)


660
661
662
663
664
665
666
# Simulates position bias for a given ranking dataset.
# The ouput dataset is identical to the input one with the exception for the relevance labels.
# The new labels are generated according to an instance of a cascade user model:
# for each query, the user is simulated to be traversing the list of documents ranked by a baseline ranker
# (in our example it is simply the ordering by some feature correlated with relevance, e.g., 34)
# and clicks on that document (new_label=1) with some probability 'pclick' depending on its true relevance;
# at each position the user may stop the traversal with some probability pstop. For the non-clicked documents,
667
# new_label=0. Thus the generated new labels are biased towards the baseline ranker.
668
669
670
671
672
673
674
675
676
677
678
679
680
681
# The positions of the documents in the ranked lists produced by the baseline, are returned.
def simulate_position_bias(file_dataset_in, file_query_in, file_dataset_out, baseline_feature):
    # a mapping of a document's true relevance (defined on a 5-grade scale) into the probability of clicking it
    def get_pclick(label):
        if label == 0:
            return 0.4
        elif label == 1:
            return 0.6
        elif label == 2:
            return 0.7
        elif label == 3:
            return 0.8
        else:
            return 0.9
682

683
684
    # an instantiation of a cascade model where the user stops with probability 0.2 after observing each document
    pstop = 0.2
685

686
687
    f_dataset_in = open(file_dataset_in, "r")
    f_dataset_out = open(file_dataset_out, "w")
688
689
690
    random.seed(10)
    positions_all = []
    for line in open(file_query_in):
691
        docs_num = int(line)
692
        lines = []
693
        index_values = []
694
695
696
697
698
699
        positions = [0] * docs_num
        for index in range(docs_num):
            features = f_dataset_in.readline().split()
            lines.append(features)
            val = 0.0
            for feature_val in features:
700
                feature_val_split = feature_val.split(":")
701
702
703
704
                if int(feature_val_split[0]) == baseline_feature:
                    val = float(feature_val_split[1])
            index_values.append([index, val])
        index_values.sort(key=lambda x: -x[1])
705
        stop = False
706
707
708
709
710
711
712
        for pos in range(docs_num):
            index = index_values[pos][0]
            new_label = 0
            if not stop:
                label = int(lines[index][0])
                pclick = get_pclick(label)
                if random.random() < pclick:
713
                    new_label = 1
714
715
716
717
                stop = random.random() < pstop
            lines[index][0] = str(new_label)
            positions[index] = pos
        for features in lines:
718
            f_dataset_out.write(" ".join(features) + "\n")
719
720
721
722
723
        positions_all.extend(positions)
    f_dataset_out.close()
    return positions_all


724
725
726
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Positions in learning to rank is not supported in CUDA version yet"
)
727
def test_ranking_with_position_information_with_file(tmp_path):
728
    rank_example_dir = Path(__file__).absolute().parents[2] / "examples" / "lambdarank"
729
    params = {
730
731
732
733
734
735
736
737
        "objective": "lambdarank",
        "verbose": -1,
        "eval_at": [3],
        "metric": "ndcg",
        "bagging_freq": 1,
        "bagging_fraction": 0.9,
        "min_data_in_leaf": 50,
        "min_sum_hessian_in_leaf": 5.0,
738
739
740
    }

    # simulate position bias for the train dataset and put the train dataset with biased labels to temp directory
741
742
743
744
745
746
747
748
749
    positions = simulate_position_bias(
        str(rank_example_dir / "rank.train"),
        str(rank_example_dir / "rank.train.query"),
        str(tmp_path / "rank.train"),
        baseline_feature=34,
    )
    copyfile(str(rank_example_dir / "rank.train.query"), str(tmp_path / "rank.train.query"))
    copyfile(str(rank_example_dir / "rank.test"), str(tmp_path / "rank.test"))
    copyfile(str(rank_example_dir / "rank.test.query"), str(tmp_path / "rank.test.query"))
750

751
752
753
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
    gbm_baseline = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
754

755
    f_positions_out = open(str(tmp_path / "rank.train.position"), "w")
756
    for pos in positions:
757
        f_positions_out.write(str(pos) + "\n")
758
759
    f_positions_out.close()

760
761
762
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
    gbm_unbiased_with_file = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
763

764
    # the performance of the unbiased LambdaMART should outperform the plain LambdaMART on the dataset with position bias
765
    assert gbm_baseline.best_score["valid_0"]["ndcg@3"] + 0.03 <= gbm_unbiased_with_file.best_score["valid_0"]["ndcg@3"]
766
767

    # add extra row to position file
768
769
    with open(str(tmp_path / "rank.train.position"), "a") as file:
        file.write("pos_1000\n")
770
        file.close()
771
772
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
773
    with pytest.raises(lgb.basic.LightGBMError, match=r"Positions size \(3006\) doesn't match data size"):
774
        lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
775
776


777
778
779
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Positions in learning to rank is not supported in CUDA version yet"
)
780
def test_ranking_with_position_information_with_dataset_constructor(tmp_path):
781
    rank_example_dir = Path(__file__).absolute().parents[2] / "examples" / "lambdarank"
782
    params = {
783
784
785
786
787
788
789
790
791
792
793
        "objective": "lambdarank",
        "verbose": -1,
        "eval_at": [3],
        "metric": "ndcg",
        "bagging_freq": 1,
        "bagging_fraction": 0.9,
        "min_data_in_leaf": 50,
        "min_sum_hessian_in_leaf": 5.0,
        "num_threads": 1,
        "deterministic": True,
        "seed": 0,
794
795
796
    }

    # simulate position bias for the train dataset and put the train dataset with biased labels to temp directory
797
798
799
800
801
802
803
804
805
    positions = simulate_position_bias(
        str(rank_example_dir / "rank.train"),
        str(rank_example_dir / "rank.train.query"),
        str(tmp_path / "rank.train"),
        baseline_feature=34,
    )
    copyfile(str(rank_example_dir / "rank.train.query"), str(tmp_path / "rank.train.query"))
    copyfile(str(rank_example_dir / "rank.test"), str(tmp_path / "rank.test"))
    copyfile(str(rank_example_dir / "rank.test.query"), str(tmp_path / "rank.test.query"))
806

807
808
809
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
    gbm_baseline = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
810
811
812
813

    positions = np.array(positions)

    # test setting positions through Dataset constructor with numpy array
814
815
816
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params, position=positions)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
    gbm_unbiased = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
817
818

    # the performance of the unbiased LambdaMART should outperform the plain LambdaMART on the dataset with position bias
819
    assert gbm_baseline.best_score["valid_0"]["ndcg@3"] + 0.03 <= gbm_unbiased.best_score["valid_0"]["ndcg@3"]
820
821
822

    if PANDAS_INSTALLED:
        # test setting positions through Dataset constructor with pandas Series
823
824
825
826
827
828
        lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params, position=pd_Series(positions))
        lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
        gbm_unbiased_pandas_series = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
        assert (
            gbm_unbiased.best_score["valid_0"]["ndcg@3"] == gbm_unbiased_pandas_series.best_score["valid_0"]["ndcg@3"]
        )
829
830

    # test setting positions through set_position
831
832
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
833
    lgb_train.set_position(positions)
834
835
    gbm_unbiased_set_position = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
    assert gbm_unbiased.best_score["valid_0"]["ndcg@3"] == gbm_unbiased_set_position.best_score["valid_0"]["ndcg@3"]
836
837
838
839
840
841

    # test get_position works
    positions_from_get = lgb_train.get_position()
    np.testing.assert_array_equal(positions_from_get, positions)


842
843
def test_early_stopping():
    X, y = load_breast_cancer(return_X_y=True)
844
    params = {"objective": "binary", "metric": "binary_logloss", "verbose": -1}
845
846
847
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
848
    valid_set_name = "valid_set"
849
    # no early stopping
850
851
852
853
854
855
856
857
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=10,
        valid_sets=lgb_eval,
        valid_names=valid_set_name,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
    )
858
859
    assert gbm.best_iteration == 10
    assert valid_set_name in gbm.best_score
860
    assert "binary_logloss" in gbm.best_score[valid_set_name]
861
    # early stopping occurs
862
863
864
865
866
867
868
869
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=40,
        valid_sets=lgb_eval,
        valid_names=valid_set_name,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
    )
870
871
    assert gbm.best_iteration <= 39
    assert valid_set_name in gbm.best_score
872
    assert "binary_logloss" in gbm.best_score[valid_set_name]
873
874


875
@pytest.mark.parametrize("use_valid", [True, False])
876
877
878
879
880
881
882
883
884
def test_early_stopping_ignores_training_set(use_valid):
    x = np.linspace(-1, 1, 100)
    X = x.reshape(-1, 1)
    y = x**2
    X_train, X_valid = X[:80], X[80:]
    y_train, y_valid = y[:80], y[80:]
    train_ds = lgb.Dataset(X_train, y_train)
    valid_ds = lgb.Dataset(X_valid, y_valid)
    valid_sets = [train_ds]
885
    valid_names = ["train"]
886
887
    if use_valid:
        valid_sets.append(valid_ds)
888
        valid_names.append("valid")
889
890
891
892
    eval_result = {}

    def train_fn():
        return lgb.train(
893
            {"num_leaves": 5},
894
895
896
897
            train_ds,
            num_boost_round=2,
            valid_sets=valid_sets,
            valid_names=valid_names,
898
            callbacks=[lgb.early_stopping(1), lgb.record_evaluation(eval_result)],
899
        )
900

901
902
903
    if use_valid:
        bst = train_fn()
        assert bst.best_iteration == 1
904
905
        assert eval_result["train"]["l2"][1] < eval_result["train"]["l2"][0]  # train improved
        assert eval_result["valid"]["l2"][1] > eval_result["valid"]["l2"][0]  # valid didn't
906
    else:
907
        with pytest.warns(UserWarning, match="Only training set found, disabling early stopping."):
908
909
910
911
912
            bst = train_fn()
        assert bst.current_iteration() == 2
        assert bst.best_iteration == 0


913
@pytest.mark.parametrize("first_metric_only", [True, False])
914
915
916
917
def test_early_stopping_via_global_params(first_metric_only):
    X, y = load_breast_cancer(return_X_y=True)
    num_trees = 5
    params = {
918
919
920
921
922
923
        "num_trees": num_trees,
        "objective": "binary",
        "metric": "None",
        "verbose": -1,
        "early_stopping_round": 2,
        "first_metric_only": first_metric_only,
924
925
926
927
    }
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
928
929
930
931
    valid_set_name = "valid_set"
    gbm = lgb.train(
        params, lgb_train, feval=[decreasing_metric, constant_metric], valid_sets=lgb_eval, valid_names=valid_set_name
    )
932
933
934
935
936
    if first_metric_only:
        assert gbm.best_iteration == num_trees
    else:
        assert gbm.best_iteration == 1
    assert valid_set_name in gbm.best_score
937
938
    assert "decreasing_metric" in gbm.best_score[valid_set_name]
    assert "error" in gbm.best_score[valid_set_name]
939
940


941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
@pytest.mark.parametrize("early_stopping_round", [-10, -1, 0, None, "None"])
def test_early_stopping_is_not_enabled_for_non_positive_stopping_rounds(early_stopping_round):
    X, y = load_breast_cancer(return_X_y=True)
    num_trees = 5
    params = {
        "num_trees": num_trees,
        "objective": "binary",
        "metric": "None",
        "verbose": -1,
        "early_stopping_round": early_stopping_round,
        "first_metric_only": True,
    }
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    valid_set_name = "valid_set"

    if early_stopping_round is None:
        gbm = lgb.train(
            params,
            lgb_train,
            feval=[constant_metric],
            valid_sets=lgb_eval,
            valid_names=valid_set_name,
        )
        assert "early_stopping_round" not in gbm.params
        assert gbm.num_trees() == num_trees
    elif early_stopping_round == "None":
        with pytest.raises(TypeError, match="early_stopping_round should be an integer. Got 'str'"):
            gbm = lgb.train(
                params,
                lgb_train,
                feval=[constant_metric],
                valid_sets=lgb_eval,
                valid_names=valid_set_name,
            )
    elif early_stopping_round <= 0:
        gbm = lgb.train(
            params,
            lgb_train,
            feval=[constant_metric],
            valid_sets=lgb_eval,
            valid_names=valid_set_name,
        )
        assert gbm.params["early_stopping_round"] == early_stopping_round
        assert gbm.num_trees() == num_trees


989
990
991
@pytest.mark.parametrize("first_only", [True, False])
@pytest.mark.parametrize("single_metric", [True, False])
@pytest.mark.parametrize("greater_is_better", [True, False])
992
993
994
995
def test_early_stopping_min_delta(first_only, single_metric, greater_is_better):
    if single_metric and not first_only:
        pytest.skip("first_metric_only doesn't affect single metric.")
    metric2min_delta = {
996
997
998
999
        "auc": 0.001,
        "binary_logloss": 0.01,
        "average_precision": 0.001,
        "mape": 0.01,
1000
1001
1002
    }
    if single_metric:
        if greater_is_better:
1003
            metric = "auc"
1004
        else:
1005
            metric = "binary_logloss"
1006
1007
1008
    else:
        if first_only:
            if greater_is_better:
1009
                metric = ["auc", "binary_logloss"]
1010
            else:
1011
                metric = ["binary_logloss", "auc"]
1012
1013
        else:
            if greater_is_better:
1014
                metric = ["auc", "average_precision"]
1015
            else:
1016
                metric = ["binary_logloss", "mape"]
1017
1018
1019
1020
1021
1022

    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=0)
    train_ds = lgb.Dataset(X_train, y_train)
    valid_ds = lgb.Dataset(X_valid, y_valid, reference=train_ds)

1023
    params = {"objective": "binary", "metric": metric, "verbose": -1}
1024
1025
1026
1027
1028
1029
    if isinstance(metric, str):
        min_delta = metric2min_delta[metric]
    elif first_only:
        min_delta = metric2min_delta[metric[0]]
    else:
        min_delta = [metric2min_delta[m] for m in metric]
1030
1031
1032
1033
1034
    train_kwargs = {
        "params": params,
        "train_set": train_ds,
        "num_boost_round": 50,
        "valid_sets": [train_ds, valid_ds],
1035
        "valid_names": ["training", "valid"],
1036
    }
1037
1038
1039

    # regular early stopping
    evals_result = {}
1040
    train_kwargs["callbacks"] = [
1041
        lgb.callback.early_stopping(10, first_only, verbose=False),
1042
        lgb.record_evaluation(evals_result),
1043
1044
    ]
    bst = lgb.train(**train_kwargs)
1045
    scores = np.vstack(list(evals_result["valid"].values())).T
1046
1047
1048

    # positive min_delta
    delta_result = {}
1049
    train_kwargs["callbacks"] = [
1050
        lgb.callback.early_stopping(10, first_only, verbose=False, min_delta=min_delta),
1051
        lgb.record_evaluation(delta_result),
1052
1053
    ]
    delta_bst = lgb.train(**train_kwargs)
1054
    delta_scores = np.vstack(list(delta_result["valid"].values())).T
1055
1056
1057
1058
1059
1060

    if first_only:
        scores = scores[:, 0]
        delta_scores = delta_scores[:, 0]

    assert delta_bst.num_trees() < bst.num_trees()
1061
    np.testing.assert_allclose(scores[: len(delta_scores)], delta_scores)
1062
1063
1064
1065
1066
1067
1068
1069
    last_score = delta_scores[-1]
    best_score = delta_scores[delta_bst.num_trees() - 1]
    if greater_is_better:
        assert np.less_equal(last_score, best_score + min_delta).any()
    else:
        assert np.greater_equal(last_score, best_score - min_delta).any()


1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
@pytest.mark.parametrize("early_stopping_min_delta", [1e3, 0.0])
def test_early_stopping_min_delta_via_global_params(early_stopping_min_delta):
    X, y = load_breast_cancer(return_X_y=True)
    num_trees = 5
    params = {
        "num_trees": num_trees,
        "num_leaves": 5,
        "objective": "binary",
        "metric": "None",
        "verbose": -1,
        "early_stopping_round": 2,
        "early_stopping_min_delta": early_stopping_min_delta,
    }
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    gbm = lgb.train(params, lgb_train, feval=decreasing_metric, valid_sets=lgb_eval)
    if early_stopping_min_delta == 0:
        assert gbm.best_iteration == num_trees
    else:
        assert gbm.best_iteration == 1


1093
1094
1095
1096
1097
1098
def test_early_stopping_can_be_triggered_via_custom_callback():
    X, y = make_synthetic_regression()

    def _early_stop_after_seventh_iteration(env):
        if env.iteration == 6:
            exc = lgb.EarlyStopException(
1099
                best_iteration=6, best_score=[("some_validation_set", "some_metric", 0.708, True)]
1100
1101
1102
1103
            )
            raise exc

    bst = lgb.train(
1104
        params={"objective": "regression", "verbose": -1, "num_leaves": 2},
1105
1106
        train_set=lgb.Dataset(X, label=y),
        num_boost_round=23,
1107
        callbacks=[_early_stop_after_seventh_iteration],
1108
1109
1110
1111
1112
1113
1114
    )
    assert bst.num_trees() == 7
    assert bst.best_score["some_validation_set"]["some_metric"] == 0.708
    assert bst.best_iteration == 7
    assert bst.current_iteration() == 7


1115
def test_continue_train():
1116
    X, y = make_synthetic_regression()
1117
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
1118
    params = {"objective": "regression", "metric": "l1", "verbose": -1}
1119
1120
1121
    lgb_train = lgb.Dataset(X_train, y_train, free_raw_data=False)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, free_raw_data=False)
    init_gbm = lgb.train(params, lgb_train, num_boost_round=20)
1122
    model_name = "model.txt"
1123
1124
    init_gbm.save_model(model_name)
    evals_result = {}
1125
1126
1127
1128
1129
1130
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=30,
        valid_sets=lgb_eval,
        # test custom eval metrics
1131
        feval=(lambda p, d: ("custom_mae", mean_absolute_error(p, d.get_label()), False)),
1132
        callbacks=[lgb.record_evaluation(evals_result)],
1133
        init_model="model.txt",
1134
    )
1135
    ret = mean_absolute_error(y_test, gbm.predict(X_test))
1136
    assert ret < 13.6
1137
1138
    assert evals_result["valid_0"]["l1"][-1] == pytest.approx(ret)
    np.testing.assert_allclose(evals_result["valid_0"]["l1"], evals_result["valid_0"]["custom_mae"])
1139
1140
1141


def test_continue_train_reused_dataset():
1142
    X, y = make_synthetic_regression()
1143
    params = {"objective": "regression", "verbose": -1}
1144
1145
1146
1147
1148
1149
1150
1151
1152
    lgb_train = lgb.Dataset(X, y, free_raw_data=False)
    init_gbm = lgb.train(params, lgb_train, num_boost_round=5)
    init_gbm_2 = lgb.train(params, lgb_train, num_boost_round=5, init_model=init_gbm)
    init_gbm_3 = lgb.train(params, lgb_train, num_boost_round=5, init_model=init_gbm_2)
    gbm = lgb.train(params, lgb_train, num_boost_round=5, init_model=init_gbm_3)
    assert gbm.current_iteration() == 20


def test_continue_train_dart():
1153
    X, y = make_synthetic_regression()
1154
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
1155
    params = {"boosting_type": "dart", "objective": "regression", "metric": "l1", "verbose": -1}
1156
1157
1158
1159
    lgb_train = lgb.Dataset(X_train, y_train, free_raw_data=False)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, free_raw_data=False)
    init_gbm = lgb.train(params, lgb_train, num_boost_round=50)
    evals_result = {}
1160
1161
1162
1163
1164
1165
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=50,
        valid_sets=lgb_eval,
        callbacks=[lgb.record_evaluation(evals_result)],
1166
        init_model=init_gbm,
1167
    )
1168
    ret = mean_absolute_error(y_test, gbm.predict(X_test))
1169
    assert ret < 13.6
1170
    assert evals_result["valid_0"]["l1"][-1] == pytest.approx(ret)
1171
1172
1173
1174
1175


def test_continue_train_multiclass():
    X, y = load_iris(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
1176
    params = {"objective": "multiclass", "metric": "multi_logloss", "num_class": 3, "verbose": -1}
1177
1178
1179
1180
    lgb_train = lgb.Dataset(X_train, y_train, params=params, free_raw_data=False)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, params=params, free_raw_data=False)
    init_gbm = lgb.train(params, lgb_train, num_boost_round=20)
    evals_result = {}
1181
1182
1183
1184
1185
1186
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=30,
        valid_sets=lgb_eval,
        callbacks=[lgb.record_evaluation(evals_result)],
1187
        init_model=init_gbm,
1188
    )
1189
1190
    ret = multi_logloss(y_test, gbm.predict(X_test))
    assert ret < 0.1
1191
    assert evals_result["valid_0"]["multi_logloss"][-1] == pytest.approx(ret)
1192
1193
1194


def test_cv():
1195
    X_train, y_train = make_synthetic_regression()
1196
    params = {"verbose": -1}
1197
1198
    lgb_train = lgb.Dataset(X_train, y_train)
    # shuffle = False, override metric in params
1199
1200
1201
1202
1203
1204
1205
    params_with_metric = {"metric": "l2", "verbose": -1}
    cv_res = lgb.cv(
        params_with_metric, lgb_train, num_boost_round=10, nfold=3, stratified=False, shuffle=False, metrics="l1"
    )
    assert "valid l1-mean" in cv_res
    assert "valid l2-mean" not in cv_res
    assert len(cv_res["valid l1-mean"]) == 10
1206
    # shuffle = True, callbacks
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
    cv_res = lgb.cv(
        params,
        lgb_train,
        num_boost_round=10,
        nfold=3,
        stratified=False,
        shuffle=True,
        metrics="l1",
        callbacks=[lgb.reset_parameter(learning_rate=lambda i: 0.1 - 0.001 * i)],
    )
    assert "valid l1-mean" in cv_res
    assert len(cv_res["valid l1-mean"]) == 10
1219
    # enable display training loss
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
    cv_res = lgb.cv(
        params_with_metric,
        lgb_train,
        num_boost_round=10,
        nfold=3,
        stratified=False,
        shuffle=False,
        metrics="l1",
        eval_train_metric=True,
    )
    assert "train l1-mean" in cv_res
    assert "valid l1-mean" in cv_res
    assert "train l2-mean" not in cv_res
    assert "valid l2-mean" not in cv_res
    assert len(cv_res["train l1-mean"]) == 10
    assert len(cv_res["valid l1-mean"]) == 10
1236
1237
1238
    # self defined folds
    tss = TimeSeriesSplit(3)
    folds = tss.split(X_train)
1239
1240
    cv_res_gen = lgb.cv(params_with_metric, lgb_train, num_boost_round=10, folds=folds)
    cv_res_obj = lgb.cv(params_with_metric, lgb_train, num_boost_round=10, folds=tss)
1241
    np.testing.assert_allclose(cv_res_gen["valid l2-mean"], cv_res_obj["valid l2-mean"])
Andrew Ziem's avatar
Andrew Ziem committed
1242
    # LambdaRank
1243
1244
1245
1246
    rank_example_dir = Path(__file__).absolute().parents[2] / "examples" / "lambdarank"
    X_train, y_train = load_svmlight_file(str(rank_example_dir / "rank.train"))
    q_train = np.loadtxt(str(rank_example_dir / "rank.train.query"))
    params_lambdarank = {"objective": "lambdarank", "verbose": -1, "eval_at": 3}
1247
1248
    lgb_train = lgb.Dataset(X_train, y_train, group=q_train)
    # ... with l2 metric
1249
    cv_res_lambda = lgb.cv(params_lambdarank, lgb_train, num_boost_round=10, nfold=3, metrics="l2")
1250
    assert len(cv_res_lambda) == 2
1251
    assert not np.isnan(cv_res_lambda["valid l2-mean"]).any()
1252
    # ... with NDCG (default) metric
1253
    cv_res_lambda = lgb.cv(params_lambdarank, lgb_train, num_boost_round=10, nfold=3)
1254
    assert len(cv_res_lambda) == 2
1255
    assert not np.isnan(cv_res_lambda["valid ndcg@3-mean"]).any()
1256
    # self defined folds with lambdarank
1257
1258
    cv_res_lambda_obj = lgb.cv(params_lambdarank, lgb_train, num_boost_round=10, folds=GroupKFold(n_splits=3))
    np.testing.assert_allclose(cv_res_lambda["valid ndcg@3-mean"], cv_res_lambda_obj["valid ndcg@3-mean"])
1259
1260


1261
1262
def test_cv_works_with_init_model(tmp_path):
    X, y = make_synthetic_regression()
1263
    params = {"objective": "regression", "verbose": -1}
1264
1265
    num_train_rounds = 2
    lgb_train = lgb.Dataset(X, y, free_raw_data=False)
1266
    bst = lgb.train(params=params, train_set=lgb_train, num_boost_round=num_train_rounds)
1267
    preds_raw = bst.predict(X, raw_score=True)
1268
    model_path_txt = str(tmp_path / "lgb.model")
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
    bst.save_model(model_path_txt)

    num_cv_rounds = 5
    cv_kwargs = {
        "num_boost_round": num_cv_rounds,
        "nfold": 3,
        "stratified": False,
        "shuffle": False,
        "seed": 708,
        "return_cvbooster": True,
1279
        "params": params,
1280
1281
1282
    }

    # init_model from an in-memory Booster
1283
    cv_res = lgb.cv(train_set=lgb_train, init_model=bst, **cv_kwargs)
1284
1285
1286
    cv_bst_w_in_mem_init_model = cv_res["cvbooster"]
    assert cv_bst_w_in_mem_init_model.current_iteration() == [num_train_rounds + num_cv_rounds] * 3
    for booster in cv_bst_w_in_mem_init_model.boosters:
1287
        np.testing.assert_allclose(preds_raw, booster.predict(X, raw_score=True, num_iteration=num_train_rounds))
1288
1289

    # init_model from a text file
1290
    cv_res = lgb.cv(train_set=lgb_train, init_model=model_path_txt, **cv_kwargs)
1291
1292
1293
    cv_bst_w_file_init_model = cv_res["cvbooster"]
    assert cv_bst_w_file_init_model.current_iteration() == [num_train_rounds + num_cv_rounds] * 3
    for booster in cv_bst_w_file_init_model.boosters:
1294
        np.testing.assert_allclose(preds_raw, booster.predict(X, raw_score=True, num_iteration=num_train_rounds))
1295
1296
1297
1298

    # predictions should be identical
    for i in range(3):
        np.testing.assert_allclose(
1299
            cv_bst_w_in_mem_init_model.boosters[i].predict(X), cv_bst_w_file_init_model.boosters[i].predict(X)
1300
1301
1302
        )


1303
1304
1305
1306
def test_cvbooster():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1307
1308
1309
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
1310
    }
1311
    nfold = 3
1312
1313
    lgb_train = lgb.Dataset(X_train, y_train)
    # with early stopping
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
    cv_res = lgb.cv(
        params,
        lgb_train,
        num_boost_round=25,
        nfold=nfold,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
        return_cvbooster=True,
    )
    assert "cvbooster" in cv_res
    cvb = cv_res["cvbooster"]
1324
1325
    assert isinstance(cvb, lgb.CVBooster)
    assert isinstance(cvb.boosters, list)
1326
    assert len(cvb.boosters) == nfold
1327
1328
1329
    assert all(isinstance(bst, lgb.Booster) for bst in cvb.boosters)
    assert cvb.best_iteration > 0
    # predict by each fold booster
1330
    preds = cvb.predict(X_test)
1331
    assert isinstance(preds, list)
1332
1333
1334
1335
1336
1337
    assert len(preds) == nfold
    # check that each booster predicted using the best iteration
    for fold_preds, bst in zip(preds, cvb.boosters):
        assert bst.best_iteration == cvb.best_iteration
        expected = bst.predict(X_test, num_iteration=cvb.best_iteration)
        np.testing.assert_allclose(fold_preds, expected)
1338
1339
1340
1341
1342
    # fold averaging
    avg_pred = np.mean(preds, axis=0)
    ret = log_loss(y_test, avg_pred)
    assert ret < 0.13
    # without early stopping
1343
1344
    cv_res = lgb.cv(params, lgb_train, num_boost_round=20, nfold=3, return_cvbooster=True)
    cvb = cv_res["cvbooster"]
1345
1346
1347
1348
1349
1350
1351
    assert cvb.best_iteration == -1
    preds = cvb.predict(X_test)
    avg_pred = np.mean(preds, axis=0)
    ret = log_loss(y_test, avg_pred)
    assert ret < 0.15


1352
1353
1354
1355
def test_cvbooster_save_load(tmp_path):
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, _ = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1356
1357
1358
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
1359
1360
1361
1362
    }
    nfold = 3
    lgb_train = lgb.Dataset(X_train, y_train)

1363
1364
1365
1366
1367
1368
1369
1370
1371
    cv_res = lgb.cv(
        params,
        lgb_train,
        num_boost_round=10,
        nfold=nfold,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
        return_cvbooster=True,
    )
    cvbooster = cv_res["cvbooster"]
1372
1373
1374
    preds = cvbooster.predict(X_test)
    best_iteration = cvbooster.best_iteration

1375
    model_path_txt = str(tmp_path / "lgb.model")
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387

    cvbooster.save_model(model_path_txt)
    model_string = cvbooster.model_to_string()
    del cvbooster

    cvbooster_from_txt_file = lgb.CVBooster(model_file=model_path_txt)
    cvbooster_from_string = lgb.CVBooster().model_from_string(model_string)
    for cvbooster_loaded in [cvbooster_from_txt_file, cvbooster_from_string]:
        assert best_iteration == cvbooster_loaded.best_iteration
        np.testing.assert_array_equal(preds, cvbooster_loaded.predict(X_test))


1388
@pytest.mark.parametrize("serializer", SERIALIZERS)
1389
1390
1391
1392
def test_cvbooster_picklable(serializer):
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, _ = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1393
1394
1395
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
1396
1397
1398
1399
    }
    nfold = 3
    lgb_train = lgb.Dataset(X_train, y_train)

1400
1401
1402
1403
1404
1405
1406
1407
1408
    cv_res = lgb.cv(
        params,
        lgb_train,
        num_boost_round=10,
        nfold=nfold,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
        return_cvbooster=True,
    )
    cvbooster = cv_res["cvbooster"]
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
    preds = cvbooster.predict(X_test)
    best_iteration = cvbooster.best_iteration

    cvbooster_from_disk = pickle_and_unpickle_object(obj=cvbooster, serializer=serializer)
    del cvbooster

    assert best_iteration == cvbooster_from_disk.best_iteration

    preds_from_disk = cvbooster_from_disk.predict(X_test)
    np.testing.assert_array_equal(preds, preds_from_disk)


1421
def test_feature_name():
1422
    X_train, y_train = make_synthetic_regression()
1423
1424
    params = {"verbose": -1}
    feature_names = [f"f_{i}" for i in range(X_train.shape[-1])]
1425
1426
    lgb_train = lgb.Dataset(X_train, y_train, feature_name=feature_names)
    gbm = lgb.train(params, lgb_train, num_boost_round=5)
1427
1428
    assert feature_names == gbm.feature_name()
    # test feature_names with whitespaces
1429
    feature_names_with_space = [f"f {i}" for i in range(X_train.shape[-1])]
1430
1431
    lgb_train.set_feature_name(feature_names_with_space)
    gbm = lgb.train(params, lgb_train, num_boost_round=5)
1432
1433
1434
    assert feature_names == gbm.feature_name()


1435
def test_feature_name_with_non_ascii(rng, tmp_path):
1436
1437
    X_train = rng.normal(size=(100, 4))
    y_train = rng.normal(size=(100,))
1438
    # This has non-ascii strings.
1439
1440
    feature_names = ["F_零", "F_一", "F_二", "F_三"]
    params = {"verbose": -1}
1441
    lgb_train = lgb.Dataset(X_train, y_train, feature_name=feature_names)
1442

1443
    gbm = lgb.train(params, lgb_train, num_boost_round=5)
1444
    assert feature_names == gbm.feature_name()
1445
1446
    model_path_txt = str(tmp_path / "lgb.model")
    gbm.save_model(model_path_txt)
1447

1448
    gbm2 = lgb.Booster(model_file=model_path_txt)
1449
1450
1451
    assert feature_names == gbm2.feature_name()


1452
1453
1454
1455
1456
1457
1458
1459
def test_parameters_are_loaded_from_model_file(tmp_path, capsys, rng):
    X = np.hstack(
        [
            rng.uniform(size=(100, 1)),
            rng.integers(low=0, high=5, size=(100, 2)),
        ]
    )
    y = rng.uniform(size=(100,))
1460
1461
    ds = lgb.Dataset(X, y)
    params = {
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
        "bagging_fraction": 0.8,
        "bagging_freq": 2,
        "boosting": "rf",
        "feature_contri": [0.5, 0.5, 0.5],
        "feature_fraction": 0.7,
        "boost_from_average": False,
        "interaction_constraints": [[0, 1], [0]],
        "metric": ["l2", "rmse"],
        "num_leaves": 5,
        "num_threads": 1,
1472
    }
1473
    model_file = tmp_path / "model.txt"
1474
1475
    orig_bst = lgb.train(params, ds, num_boost_round=1, categorical_feature=[1, 2])
    orig_bst.save_model(model_file)
1476
    with model_file.open("rt") as f:
1477
        model_contents = f.readlines()
1478
1479
1480
    params_start = model_contents.index("parameters:\n")
    model_contents.insert(params_start + 1, "[max_conflict_rate: 0]\n")
    with model_file.open("wt") as f:
1481
        f.writelines(model_contents)
1482
    bst = lgb.Booster(model_file=model_file)
1483
1484
1485
    expected_msg = "[LightGBM] [Warning] Ignoring unrecognized parameter 'max_conflict_rate' found in model string."
    stdout = capsys.readouterr().out
    assert expected_msg in stdout
1486
1487
    set_params = {k: bst.params[k] for k in params.keys()}
    assert set_params == params
1488
    assert bst.params["categorical_feature"] == [1, 2]
1489
1490

    # check that passing parameters to the constructor raises warning and ignores them
1491
1492
    with pytest.warns(UserWarning, match="Ignoring params argument"):
        bst2 = lgb.Booster(params={"num_leaves": 7}, model_file=model_file)
1493
1494
    assert bst.params == bst2.params

1495
1496
1497
1498
1499
    # check inference isn't affected by unknown parameter
    orig_preds = orig_bst.predict(X)
    preds = bst.predict(X)
    np.testing.assert_allclose(preds, orig_preds)

1500

1501
def test_save_load_copy_pickle(tmp_path):
1502
    def train_and_predict(init_model=None, return_model=False):
1503
        X, y = make_synthetic_regression()
1504
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
1505
        params = {"objective": "regression", "metric": "l2", "verbose": -1}
1506
        lgb_train = lgb.Dataset(X_train, y_train)
1507
1508
1509
1510
1511
1512
        gbm_template = lgb.train(params, lgb_train, num_boost_round=10, init_model=init_model)
        return gbm_template if return_model else mean_squared_error(y_test, gbm_template.predict(X_test))

    gbm = train_and_predict(return_model=True)
    ret_origin = train_and_predict(init_model=gbm)
    other_ret = []
1513
1514
1515
    model_path_txt = str(tmp_path / "lgb.model")
    gbm.save_model(model_path_txt)
    with open(model_path_txt) as f:  # check all params are logged into model file correctly
1516
        assert f.read().find("[num_iterations: 10]") != -1
1517
1518
    other_ret.append(train_and_predict(init_model=model_path_txt))
    gbm_load = lgb.Booster(model_file=model_path_txt)
1519
1520
1521
    other_ret.append(train_and_predict(init_model=gbm_load))
    other_ret.append(train_and_predict(init_model=copy.copy(gbm)))
    other_ret.append(train_and_predict(init_model=copy.deepcopy(gbm)))
1522
1523
    model_path_pkl = str(tmp_path / "lgb.pkl")
    with open(model_path_pkl, "wb") as f:
1524
        pickle.dump(gbm, f)
1525
    with open(model_path_pkl, "rb") as f:
1526
1527
1528
1529
1530
1531
1532
1533
        gbm_pickle = pickle.load(f)
    other_ret.append(train_and_predict(init_model=gbm_pickle))
    gbm_pickles = pickle.loads(pickle.dumps(gbm))
    other_ret.append(train_and_predict(init_model=gbm_pickles))
    for ret in other_ret:
        assert ret_origin == pytest.approx(ret)


1534
1535
1536
def test_all_expected_params_are_written_out_to_model_text(tmp_path):
    X, y = make_synthetic_regression()
    params = {
1537
1538
1539
1540
1541
1542
        "objective": "mape",
        "metric": ["l2", "mae"],
        "seed": 708,
        "data_sample_strategy": "bagging",
        "sub_row": 0.8234,
        "verbose": -1,
1543
1544
    }
    dtrain = lgb.Dataset(data=X, label=y)
1545
    gbm = lgb.train(params=params, train_set=dtrain, num_boost_round=3)
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590

    model_txt_from_memory = gbm.model_to_string()
    model_file = tmp_path / "out.model"
    gbm.save_model(filename=model_file)
    with open(model_file, "r") as f:
        model_txt_from_file = f.read()

    assert model_txt_from_memory == model_txt_from_file

    # entries whose values should reflect params passed to lgb.train()
    non_default_param_entries = [
        "[objective: mape]",
        # 'l1' was passed in with alias 'mae'
        "[metric: l2,l1]",
        "[data_sample_strategy: bagging]",
        "[seed: 708]",
        # NOTE: this was passed in with alias 'sub_row'
        "[bagging_fraction: 0.8234]",
        "[num_iterations: 3]",
    ]

    # entries with default values of params
    default_param_entries = [
        "[boosting: gbdt]",
        "[tree_learner: serial]",
        "[data: ]",
        "[valid: ]",
        "[learning_rate: 0.1]",
        "[num_leaves: 31]",
        "[num_threads: 0]",
        "[deterministic: 0]",
        "[histogram_pool_size: -1]",
        "[max_depth: -1]",
        "[min_data_in_leaf: 20]",
        "[min_sum_hessian_in_leaf: 0.001]",
        "[pos_bagging_fraction: 1]",
        "[neg_bagging_fraction: 1]",
        "[bagging_freq: 0]",
        "[bagging_seed: 15415]",
        "[feature_fraction: 1]",
        "[feature_fraction_bynode: 1]",
        "[feature_fraction_seed: 32671]",
        "[extra_trees: 0]",
        "[extra_seed: 6642]",
        "[early_stopping_round: 0]",
1591
        "[early_stopping_min_delta: 0]",
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
        "[first_metric_only: 0]",
        "[max_delta_step: 0]",
        "[lambda_l1: 0]",
        "[lambda_l2: 0]",
        "[linear_lambda: 0]",
        "[min_gain_to_split: 0]",
        "[drop_rate: 0.1]",
        "[max_drop: 50]",
        "[skip_drop: 0.5]",
        "[xgboost_dart_mode: 0]",
        "[uniform_drop: 0]",
        "[drop_seed: 20623]",
        "[top_rate: 0.2]",
        "[other_rate: 0.1]",
        "[min_data_per_group: 100]",
        "[max_cat_threshold: 32]",
        "[cat_l2: 10]",
        "[cat_smooth: 10]",
        "[max_cat_to_onehot: 4]",
        "[top_k: 20]",
        "[monotone_constraints: ]",
        "[monotone_constraints_method: basic]",
        "[monotone_penalty: 0]",
        "[feature_contri: ]",
        "[forcedsplits_filename: ]",
        "[refit_decay_rate: 0.9]",
        "[cegb_tradeoff: 1]",
        "[cegb_penalty_split: 0]",
        "[cegb_penalty_feature_lazy: ]",
        "[cegb_penalty_feature_coupled: ]",
        "[path_smooth: 0]",
        "[interaction_constraints: ]",
        "[verbosity: -1]",
        "[saved_feature_importance_type: 0]",
        "[use_quantized_grad: 0]",
        "[num_grad_quant_bins: 4]",
        "[quant_train_renew_leaf: 0]",
        "[stochastic_rounding: 1]",
        "[linear_tree: 0]",
        "[max_bin: 255]",
        "[max_bin_by_feature: ]",
        "[min_data_in_bin: 3]",
        "[bin_construct_sample_cnt: 200000]",
        "[data_random_seed: 2350]",
        "[is_enable_sparse: 1]",
        "[enable_bundle: 1]",
        "[use_missing: 1]",
        "[zero_as_missing: 0]",
        "[feature_pre_filter: 1]",
        "[pre_partition: 0]",
        "[two_round: 0]",
        "[header: 0]",
        "[label_column: ]",
        "[weight_column: ]",
        "[group_column: ]",
        "[ignore_column: ]",
        "[categorical_feature: ]",
        "[forcedbins_filename: ]",
        "[precise_float_parser: 0]",
        "[parser_config_file: ]",
        "[objective_seed: 4309]",
        "[num_class: 1]",
        "[is_unbalance: 0]",
        "[scale_pos_weight: 1]",
        "[sigmoid: 1]",
        "[boost_from_average: 1]",
        "[reg_sqrt: 0]",
        "[alpha: 0.9]",
        "[fair_c: 1]",
        "[poisson_max_delta_step: 0.7]",
        "[tweedie_variance_power: 1.5]",
        "[lambdarank_truncation_level: 30]",
        "[lambdarank_norm: 1]",
        "[label_gain: ]",
        "[lambdarank_position_bias_regularization: 0]",
        "[eval_at: ]",
        "[multi_error_top_k: 1]",
        "[auc_mu_weights: ]",
        "[num_machines: 1]",
        "[local_listen_port: 12400]",
        "[time_out: 120]",
        "[machine_list_filename: ]",
        "[machines: ]",
        "[gpu_platform_id: -1]",
        "[gpu_device_id: -1]",
        "[num_gpu: 1]",
    ]
    all_param_entries = non_default_param_entries + default_param_entries

    # add device-specific entries
    #
    # passed-in force_col_wise / force_row_wise parameters are ignored on CUDA and GPU builds...
    # https://github.com/microsoft/LightGBM/blob/1d7ee63686272bceffd522284127573b511df6be/src/io/config.cpp#L375-L377
1685
1686
1687
1688
    if getenv("TASK", "") == "cuda":
        device_entries = ["[force_col_wise: 0]", "[force_row_wise: 1]", "[device_type: cuda]", "[gpu_use_dp: 1]"]
    elif getenv("TASK", "") == "gpu":
        device_entries = ["[force_col_wise: 1]", "[force_row_wise: 0]", "[device_type: gpu]", "[gpu_use_dp: 0]"]
1689
    else:
1690
        device_entries = ["[force_col_wise: 0]", "[force_row_wise: 0]", "[device_type: cpu]", "[gpu_use_dp: 0]"]
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712

    all_param_entries += device_entries

    # check that model text has all expected param entries
    for param_str in all_param_entries:
        assert param_str in model_txt_from_file
        assert param_str in model_txt_from_memory

    # since Booster.model_to_string() is used when pickling, check that parameters all
    # roundtrip pickling successfully too
    gbm_pkl = pickle_and_unpickle_object(gbm, serializer="joblib")
    model_txt_from_memory = gbm_pkl.model_to_string()
    model_file = tmp_path / "out-pkl.model"
    gbm_pkl.save_model(filename=model_file)
    with open(model_file, "r") as f:
        model_txt_from_file = f.read()

    for param_str in all_param_entries:
        assert param_str in model_txt_from_file
        assert param_str in model_txt_from_memory


1713
1714
1715
# why fixed seed?
# sometimes there is no difference how cols are treated (cat or not cat)
def test_pandas_categorical(rng_fixed_seed):
1716
    pd = pytest.importorskip("pandas")
1717
1718
    X = pd.DataFrame(
        {
1719
1720
1721
1722
1723
            "A": rng_fixed_seed.permutation(["a", "b", "c", "d"] * 75),  # str
            "B": rng_fixed_seed.permutation([1, 2, 3] * 100),  # int
            "C": rng_fixed_seed.permutation([0.1, 0.2, -0.1, -0.1, 0.2] * 60),  # float
            "D": rng_fixed_seed.permutation([True, False] * 150),  # bool
            "E": pd.Categorical(rng_fixed_seed.permutation(["z", "y", "x", "w", "v"] * 60), ordered=True),
1724
1725
        }
    )  # str and ordered categorical
1726
    y = rng_fixed_seed.permutation([0, 1] * 150)
1727
1728
    X_test = pd.DataFrame(
        {
1729
1730
1731
1732
1733
            "A": rng_fixed_seed.permutation(["a", "b", "e"] * 20),  # unseen category
            "B": rng_fixed_seed.permutation([1, 3] * 30),
            "C": rng_fixed_seed.permutation([0.1, -0.1, 0.2, 0.2] * 15),
            "D": rng_fixed_seed.permutation([True, False] * 30),
            "E": pd.Categorical(rng_fixed_seed.permutation(["z", "y"] * 30), ordered=True),
1734
1735
        }
    )
1736
1737
    cat_cols_actual = ["A", "B", "C", "D"]
    cat_cols_to_store = cat_cols_actual + ["E"]
1738
1739
    X[cat_cols_actual] = X[cat_cols_actual].astype("category")
    X_test[cat_cols_actual] = X_test[cat_cols_actual].astype("category")
1740
    cat_values = [X[col].cat.categories.tolist() for col in cat_cols_to_store]
1741
    params = {"objective": "binary", "metric": "binary_logloss", "verbose": -1}
1742
1743
1744
    lgb_train = lgb.Dataset(X, y)
    gbm0 = lgb.train(params, lgb_train, num_boost_round=10)
    pred0 = gbm0.predict(X_test)
1745
    assert lgb_train.categorical_feature == "auto"
1746
1747
1748
1749
1750
    lgb_train = lgb.Dataset(X, pd.DataFrame(y))  # also test that label can be one-column pd.DataFrame
    gbm1 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=[0])
    pred1 = gbm1.predict(X_test)
    assert lgb_train.categorical_feature == [0]
    lgb_train = lgb.Dataset(X, pd.Series(y))  # also test that label can be pd.Series
1751
    gbm2 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=["A"])
1752
    pred2 = gbm2.predict(X_test)
1753
    assert lgb_train.categorical_feature == ["A"]
1754
    lgb_train = lgb.Dataset(X, y)
1755
    gbm3 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=["A", "B", "C", "D"])
1756
    pred3 = gbm3.predict(X_test)
1757
1758
1759
    assert lgb_train.categorical_feature == ["A", "B", "C", "D"]
    gbm3.save_model("categorical.model")
    gbm4 = lgb.Booster(model_file="categorical.model")
1760
1761
    pred4 = gbm4.predict(X_test)
    model_str = gbm4.model_to_string()
1762
    gbm4.model_from_string(model_str)
1763
1764
1765
1766
    pred5 = gbm4.predict(X_test)
    gbm5 = lgb.Booster(model_str=model_str)
    pred6 = gbm5.predict(X_test)
    lgb_train = lgb.Dataset(X, y)
1767
    gbm6 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=["A", "B", "C", "D", "E"])
1768
    pred7 = gbm6.predict(X_test)
1769
    assert lgb_train.categorical_feature == ["A", "B", "C", "D", "E"]
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
    lgb_train = lgb.Dataset(X, y)
    gbm7 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=[])
    pred8 = gbm7.predict(X_test)
    assert lgb_train.categorical_feature == []
    with pytest.raises(AssertionError):
        np.testing.assert_allclose(pred0, pred1)
    with pytest.raises(AssertionError):
        np.testing.assert_allclose(pred0, pred2)
    np.testing.assert_allclose(pred1, pred2)
    np.testing.assert_allclose(pred0, pred3)
    np.testing.assert_allclose(pred0, pred4)
    np.testing.assert_allclose(pred0, pred5)
    np.testing.assert_allclose(pred0, pred6)
    with pytest.raises(AssertionError):
        np.testing.assert_allclose(pred0, pred7)  # ordered cat features aren't treated as cat features by default
    with pytest.raises(AssertionError):
1786
        np.testing.assert_allclose(pred0, pred8)
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
    assert gbm0.pandas_categorical == cat_values
    assert gbm1.pandas_categorical == cat_values
    assert gbm2.pandas_categorical == cat_values
    assert gbm3.pandas_categorical == cat_values
    assert gbm4.pandas_categorical == cat_values
    assert gbm5.pandas_categorical == cat_values
    assert gbm6.pandas_categorical == cat_values
    assert gbm7.pandas_categorical == cat_values


1797
def test_pandas_sparse(rng):
1798
    pd = pytest.importorskip("pandas")
1799
1800
    X = pd.DataFrame(
        {
1801
1802
1803
            "A": pd.arrays.SparseArray(rng.permutation([0, 1, 2] * 100)),
            "B": pd.arrays.SparseArray(rng.permutation([0.0, 0.1, 0.2, -0.1, 0.2] * 60)),
            "C": pd.arrays.SparseArray(rng.permutation([True, False] * 150)),
1804
1805
        }
    )
1806
    y = pd.Series(pd.arrays.SparseArray(rng.permutation([0, 1] * 150)))
1807
1808
    X_test = pd.DataFrame(
        {
1809
1810
1811
            "A": pd.arrays.SparseArray(rng.permutation([0, 2] * 30)),
            "B": pd.arrays.SparseArray(rng.permutation([0.0, 0.1, 0.2, -0.1] * 15)),
            "C": pd.arrays.SparseArray(rng.permutation([True, False] * 30)),
1812
1813
        }
    )
1814
1815
    for dtype in pd.concat([X.dtypes, X_test.dtypes, pd.Series(y.dtypes)]):
        assert pd.api.types.is_sparse(dtype)
1816
    params = {"objective": "binary", "verbose": -1}
1817
1818
1819
    lgb_train = lgb.Dataset(X, y)
    gbm = lgb.train(params, lgb_train, num_boost_round=10)
    pred_sparse = gbm.predict(X_test, raw_score=True)
1820
    if hasattr(X_test, "sparse"):
1821
1822
1823
1824
1825
1826
        pred_dense = gbm.predict(X_test.sparse.to_dense(), raw_score=True)
    else:
        pred_dense = gbm.predict(X_test.to_dense(), raw_score=True)
    np.testing.assert_allclose(pred_sparse, pred_dense)


1827
1828
1829
def test_reference_chain(rng):
    X = rng.normal(size=(100, 2))
    y = rng.normal(size=(100,))
1830
1831
1832
1833
    tmp_dat = lgb.Dataset(X, y)
    # take subsets and train
    tmp_dat_train = tmp_dat.subset(np.arange(80))
    tmp_dat_val = tmp_dat.subset(np.arange(80, 100)).subset(np.arange(18))
1834
    params = {"objective": "regression_l2", "metric": "rmse"}
1835
    evals_result = {}
1836
1837
1838
1839
1840
    lgb.train(
        params,
        tmp_dat_train,
        num_boost_round=20,
        valid_sets=[tmp_dat_train, tmp_dat_val],
1841
        callbacks=[lgb.record_evaluation(evals_result)],
1842
    )
1843
1844
    assert len(evals_result["training"]["rmse"]) == 20
    assert len(evals_result["valid_1"]["rmse"]) == 20
1845
1846
1847
1848
1849
1850


def test_contribs():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1851
1852
1853
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
1854
1855
1856
1857
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)

1858
1859
1860
1861
    assert (
        np.linalg.norm(gbm.predict(X_test, raw_score=True) - np.sum(gbm.predict(X_test, pred_contrib=True), axis=1))
        < 1e-4
    )
1862
1863
1864
1865
1866
1867


def test_contribs_sparse():
    n_features = 20
    n_samples = 100
    # generate CSR sparse dataset
1868
1869
1870
    X, y = make_multilabel_classification(
        n_samples=n_samples, sparse=True, n_features=n_features, n_classes=1, n_labels=2
    )
1871
1872
1873
    y = y.flatten()
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1874
1875
        "objective": "binary",
        "verbose": -1,
1876
1877
1878
1879
1880
1881
1882
1883
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)
    contribs_csr = gbm.predict(X_test, pred_contrib=True)
    assert isspmatrix_csr(contribs_csr)
    # convert data to dense and get back same contribs
    contribs_dense = gbm.predict(X_test.toarray(), pred_contrib=True)
    # validate the values are the same
1884
    if platform.machine() == "aarch64":
1885
1886
1887
        np.testing.assert_allclose(contribs_csr.toarray(), contribs_dense, rtol=1, atol=1e-12)
    else:
        np.testing.assert_allclose(contribs_csr.toarray(), contribs_dense)
1888
    assert np.linalg.norm(gbm.predict(X_test, raw_score=True) - np.sum(contribs_dense, axis=1)) < 1e-4
1889
1890
1891
1892
1893
    # validate using CSC matrix
    X_test_csc = X_test.tocsc()
    contribs_csc = gbm.predict(X_test_csc, pred_contrib=True)
    assert isspmatrix_csc(contribs_csc)
    # validate the values are the same
1894
    if platform.machine() == "aarch64":
1895
1896
1897
        np.testing.assert_allclose(contribs_csc.toarray(), contribs_dense, rtol=1, atol=1e-12)
    else:
        np.testing.assert_allclose(contribs_csc.toarray(), contribs_dense)
1898
1899
1900
1901
1902
1903
1904


def test_contribs_sparse_multiclass():
    n_features = 20
    n_samples = 100
    n_labels = 4
    # generate CSR sparse dataset
1905
1906
1907
    X, y = make_multilabel_classification(
        n_samples=n_samples, sparse=True, n_features=n_features, n_classes=1, n_labels=n_labels
    )
1908
1909
1910
    y = y.flatten()
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1911
1912
1913
        "objective": "multiclass",
        "num_class": n_labels,
        "verbose": -1,
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)
    contribs_csr = gbm.predict(X_test, pred_contrib=True)
    assert isinstance(contribs_csr, list)
    for perclass_contribs_csr in contribs_csr:
        assert isspmatrix_csr(perclass_contribs_csr)
    # convert data to dense and get back same contribs
    contribs_dense = gbm.predict(X_test.toarray(), pred_contrib=True)
    # validate the values are the same
1924
    contribs_csr_array = np.swapaxes(np.array([sparse_array.toarray() for sparse_array in contribs_csr]), 0, 1)
1925
1926
1927
1928
    contribs_csr_arr_re = contribs_csr_array.reshape(
        (contribs_csr_array.shape[0], contribs_csr_array.shape[1] * contribs_csr_array.shape[2])
    )
    if platform.machine() == "aarch64":
1929
1930
1931
        np.testing.assert_allclose(contribs_csr_arr_re, contribs_dense, rtol=1, atol=1e-12)
    else:
        np.testing.assert_allclose(contribs_csr_arr_re, contribs_dense)
1932
1933
1934
1935
1936
1937
1938
1939
1940
    contribs_dense_re = contribs_dense.reshape(contribs_csr_array.shape)
    assert np.linalg.norm(gbm.predict(X_test, raw_score=True) - np.sum(contribs_dense_re, axis=2)) < 1e-4
    # validate using CSC matrix
    X_test_csc = X_test.tocsc()
    contribs_csc = gbm.predict(X_test_csc, pred_contrib=True)
    assert isinstance(contribs_csc, list)
    for perclass_contribs_csc in contribs_csc:
        assert isspmatrix_csc(perclass_contribs_csc)
    # validate the values are the same
1941
    contribs_csc_array = np.swapaxes(np.array([sparse_array.toarray() for sparse_array in contribs_csc]), 0, 1)
1942
1943
1944
1945
    contribs_csc_array = contribs_csc_array.reshape(
        (contribs_csc_array.shape[0], contribs_csc_array.shape[1] * contribs_csc_array.shape[2])
    )
    if platform.machine() == "aarch64":
1946
1947
1948
        np.testing.assert_allclose(contribs_csc_array, contribs_dense, rtol=1, atol=1e-12)
    else:
        np.testing.assert_allclose(contribs_csc_array, contribs_dense)
1949
1950


1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
# @pytest.mark.skipif(psutil.virtual_memory().available / 1024 / 1024 / 1024 < 3, reason="not enough RAM")
# def test_int32_max_sparse_contribs(rng):
#     params = {"objective": "binary"}
#     train_features = rng.uniform(size=(100, 1000))
#     train_targets = [0] * 50 + [1] * 50
#     lgb_train = lgb.Dataset(train_features, train_targets)
#     gbm = lgb.train(params, lgb_train, num_boost_round=2)
#     csr_input_shape = (3000000, 1000)
#     test_features = csr_matrix(csr_input_shape)
#     for i in range(0, csr_input_shape[0], csr_input_shape[0] // 6):
#         for j in range(0, 1000, 100):
#             test_features[i, j] = random.random()
#     y_pred_csr = gbm.predict(test_features, pred_contrib=True)
#     # Note there is an extra column added to the output for the expected value
#     csr_output_shape = (csr_input_shape[0], csr_input_shape[1] + 1)
#     assert y_pred_csr.shape == csr_output_shape
#     y_pred_csc = gbm.predict(test_features.tocsc(), pred_contrib=True)
#     # Note output CSC shape should be same as CSR output shape
#     assert y_pred_csc.shape == csr_output_shape


def test_sliced_data(rng):
1973
1974
1975
    def train_and_get_predictions(features, labels):
        dataset = lgb.Dataset(features, label=labels)
        lgb_params = {
1976
1977
1978
            "application": "binary",
            "verbose": -1,
            "min_data": 5,
1979
        }
1980
1981
1982
1983
1984
1985
1986
1987
        gbm = lgb.train(
            params=lgb_params,
            train_set=dataset,
            num_boost_round=10,
        )
        return gbm.predict(features)

    num_samples = 100
1988
    features = rng.uniform(size=(num_samples, 5))
1989
    positive_samples = int(num_samples * 0.25)
1990
1991
1992
    labels = np.append(
        np.ones(positive_samples, dtype=np.float32), np.zeros(num_samples - positive_samples, dtype=np.float32)
    )
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
    # test sliced labels
    origin_pred = train_and_get_predictions(features, labels)
    stacked_labels = np.column_stack((labels, np.ones(num_samples, dtype=np.float32)))
    sliced_labels = stacked_labels[:, 0]
    sliced_pred = train_and_get_predictions(features, sliced_labels)
    np.testing.assert_allclose(origin_pred, sliced_pred)
    # append some columns
    stacked_features = np.column_stack((np.ones(num_samples, dtype=np.float32), features))
    stacked_features = np.column_stack((np.ones(num_samples, dtype=np.float32), stacked_features))
    stacked_features = np.column_stack((stacked_features, np.ones(num_samples, dtype=np.float32)))
    stacked_features = np.column_stack((stacked_features, np.ones(num_samples, dtype=np.float32)))
    # append some rows
    stacked_features = np.concatenate((np.ones(9, dtype=np.float32).reshape((1, 9)), stacked_features), axis=0)
    stacked_features = np.concatenate((np.ones(9, dtype=np.float32).reshape((1, 9)), stacked_features), axis=0)
    stacked_features = np.concatenate((stacked_features, np.ones(9, dtype=np.float32).reshape((1, 9))), axis=0)
    stacked_features = np.concatenate((stacked_features, np.ones(9, dtype=np.float32).reshape((1, 9))), axis=0)
    # test sliced 2d matrix
    sliced_features = stacked_features[2:102, 2:7]
    assert np.all(sliced_features == features)
    sliced_pred = train_and_get_predictions(sliced_features, sliced_labels)
    np.testing.assert_allclose(origin_pred, sliced_pred)
    # test sliced CSR
    stacked_csr = csr_matrix(stacked_features)
    sliced_csr = stacked_csr[2:102, 2:7]
    assert np.all(sliced_csr == features)
    sliced_pred = train_and_get_predictions(sliced_csr, sliced_labels)
    np.testing.assert_allclose(origin_pred, sliced_pred)


2022
2023
def test_init_with_subset(rng):
    data = rng.uniform(size=(50, 2))
2024
2025
    y = [1] * 25 + [0] * 25
    lgb_train = lgb.Dataset(data, y, free_raw_data=False)
2026
    subset_index_1 = rng.choice(a=np.arange(50), size=30, replace=False)
2027
    subset_data_1 = lgb_train.subset(subset_index_1)
2028
    subset_index_2 = rng.choice(a=np.arange(50), size=20, replace=False)
2029
    subset_data_2 = lgb_train.subset(subset_index_2)
2030
2031
2032
    params = {"objective": "binary", "verbose": -1}
    init_gbm = lgb.train(params=params, train_set=subset_data_1, num_boost_round=10, keep_training_booster=True)
    lgb.train(params=params, train_set=subset_data_2, num_boost_round=10, init_model=init_gbm)
2033
2034
2035
2036
    assert lgb_train.get_data().shape[0] == 50
    assert subset_data_1.get_data().shape[0] == 30
    assert subset_data_2.get_data().shape[0] == 20
    lgb_train.save_binary("lgb_train_data.bin")
2037
    lgb_train_from_file = lgb.Dataset("lgb_train_data.bin", free_raw_data=False)
2038
2039
    subset_data_3 = lgb_train_from_file.subset(subset_index_1)
    subset_data_4 = lgb_train_from_file.subset(subset_index_2)
2040
    init_gbm_2 = lgb.train(params=params, train_set=subset_data_3, num_boost_round=10, keep_training_booster=True)
2041
    with np.testing.assert_raises_regex(lgb.basic.LightGBMError, "Unknown format of training data"):
2042
        lgb.train(params=params, train_set=subset_data_4, num_boost_round=10, init_model=init_gbm_2)
2043
2044
2045
2046
2047
    assert lgb_train_from_file.get_data() == "lgb_train_data.bin"
    assert subset_data_3.get_data() == "lgb_train_data.bin"
    assert subset_data_4.get_data() == "lgb_train_data.bin"


2048
2049
2050
def test_training_on_constructed_subset_without_params(rng):
    X = rng.uniform(size=(100, 10))
    y = rng.uniform(size=(100,))
2051
2052
2053
2054
2055
2056
2057
2058
2059
    lgb_data = lgb.Dataset(X, y)
    subset_indices = [1, 2, 3, 4]
    subset = lgb_data.subset(subset_indices).construct()
    bst = lgb.train({}, subset, num_boost_round=1)
    assert subset.get_params() == {}
    assert subset.num_data() == len(subset_indices)
    assert bst.current_iteration() == 1


2060
2061
def generate_trainset_for_monotone_constraints_tests(x3_to_category=True):
    number_of_dpoints = 3000
2062
2063
2064
2065
    rng = np.random.default_rng()
    x1_positively_correlated_with_y = rng.uniform(size=number_of_dpoints)
    x2_negatively_correlated_with_y = rng.uniform(size=number_of_dpoints)
    x3_negatively_correlated_with_y = rng.uniform(size=number_of_dpoints)
2066
    x = np.column_stack(
2067
2068
        (
            x1_positively_correlated_with_y,
2069
            x2_negatively_correlated_with_y,
2070
2071
2072
            categorize(x3_negatively_correlated_with_y) if x3_to_category else x3_negatively_correlated_with_y,
        )
    )
2073

2074
2075
    zs = rng.normal(loc=0.0, scale=0.01, size=number_of_dpoints)
    scales = 10.0 * (rng.uniform(size=6) + 0.5)
2076
2077
2078
2079
2080
2081
2082
2083
2084
    y = (
        scales[0] * x1_positively_correlated_with_y
        + np.sin(scales[1] * np.pi * x1_positively_correlated_with_y)
        - scales[2] * x2_negatively_correlated_with_y
        - np.cos(scales[3] * np.pi * x2_negatively_correlated_with_y)
        - scales[4] * x3_negatively_correlated_with_y
        - np.cos(scales[5] * np.pi * x3_negatively_correlated_with_y)
        + zs
    )
2085
2086
2087
    categorical_features = []
    if x3_to_category:
        categorical_features = [2]
2088
    return lgb.Dataset(x, label=y, categorical_feature=categorical_features, free_raw_data=False)
2089
2090


2091
@pytest.mark.skipif(getenv("TASK", "") == "cuda", reason="Monotone constraints are not yet supported by CUDA version")
2092
2093
@pytest.mark.parametrize("test_with_categorical_variable", [True, False])
def test_monotone_constraints(test_with_categorical_variable):
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
    def is_increasing(y):
        return (np.diff(y) >= 0.0).all()

    def is_decreasing(y):
        return (np.diff(y) <= 0.0).all()

    def is_non_monotone(y):
        return (np.diff(y) < 0.0).any() and (np.diff(y) > 0.0).any()

    def is_correctly_constrained(learner, x3_to_category=True):
        iterations = 10
        n = 1000
        variable_x = np.linspace(0, 1, n).reshape((n, 1))
        fixed_xs_values = np.linspace(0, 1, n)
        for i in range(iterations):
            fixed_x = fixed_xs_values[i] * np.ones((n, 1))
            monotonically_increasing_x = np.column_stack((variable_x, fixed_x, fixed_x))
            monotonically_increasing_y = learner.predict(monotonically_increasing_x)
            monotonically_decreasing_x = np.column_stack((fixed_x, variable_x, fixed_x))
            monotonically_decreasing_y = learner.predict(monotonically_decreasing_x)
2114
2115
2116
2117
2118
2119
2120
            non_monotone_x = np.column_stack(
                (
                    fixed_x,
                    fixed_x,
                    categorize(variable_x) if x3_to_category else variable_x,
                )
            )
2121
            non_monotone_y = learner.predict(non_monotone_x)
2122
2123
2124
2125
2126
            if not (
                is_increasing(monotonically_increasing_y)
                and is_decreasing(monotonically_decreasing_y)
                and is_non_monotone(non_monotone_y)
            ):
2127
                return False
2128
        return True
2129

2130
2131
2132
2133
2134
2135
2136
2137
    def are_interactions_enforced(gbm, feature_sets):
        def parse_tree_features(gbm):
            # trees start at position 1.
            tree_str = gbm.model_to_string().split("Tree")[1:]
            feature_sets = []
            for tree in tree_str:
                # split_features are in 4th line.
                features = tree.splitlines()[3].split("=")[1].split(" ")
2138
                features = {f"Column_{f}" for f in features}
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
                feature_sets.append(features)
            return np.array(feature_sets)

        def has_interaction(treef):
            n = 0
            for fs in feature_sets:
                if len(treef.intersection(fs)) > 0:
                    n += 1
            return n > 1

        tree_features = parse_tree_features(gbm)
2150
        has_interaction_flag = np.array([has_interaction(treef) for treef in tree_features])
2151
2152
2153

        return not has_interaction_flag.any()

2154
    trainset = generate_trainset_for_monotone_constraints_tests(test_with_categorical_variable)
2155
    for test_with_interaction_constraints in [True, False]:
2156
2157
2158
2159
        error_msg = (
            "Model not correctly constrained "
            f"(test_with_interaction_constraints={test_with_interaction_constraints})"
        )
2160
        for monotone_constraints_method in ["basic", "intermediate", "advanced"]:
2161
            params = {
2162
2163
2164
                "min_data": 20,
                "num_leaves": 20,
                "monotone_constraints": [1, -1, 0],
2165
                "monotone_constraints_method": monotone_constraints_method,
2166
                "use_missing": False,
2167
            }
2168
2169
            if test_with_interaction_constraints:
                params["interaction_constraints"] = [[0], [1], [2]]
2170
            constrained_model = lgb.train(params, trainset)
2171
            assert is_correctly_constrained(constrained_model, test_with_categorical_variable), error_msg
2172
2173
2174
            if test_with_interaction_constraints:
                feature_sets = [["Column_0"], ["Column_1"], "Column_2"]
                assert are_interactions_enforced(constrained_model, feature_sets)
2175
2176


2177
@pytest.mark.skipif(getenv("TASK", "") == "cuda", reason="Monotone constraints are not yet supported by CUDA version")
2178
2179
2180
2181
2182
2183
2184
2185
def test_monotone_penalty():
    def are_first_splits_non_monotone(tree, n, monotone_constraints):
        if n <= 0:
            return True
        if "leaf_value" in tree:
            return True
        if monotone_constraints[tree["split_feature"]] != 0:
            return False
2186
2187
2188
        return are_first_splits_non_monotone(
            tree["left_child"], n - 1, monotone_constraints
        ) and are_first_splits_non_monotone(tree["right_child"], n - 1, monotone_constraints)
2189
2190
2191
2192
2193
2194

    def are_there_monotone_splits(tree, monotone_constraints):
        if "leaf_value" in tree:
            return False
        if monotone_constraints[tree["split_feature"]] != 0:
            return True
2195
2196
2197
        return are_there_monotone_splits(tree["left_child"], monotone_constraints) or are_there_monotone_splits(
            tree["right_child"], monotone_constraints
        )
2198
2199
2200
2201
2202
2203

    max_depth = 5
    monotone_constraints = [1, -1, 0]
    penalization_parameter = 2.0
    trainset = generate_trainset_for_monotone_constraints_tests(x3_to_category=False)
    for monotone_constraints_method in ["basic", "intermediate", "advanced"]:
2204
        params = {
2205
2206
2207
            "max_depth": max_depth,
            "monotone_constraints": monotone_constraints,
            "monotone_penalty": penalization_parameter,
2208
            "monotone_constraints_method": monotone_constraints_method,
2209
        }
2210
2211
2212
        constrained_model = lgb.train(params, trainset, 10)
        dumped_model = constrained_model.dump_model()["tree_info"]
        for tree in dumped_model:
2213
2214
2215
            assert are_first_splits_non_monotone(
                tree["tree_structure"], int(penalization_parameter), monotone_constraints
            )
2216
2217
2218
2219
            assert are_there_monotone_splits(tree["tree_structure"], monotone_constraints)


# test if a penalty as high as the depth indeed prohibits all monotone splits
2220
@pytest.mark.skipif(getenv("TASK", "") == "cuda", reason="Monotone constraints are not yet supported by CUDA version")
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
def test_monotone_penalty_max():
    max_depth = 5
    monotone_constraints = [1, -1, 0]
    penalization_parameter = max_depth
    trainset_constrained_model = generate_trainset_for_monotone_constraints_tests(x3_to_category=False)
    x = trainset_constrained_model.data
    y = trainset_constrained_model.label
    x3_negatively_correlated_with_y = x[:, 2]
    trainset_unconstrained_model = lgb.Dataset(x3_negatively_correlated_with_y.reshape(-1, 1), label=y)
    params_constrained_model = {
2231
2232
        "monotone_constraints": monotone_constraints,
        "monotone_penalty": penalization_parameter,
2233
2234
2235
2236
2237
2238
2239
2240
2241
        "max_depth": max_depth,
        "gpu_use_dp": True,
    }
    params_unconstrained_model = {
        "max_depth": max_depth,
        "gpu_use_dp": True,
    }

    unconstrained_model = lgb.train(params_unconstrained_model, trainset_unconstrained_model, 10)
2242
    unconstrained_model_predictions = unconstrained_model.predict(x3_negatively_correlated_with_y.reshape(-1, 1))
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259

    for monotone_constraints_method in ["basic", "intermediate", "advanced"]:
        params_constrained_model["monotone_constraints_method"] = monotone_constraints_method
        # The penalization is so high that the first 2 features should not be used here
        constrained_model = lgb.train(params_constrained_model, trainset_constrained_model, 10)

        # Check that a very high penalization is the same as not using the features at all
        np.testing.assert_array_equal(constrained_model.predict(x), unconstrained_model_predictions)


def test_max_bin_by_feature():
    col1 = np.arange(0, 100)[:, np.newaxis]
    col2 = np.zeros((100, 1))
    col2[20:] = 1
    X = np.concatenate([col1, col2], axis=1)
    y = np.arange(0, 100)
    params = {
2260
2261
2262
2263
2264
2265
2266
        "objective": "regression_l2",
        "verbose": -1,
        "num_leaves": 100,
        "min_data_in_leaf": 1,
        "min_sum_hessian_in_leaf": 0,
        "min_data_in_bin": 1,
        "max_bin_by_feature": [100, 2],
2267
2268
2269
2270
    }
    lgb_data = lgb.Dataset(X, label=y)
    est = lgb.train(params, lgb_data, num_boost_round=1)
    assert len(np.unique(est.predict(X))) == 100
2271
    params["max_bin_by_feature"] = [2, 100]
2272
2273
2274
2275
2276
    lgb_data = lgb.Dataset(X, label=y)
    est = lgb.train(params, lgb_data, num_boost_round=1)
    assert len(np.unique(est.predict(X))) == 3


2277
2278
def test_small_max_bin(rng_fixed_seed):
    y = rng_fixed_seed.choice([0, 1], 100)
2279
    x = np.ones((100, 1))
2280
2281
    x[:30, 0] = -1
    x[60:, 0] = 2
2282
    params = {"objective": "binary", "seed": 0, "min_data_in_leaf": 1, "verbose": -1, "max_bin": 2}
2283
2284
2285
    lgb_x = lgb.Dataset(x, label=y)
    lgb.train(params, lgb_x, num_boost_round=5)
    x[0, 0] = np.nan
2286
    params["max_bin"] = 3
2287
2288
2289
2290
2291
2292
2293
    lgb_x = lgb.Dataset(x, label=y)
    lgb.train(params, lgb_x, num_boost_round=5)


def test_refit():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
2294
    params = {"objective": "binary", "metric": "binary_logloss", "verbose": -1, "min_data": 10}
2295
2296
2297
2298
2299
2300
2301
2302
    lgb_train = lgb.Dataset(X_train, y_train)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)
    err_pred = log_loss(y_test, gbm.predict(X_test))
    new_gbm = gbm.refit(X_test, y_test)
    new_err_pred = log_loss(y_test, new_gbm.predict(X_test))
    assert err_pred > new_err_pred


2303
def test_refit_dataset_params(rng):
2304
2305
2306
    # check refit accepts dataset_params
    X, y = load_breast_cancer(return_X_y=True)
    lgb_train = lgb.Dataset(X, y, init_score=np.zeros(y.size))
2307
    train_params = {"objective": "binary", "verbose": -1, "seed": 123}
2308
2309
    gbm = lgb.train(train_params, lgb_train, num_boost_round=10)
    non_weight_err_pred = log_loss(y, gbm.predict(X))
2310
    refit_weight = rng.uniform(size=(y.shape[0],))
2311
    dataset_params = {
2312
2313
2314
        "max_bin": 260,
        "min_data_in_bin": 5,
        "data_random_seed": 123,
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
    }
    new_gbm = gbm.refit(
        data=X,
        label=y,
        weight=refit_weight,
        dataset_params=dataset_params,
        decay_rate=0.0,
    )
    weight_err_pred = log_loss(y, new_gbm.predict(X))
    train_set_params = new_gbm.train_set.get_params()
    stored_weights = new_gbm.train_set.get_weight()
    assert weight_err_pred != non_weight_err_pred
    assert train_set_params["max_bin"] == 260
    assert train_set_params["min_data_in_bin"] == 5
    assert train_set_params["data_random_seed"] == 123
    np.testing.assert_allclose(stored_weights, refit_weight)


2333
@pytest.mark.parametrize("boosting_type", ["rf", "dart"])
2334
2335
2336
def test_mape_for_specific_boosting_types(boosting_type):
    X, y = make_synthetic_regression()
    y = abs(y)
2337
    params = {
2338
2339
2340
2341
2342
2343
2344
        "boosting_type": boosting_type,
        "objective": "mape",
        "verbose": -1,
        "bagging_freq": 1,
        "bagging_fraction": 0.8,
        "feature_fraction": 0.8,
        "boost_from_average": True,
2345
2346
2347
2348
2349
    }
    lgb_train = lgb.Dataset(X, y)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)
    pred = gbm.predict(X)
    pred_mean = pred.mean()
2350
2351
2352
    # the following checks that dart and rf with mape can predict outside the 0-1 range
    # https://github.com/microsoft/LightGBM/issues/1579
    assert pred_mean > 8
2353
2354
2355
2356
2357
2358


def check_constant_features(y_true, expected_pred, more_params):
    X_train = np.ones((len(y_true), 1))
    y_train = np.array(y_true)
    params = {
2359
2360
2361
2362
2363
2364
2365
2366
        "objective": "regression",
        "num_class": 1,
        "verbose": -1,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "boost_from_average": True,
2367
2368
2369
2370
2371
2372
2373
2374
2375
    }
    params.update(more_params)
    lgb_train = lgb.Dataset(X_train, y_train, params=params)
    gbm = lgb.train(params, lgb_train, num_boost_round=2)
    pred = gbm.predict(X_train)
    assert np.allclose(pred, expected_pred)


def test_constant_features_regression():
2376
    params = {"objective": "regression"}
2377
2378
2379
2380
2381
2382
    check_constant_features([0.0, 10.0, 0.0, 10.0], 5.0, params)
    check_constant_features([0.0, 1.0, 2.0, 3.0], 1.5, params)
    check_constant_features([-1.0, 1.0, -2.0, 2.0], 0.0, params)


def test_constant_features_binary():
2383
    params = {"objective": "binary"}
2384
2385
2386
2387
2388
    check_constant_features([0.0, 10.0, 0.0, 10.0], 0.5, params)
    check_constant_features([0.0, 1.0, 2.0, 3.0], 0.75, params)


def test_constant_features_multiclass():
2389
    params = {"objective": "multiclass", "num_class": 3}
2390
2391
2392
2393
2394
    check_constant_features([0.0, 1.0, 2.0, 0.0], [0.5, 0.25, 0.25], params)
    check_constant_features([0.0, 1.0, 2.0, 1.0], [0.25, 0.5, 0.25], params)


def test_constant_features_multiclassova():
2395
    params = {"objective": "multiclassova", "num_class": 3}
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
    check_constant_features([0.0, 1.0, 2.0, 0.0], [0.5, 0.25, 0.25], params)
    check_constant_features([0.0, 1.0, 2.0, 1.0], [0.25, 0.5, 0.25], params)


def test_fpreproc():
    def preprocess_data(dtrain, dtest, params):
        train_data = dtrain.construct().get_data()
        test_data = dtest.construct().get_data()
        train_data[:, 0] += 1
        test_data[:, 0] += 1
        dtrain.label[-5:] = 3
        dtest.label[-5:] = 3
        dtrain = lgb.Dataset(train_data, dtrain.label)
        dtest = lgb.Dataset(test_data, dtest.label, reference=dtrain)
2410
        params["num_class"] = 4
2411
2412
2413
2414
        return dtrain, dtest, params

    X, y = load_iris(return_X_y=True)
    dataset = lgb.Dataset(X, y, free_raw_data=False)
2415
    params = {"objective": "multiclass", "num_class": 3, "verbose": -1}
2416
    results = lgb.cv(params, dataset, num_boost_round=10, fpreproc=preprocess_data)
2417
2418
    assert "valid multi_logloss-mean" in results
    assert len(results["valid multi_logloss-mean"]) == 10
2419
2420
2421
2422
2423


def test_metrics():
    X, y = load_digits(n_class=2, return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
2424
2425
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_valid = lgb.Dataset(X_test, y_test, reference=lgb_train)
2426
2427

    evals_result = {}
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
    params_dummy_obj_verbose = {"verbose": -1, "objective": dummy_obj}
    params_obj_verbose = {"objective": "binary", "verbose": -1}
    params_obj_metric_log_verbose = {"objective": "binary", "metric": "binary_logloss", "verbose": -1}
    params_obj_metric_err_verbose = {"objective": "binary", "metric": "binary_error", "verbose": -1}
    params_obj_metric_inv_verbose = {"objective": "binary", "metric": "invalid_metric", "verbose": -1}
    params_obj_metric_quant_verbose = {"objective": "regression", "metric": "quantile", "verbose": 2}
    params_obj_metric_multi_verbose = {
        "objective": "binary",
        "metric": ["binary_logloss", "binary_error"],
        "verbose": -1,
    }
    params_obj_metric_none_verbose = {"objective": "binary", "metric": "None", "verbose": -1}
    params_dummy_obj_metric_log_verbose = {"objective": dummy_obj, "metric": "binary_logloss", "verbose": -1}
    params_dummy_obj_metric_err_verbose = {"objective": dummy_obj, "metric": "binary_error", "verbose": -1}
    params_dummy_obj_metric_inv_verbose = {"objective": dummy_obj, "metric_types": "invalid_metric", "verbose": -1}
    params_dummy_obj_metric_multi_verbose = {
        "objective": dummy_obj,
        "metric": ["binary_logloss", "binary_error"],
        "verbose": -1,
    }
    params_dummy_obj_metric_none_verbose = {"objective": dummy_obj, "metric": "None", "verbose": -1}
2449
2450

    def get_cv_result(params=params_obj_verbose, **kwargs):
2451
        return lgb.cv(params, lgb_train, num_boost_round=2, **kwargs)
2452
2453

    def train_booster(params=params_obj_verbose, **kwargs):
2454
2455
2456
2457
2458
2459
        lgb.train(
            params,
            lgb_train,
            num_boost_round=2,
            valid_sets=[lgb_valid],
            callbacks=[lgb.record_evaluation(evals_result)],
2460
            **kwargs,
2461
        )
2462

2463
    # no custom objective, no feval
2464
2465
2466
    # default metric
    res = get_cv_result()
    assert len(res) == 2
2467
    assert "valid binary_logloss-mean" in res
2468
2469
2470
2471

    # non-default metric in params
    res = get_cv_result(params=params_obj_metric_err_verbose)
    assert len(res) == 2
2472
    assert "valid binary_error-mean" in res
2473
2474

    # default metric in args
2475
    res = get_cv_result(metrics="binary_logloss")
2476
    assert len(res) == 2
2477
    assert "valid binary_logloss-mean" in res
2478
2479

    # non-default metric in args
2480
    res = get_cv_result(metrics="binary_error")
2481
    assert len(res) == 2
2482
    assert "valid binary_error-mean" in res
2483
2484

    # metric in args overwrites one in params
2485
    res = get_cv_result(params=params_obj_metric_inv_verbose, metrics="binary_error")
2486
    assert len(res) == 2
2487
    assert "valid binary_error-mean" in res
2488

2489
2490
2491
    # metric in args overwrites one in params
    res = get_cv_result(params=params_obj_metric_quant_verbose)
    assert len(res) == 2
2492
    assert "valid quantile-mean" in res
2493

2494
2495
2496
    # multiple metrics in params
    res = get_cv_result(params=params_obj_metric_multi_verbose)
    assert len(res) == 4
2497
2498
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
2499
2500

    # multiple metrics in args
2501
    res = get_cv_result(metrics=["binary_logloss", "binary_error"])
2502
    assert len(res) == 4
2503
2504
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
2505
2506

    # remove default metric by 'None' in list
2507
    res = get_cv_result(metrics=["None"])
2508
2509
2510
    assert len(res) == 0

    # remove default metric by 'None' aliases
2511
    for na_alias in ("None", "na", "null", "custom"):
2512
2513
2514
        res = get_cv_result(metrics=na_alias)
        assert len(res) == 0

2515
    # custom objective, no feval
2516
    # no default metric
2517
    res = get_cv_result(params=params_dummy_obj_verbose)
2518
2519
2520
    assert len(res) == 0

    # metric in params
2521
    res = get_cv_result(params=params_dummy_obj_metric_err_verbose)
2522
    assert len(res) == 2
2523
    assert "valid binary_error-mean" in res
2524
2525

    # metric in args
2526
    res = get_cv_result(params=params_dummy_obj_verbose, metrics="binary_error")
2527
    assert len(res) == 2
2528
    assert "valid binary_error-mean" in res
2529
2530

    # metric in args overwrites its' alias in params
2531
    res = get_cv_result(params=params_dummy_obj_metric_inv_verbose, metrics="binary_error")
2532
    assert len(res) == 2
2533
    assert "valid binary_error-mean" in res
2534
2535

    # multiple metrics in params
2536
    res = get_cv_result(params=params_dummy_obj_metric_multi_verbose)
2537
    assert len(res) == 4
2538
2539
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
2540
2541

    # multiple metrics in args
2542
    res = get_cv_result(params=params_dummy_obj_verbose, metrics=["binary_logloss", "binary_error"])
2543
    assert len(res) == 4
2544
2545
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
2546

2547
    # no custom objective, feval
2548
2549
2550
    # default metric with custom one
    res = get_cv_result(feval=constant_metric)
    assert len(res) == 4
2551
2552
    assert "valid binary_logloss-mean" in res
    assert "valid error-mean" in res
2553
2554
2555
2556

    # non-default metric in params with custom one
    res = get_cv_result(params=params_obj_metric_err_verbose, feval=constant_metric)
    assert len(res) == 4
2557
2558
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2559
2560

    # default metric in args with custom one
2561
    res = get_cv_result(metrics="binary_logloss", feval=constant_metric)
2562
    assert len(res) == 4
2563
2564
    assert "valid binary_logloss-mean" in res
    assert "valid error-mean" in res
2565
2566

    # non-default metric in args with custom one
2567
    res = get_cv_result(metrics="binary_error", feval=constant_metric)
2568
    assert len(res) == 4
2569
2570
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2571
2572

    # metric in args overwrites one in params, custom one is evaluated too
2573
    res = get_cv_result(params=params_obj_metric_inv_verbose, metrics="binary_error", feval=constant_metric)
2574
    assert len(res) == 4
2575
2576
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2577
2578
2579
2580

    # multiple metrics in params with custom one
    res = get_cv_result(params=params_obj_metric_multi_verbose, feval=constant_metric)
    assert len(res) == 6
2581
2582
2583
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2584
2585

    # multiple metrics in args with custom one
2586
    res = get_cv_result(metrics=["binary_logloss", "binary_error"], feval=constant_metric)
2587
    assert len(res) == 6
2588
2589
2590
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2591
2592

    # custom metric is evaluated despite 'None' is passed
2593
    res = get_cv_result(metrics=["None"], feval=constant_metric)
2594
    assert len(res) == 2
2595
    assert "valid error-mean" in res
2596

2597
    # custom objective, feval
2598
    # no default metric, only custom one
2599
    res = get_cv_result(params=params_dummy_obj_verbose, feval=constant_metric)
2600
    assert len(res) == 2
2601
    assert "valid error-mean" in res
2602
2603

    # metric in params with custom one
2604
    res = get_cv_result(params=params_dummy_obj_metric_err_verbose, feval=constant_metric)
2605
    assert len(res) == 4
2606
2607
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2608
2609

    # metric in args with custom one
2610
    res = get_cv_result(params=params_dummy_obj_verbose, feval=constant_metric, metrics="binary_error")
2611
    assert len(res) == 4
2612
2613
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2614
2615

    # metric in args overwrites one in params, custom one is evaluated too
2616
    res = get_cv_result(params=params_dummy_obj_metric_inv_verbose, feval=constant_metric, metrics="binary_error")
2617
    assert len(res) == 4
2618
2619
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2620
2621

    # multiple metrics in params with custom one
2622
    res = get_cv_result(params=params_dummy_obj_metric_multi_verbose, feval=constant_metric)
2623
    assert len(res) == 6
2624
2625
2626
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2627
2628

    # multiple metrics in args with custom one
2629
2630
2631
    res = get_cv_result(
        params=params_dummy_obj_verbose, feval=constant_metric, metrics=["binary_logloss", "binary_error"]
    )
2632
    assert len(res) == 6
2633
2634
2635
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2636
2637

    # custom metric is evaluated despite 'None' is passed
2638
    res = get_cv_result(params=params_dummy_obj_metric_none_verbose, feval=constant_metric)
2639
    assert len(res) == 2
2640
    assert "valid error-mean" in res
2641

2642
    # no custom objective, no feval
2643
2644
    # default metric
    train_booster()
2645
2646
    assert len(evals_result["valid_0"]) == 1
    assert "binary_logloss" in evals_result["valid_0"]
2647
2648
2649

    # default metric in params
    train_booster(params=params_obj_metric_log_verbose)
2650
2651
    assert len(evals_result["valid_0"]) == 1
    assert "binary_logloss" in evals_result["valid_0"]
2652
2653
2654

    # non-default metric in params
    train_booster(params=params_obj_metric_err_verbose)
2655
2656
    assert len(evals_result["valid_0"]) == 1
    assert "binary_error" in evals_result["valid_0"]
2657
2658
2659

    # multiple metrics in params
    train_booster(params=params_obj_metric_multi_verbose)
2660
2661
2662
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "binary_error" in evals_result["valid_0"]
2663
2664

    # remove default metric by 'None' aliases
2665
2666
    for na_alias in ("None", "na", "null", "custom"):
        params = {"objective": "binary", "metric": na_alias, "verbose": -1}
2667
2668
2669
        train_booster(params=params)
        assert len(evals_result) == 0

2670
    # custom objective, no feval
2671
    # no default metric
2672
    train_booster(params=params_dummy_obj_verbose)
2673
2674
2675
    assert len(evals_result) == 0

    # metric in params
2676
    train_booster(params=params_dummy_obj_metric_log_verbose)
2677
2678
    assert len(evals_result["valid_0"]) == 1
    assert "binary_logloss" in evals_result["valid_0"]
2679
2680

    # multiple metrics in params
2681
    train_booster(params=params_dummy_obj_metric_multi_verbose)
2682
2683
2684
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "binary_error" in evals_result["valid_0"]
2685

2686
    # no custom objective, feval
2687
2688
    # default metric with custom one
    train_booster(feval=constant_metric)
2689
2690
2691
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2692
2693
2694

    # default metric in params with custom one
    train_booster(params=params_obj_metric_log_verbose, feval=constant_metric)
2695
2696
2697
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2698
2699
2700

    # non-default metric in params with custom one
    train_booster(params=params_obj_metric_err_verbose, feval=constant_metric)
2701
2702
2703
    assert len(evals_result["valid_0"]) == 2
    assert "binary_error" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2704
2705
2706

    # multiple metrics in params with custom one
    train_booster(params=params_obj_metric_multi_verbose, feval=constant_metric)
2707
2708
2709
2710
    assert len(evals_result["valid_0"]) == 3
    assert "binary_logloss" in evals_result["valid_0"]
    assert "binary_error" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2711
2712
2713
2714

    # custom metric is evaluated despite 'None' is passed
    train_booster(params=params_obj_metric_none_verbose, feval=constant_metric)
    assert len(evals_result) == 1
2715
    assert "error" in evals_result["valid_0"]
2716

2717
    # custom objective, feval
2718
    # no default metric, only custom one
2719
    train_booster(params=params_dummy_obj_verbose, feval=constant_metric)
2720
2721
    assert len(evals_result["valid_0"]) == 1
    assert "error" in evals_result["valid_0"]
2722
2723

    # metric in params with custom one
2724
    train_booster(params=params_dummy_obj_metric_log_verbose, feval=constant_metric)
2725
2726
2727
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2728
2729

    # multiple metrics in params with custom one
2730
    train_booster(params=params_dummy_obj_metric_multi_verbose, feval=constant_metric)
2731
2732
2733
2734
    assert len(evals_result["valid_0"]) == 3
    assert "binary_logloss" in evals_result["valid_0"]
    assert "binary_error" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2735
2736

    # custom metric is evaluated despite 'None' is passed
2737
    train_booster(params=params_dummy_obj_metric_none_verbose, feval=constant_metric)
2738
    assert len(evals_result) == 1
2739
    assert "error" in evals_result["valid_0"]
2740
2741

    X, y = load_digits(n_class=3, return_X_y=True)
2742
    lgb_train = lgb.Dataset(X, y)
2743

2744
    obj_multi_aliases = ["multiclass", "softmax", "multiclassova", "multiclass_ova", "ova", "ovr"]
2745
    for obj_multi_alias in obj_multi_aliases:
2746
        # Custom objective replaces multiclass
2747
2748
2749
2750
2751
        params_obj_class_3_verbose = {"objective": obj_multi_alias, "num_class": 3, "verbose": -1}
        params_dummy_obj_class_3_verbose = {"objective": dummy_obj, "num_class": 3, "verbose": -1}
        params_dummy_obj_class_1_verbose = {"objective": dummy_obj, "num_class": 1, "verbose": -1}
        params_obj_verbose = {"objective": obj_multi_alias, "verbose": -1}
        params_dummy_obj_verbose = {"objective": dummy_obj, "verbose": -1}
2752
2753
2754
        # multiclass default metric
        res = get_cv_result(params_obj_class_3_verbose)
        assert len(res) == 2
2755
        assert "valid multi_logloss-mean" in res
2756
2757
2758
        # multiclass default metric with custom one
        res = get_cv_result(params_obj_class_3_verbose, feval=constant_metric)
        assert len(res) == 4
2759
2760
        assert "valid multi_logloss-mean" in res
        assert "valid error-mean" in res
2761
        # multiclass metric alias with custom one for custom objective
2762
        res = get_cv_result(params_dummy_obj_class_3_verbose, feval=constant_metric)
2763
        assert len(res) == 2
2764
        assert "valid error-mean" in res
2765
        # no metric for invalid class_num
2766
        res = get_cv_result(params_dummy_obj_class_1_verbose)
2767
2768
        assert len(res) == 0
        # custom metric for invalid class_num
2769
        res = get_cv_result(params_dummy_obj_class_1_verbose, feval=constant_metric)
2770
        assert len(res) == 2
2771
        assert "valid error-mean" in res
2772
2773
        # multiclass metric alias with custom one with invalid class_num
        with pytest.raises(lgb.basic.LightGBMError):
2774
            get_cv_result(params_dummy_obj_class_1_verbose, metrics=obj_multi_alias, feval=constant_metric)
2775
2776
2777
        # multiclass default metric without num_class
        with pytest.raises(lgb.basic.LightGBMError):
            get_cv_result(params_obj_verbose)
2778
        for metric_multi_alias in obj_multi_aliases + ["multi_logloss"]:
2779
2780
2781
            # multiclass metric alias
            res = get_cv_result(params_obj_class_3_verbose, metrics=metric_multi_alias)
            assert len(res) == 2
2782
            assert "valid multi_logloss-mean" in res
2783
        # multiclass metric
2784
        res = get_cv_result(params_obj_class_3_verbose, metrics="multi_error")
2785
        assert len(res) == 2
2786
        assert "valid multi_error-mean" in res
2787
2788
        # non-valid metric for multiclass objective
        with pytest.raises(lgb.basic.LightGBMError):
2789
2790
            get_cv_result(params_obj_class_3_verbose, metrics="binary_logloss")
    params_class_3_verbose = {"num_class": 3, "verbose": -1}
2791
2792
2793
2794
    # non-default num_class for default objective
    with pytest.raises(lgb.basic.LightGBMError):
        get_cv_result(params_class_3_verbose)
    # no metric with non-default num_class for custom objective
2795
    res = get_cv_result(params_dummy_obj_class_3_verbose)
2796
    assert len(res) == 0
2797
    for metric_multi_alias in obj_multi_aliases + ["multi_logloss"]:
2798
        # multiclass metric alias for custom objective
2799
        res = get_cv_result(params_dummy_obj_class_3_verbose, metrics=metric_multi_alias)
2800
        assert len(res) == 2
2801
        assert "valid multi_logloss-mean" in res
2802
    # multiclass metric for custom objective
2803
    res = get_cv_result(params_dummy_obj_class_3_verbose, metrics="multi_error")
2804
    assert len(res) == 2
2805
    assert "valid multi_error-mean" in res
2806
2807
    # binary metric with non-default num_class for custom objective
    with pytest.raises(lgb.basic.LightGBMError):
2808
        get_cv_result(params_dummy_obj_class_3_verbose, metrics="binary_error")
2809
2810
2811
2812
2813


def test_multiple_feval_train():
    X, y = load_breast_cancer(return_X_y=True)

2814
    params = {"verbose": -1, "objective": "binary", "metric": "binary_logloss"}
2815
2816
2817

    X_train, X_validation, y_train, y_validation = train_test_split(X, y, test_size=0.2)

2818
2819
    train_dataset = lgb.Dataset(data=X_train, label=y_train)
    validation_dataset = lgb.Dataset(data=X_validation, label=y_validation, reference=train_dataset)
2820
2821
2822
2823
2824
2825
2826
    evals_result = {}
    lgb.train(
        params=params,
        train_set=train_dataset,
        valid_sets=validation_dataset,
        num_boost_round=5,
        feval=[constant_metric, decreasing_metric],
2827
        callbacks=[lgb.record_evaluation(evals_result)],
2828
    )
2829

2830
2831
2832
2833
    assert len(evals_result["valid_0"]) == 3
    assert "binary_logloss" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
    assert "decreasing_metric" in evals_result["valid_0"]
2834
2835


2836
2837
def test_objective_callable_train_binary_classification():
    X, y = load_breast_cancer(return_X_y=True)
2838
    params = {"verbose": -1, "objective": logloss_obj, "learning_rate": 0.01}
2839
    train_dataset = lgb.Dataset(X, y)
2840
    booster = lgb.train(params=params, train_set=train_dataset, num_boost_round=20)
2841
2842
2843
    y_pred = logistic_sigmoid(booster.predict(X))
    logloss_error = log_loss(y, y_pred)
    rocauc_error = roc_auc_score(y, y_pred)
2844
    assert booster.params["objective"] == "none"
2845
2846
    assert logloss_error == pytest.approx(0.547907)
    assert rocauc_error == pytest.approx(0.995944)
2847
2848
2849
2850


def test_objective_callable_train_regression():
    X, y = make_synthetic_regression()
2851
    params = {"verbose": -1, "objective": mse_obj}
2852
    lgb_train = lgb.Dataset(X, y)
2853
    booster = lgb.train(params, lgb_train, num_boost_round=20)
2854
2855
    y_pred = booster.predict(X)
    mse_error = mean_squared_error(y, y_pred)
2856
    assert booster.params["objective"] == "none"
2857
    assert mse_error == pytest.approx(286.724194)
2858
2859
2860
2861


def test_objective_callable_cv_binary_classification():
    X, y = load_breast_cancer(return_X_y=True)
2862
    params = {"verbose": -1, "objective": logloss_obj, "learning_rate": 0.01}
2863
    train_dataset = lgb.Dataset(X, y)
2864
2865
2866
2867
    cv_res = lgb.cv(params, train_dataset, num_boost_round=20, nfold=3, return_cvbooster=True)
    cv_booster = cv_res["cvbooster"].boosters
    cv_logloss_errors = [log_loss(y, logistic_sigmoid(cb.predict(X))) < 0.56 for cb in cv_booster]
    cv_objs = [cb.params["objective"] == "none" for cb in cv_booster]
2868
2869
2870
2871
2872
2873
2874
    assert all(cv_objs)
    assert all(cv_logloss_errors)


def test_objective_callable_cv_regression():
    X, y = make_synthetic_regression()
    lgb_train = lgb.Dataset(X, y)
2875
2876
2877
2878
2879
    params = {"verbose": -1, "objective": mse_obj}
    cv_res = lgb.cv(params, lgb_train, num_boost_round=20, nfold=3, stratified=False, return_cvbooster=True)
    cv_booster = cv_res["cvbooster"].boosters
    cv_mse_errors = [mean_squared_error(y, cb.predict(X)) < 463 for cb in cv_booster]
    cv_objs = [cb.params["objective"] == "none" for cb in cv_booster]
2880
2881
2882
2883
    assert all(cv_objs)
    assert all(cv_mse_errors)


2884
2885
2886
def test_multiple_feval_cv():
    X, y = load_breast_cancer(return_X_y=True)

2887
    params = {"verbose": -1, "objective": "binary", "metric": "binary_logloss"}
2888

2889
    train_dataset = lgb.Dataset(data=X, label=y)
2890
2891

    cv_results = lgb.cv(
2892
2893
        params=params, train_set=train_dataset, num_boost_round=5, feval=[constant_metric, decreasing_metric]
    )
2894
2895
2896

    # Expect three metrics but mean and stdv for each metric
    assert len(cv_results) == 6
2897
2898
2899
2900
2901
2902
    assert "valid binary_logloss-mean" in cv_results
    assert "valid error-mean" in cv_results
    assert "valid decreasing_metric-mean" in cv_results
    assert "valid binary_logloss-stdv" in cv_results
    assert "valid error-stdv" in cv_results
    assert "valid decreasing_metric-stdv" in cv_results
2903
2904


2905
2906
2907
2908
2909
2910
def test_default_objective_and_metric():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
    train_dataset = lgb.Dataset(data=X_train, label=y_train)
    validation_dataset = lgb.Dataset(data=X_test, label=y_test, reference=train_dataset)
    evals_result = {}
2911
    params = {"verbose": -1}
2912
2913
2914
2915
2916
    lgb.train(
        params=params,
        train_set=train_dataset,
        valid_sets=validation_dataset,
        num_boost_round=5,
2917
        callbacks=[lgb.record_evaluation(evals_result)],
2918
2919
    )

2920
2921
2922
2923
    assert "valid_0" in evals_result
    assert len(evals_result["valid_0"]) == 1
    assert "l2" in evals_result["valid_0"]
    assert len(evals_result["valid_0"]["l2"]) == 5
2924
2925


2926
@pytest.mark.parametrize("use_weight", [True, False])
2927
def test_multiclass_custom_objective(use_weight):
2928
2929
    def custom_obj(y_pred, ds):
        y_true = ds.get_label()
2930
2931
2932
        weight = ds.get_weight()
        grad, hess = sklearn_multiclass_custom_objective(y_true, y_pred, weight)
        return grad, hess
2933
2934
2935

    centers = [[-4, -4], [4, 4], [-4, 4]]
    X, y = make_blobs(n_samples=1_000, centers=centers, random_state=42)
2936
    weight = np.full_like(y, 2)
2937
    ds = lgb.Dataset(X, y)
2938
2939
    if use_weight:
        ds.set_weight(weight)
2940
    params = {"objective": "multiclass", "num_class": 3, "num_leaves": 7}
2941
2942
2943
    builtin_obj_bst = lgb.train(params, ds, num_boost_round=10)
    builtin_obj_preds = builtin_obj_bst.predict(X)

2944
    params["objective"] = custom_obj
2945
    custom_obj_bst = lgb.train(params, ds, num_boost_round=10)
2946
2947
2948
2949
2950
    custom_obj_preds = softmax(custom_obj_bst.predict(X))

    np.testing.assert_allclose(builtin_obj_preds, custom_obj_preds, rtol=0.01)


2951
@pytest.mark.parametrize("use_weight", [True, False])
2952
def test_multiclass_custom_eval(use_weight):
2953
2954
    def custom_eval(y_pred, ds):
        y_true = ds.get_label()
2955
2956
        weight = ds.get_weight()  # weight is None when not set
        loss = log_loss(y_true, y_pred, sample_weight=weight)
2957
        return "custom_logloss", loss, False
2958
2959
2960

    centers = [[-4, -4], [4, 4], [-4, 4]]
    X, y = make_blobs(n_samples=1_000, centers=centers, random_state=42)
2961
2962
2963
2964
    weight = np.full_like(y, 2)
    X_train, X_valid, y_train, y_valid, weight_train, weight_valid = train_test_split(
        X, y, weight, test_size=0.2, random_state=0
    )
2965
2966
    train_ds = lgb.Dataset(X_train, y_train)
    valid_ds = lgb.Dataset(X_valid, y_valid, reference=train_ds)
2967
2968
2969
    if use_weight:
        train_ds.set_weight(weight_train)
        valid_ds.set_weight(weight_valid)
2970
    params = {"objective": "multiclass", "num_class": 3, "num_leaves": 7}
2971
2972
2973
2974
2975
2976
    eval_result = {}
    bst = lgb.train(
        params,
        train_ds,
        num_boost_round=10,
        valid_sets=[train_ds, valid_ds],
2977
        valid_names=["train", "valid"],
2978
2979
2980
2981
2982
        feval=custom_eval,
        callbacks=[lgb.record_evaluation(eval_result)],
        keep_training_booster=True,
    )

2983
2984
    for key, ds in zip(["train", "valid"], [train_ds, valid_ds]):
        np.testing.assert_allclose(eval_result[key]["multi_logloss"], eval_result[key]["custom_logloss"])
2985
        _, metric, value, _ = bst.eval(ds, key, feval=custom_eval)[1]  # first element is multi_logloss
2986
        assert metric == "custom_logloss"
2987
2988
2989
        np.testing.assert_allclose(value, eval_result[key][metric][-1])


2990
@pytest.mark.skipif(psutil.virtual_memory().available / 1024 / 1024 / 1024 < 3, reason="not enough RAM")
2991
def test_model_size():
2992
    X, y = make_synthetic_regression()
2993
    data = lgb.Dataset(X, y)
2994
    bst = lgb.train({"verbose": -1}, data, num_boost_round=2)
2995
2996
    y_pred = bst.predict(X)
    model_str = bst.model_to_string()
2997
    one_tree = model_str[model_str.find("Tree=1") : model_str.find("end of trees")]
2998
    one_tree_size = len(one_tree)
2999
    one_tree = one_tree.replace("Tree=1", "Tree={}")
3000
3001
3002
    multiplier = 100
    total_trees = multiplier + 2
    try:
3003
3004
        before_tree_sizes = model_str[: model_str.find("tree_sizes")]
        trees = model_str[model_str.find("Tree=0") : model_str.find("end of trees")]
3005
        more_trees = (one_tree * multiplier).format(*range(2, total_trees))
3006
        after_trees = model_str[model_str.find("end of trees") :]
3007
3008
        num_end_spaces = 2**31 - one_tree_size * total_trees
        new_model_str = f"{before_tree_sizes}\n\n{trees}{more_trees}{after_trees}{'':{num_end_spaces}}"
3009
        assert len(new_model_str) > 2**31
3010
        bst.model_from_string(new_model_str)
3011
3012
3013
3014
        assert bst.num_trees() == total_trees
        y_pred_new = bst.predict(X, num_iteration=2)
        np.testing.assert_allclose(y_pred, y_pred_new)
    except MemoryError:
3015
        pytest.skipTest("not enough RAM")
3016
3017


3018
3019
3020
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Skip due to differences in implementation details of CUDA version"
)
3021
def test_get_split_value_histogram(rng_fixed_seed):
3022
3023
3024
3025
    X, y = make_synthetic_regression()
    X = np.repeat(X, 3, axis=0)
    y = np.repeat(y, 3, axis=0)
    X[:, 2] = np.random.default_rng(0).integers(0, 20, size=X.shape[0])
3026
    lgb_train = lgb.Dataset(X, y, categorical_feature=[2])
3027
    gbm = lgb.train({"verbose": -1}, lgb_train, num_boost_round=20)
3028
    # test XGBoost-style return value
3029
    params = {"feature": 0, "xgboost_style": True}
3030
3031
    assert gbm.get_split_value_histogram(**params).shape == (12, 2)
    assert gbm.get_split_value_histogram(bins=999, **params).shape == (12, 2)
3032
3033
3034
3035
    assert gbm.get_split_value_histogram(bins=-1, **params).shape == (1, 2)
    assert gbm.get_split_value_histogram(bins=0, **params).shape == (1, 2)
    assert gbm.get_split_value_histogram(bins=1, **params).shape == (1, 2)
    assert gbm.get_split_value_histogram(bins=2, **params).shape == (2, 2)
3036
3037
    assert gbm.get_split_value_histogram(bins=6, **params).shape == (6, 2)
    assert gbm.get_split_value_histogram(bins=7, **params).shape == (7, 2)
3038
3039
3040
    if lgb.compat.PANDAS_INSTALLED:
        np.testing.assert_allclose(
            gbm.get_split_value_histogram(0, xgboost_style=True).values,
3041
            gbm.get_split_value_histogram(gbm.feature_name()[0], xgboost_style=True).values,
3042
3043
3044
        )
        np.testing.assert_allclose(
            gbm.get_split_value_histogram(X.shape[-1] - 1, xgboost_style=True).values,
3045
            gbm.get_split_value_histogram(gbm.feature_name()[X.shape[-1] - 1], xgboost_style=True).values,
3046
3047
3048
3049
        )
    else:
        np.testing.assert_allclose(
            gbm.get_split_value_histogram(0, xgboost_style=True),
3050
            gbm.get_split_value_histogram(gbm.feature_name()[0], xgboost_style=True),
3051
3052
3053
        )
        np.testing.assert_allclose(
            gbm.get_split_value_histogram(X.shape[-1] - 1, xgboost_style=True),
3054
            gbm.get_split_value_histogram(gbm.feature_name()[X.shape[-1] - 1], xgboost_style=True),
3055
3056
3057
        )
    # test numpy-style return value
    hist, bins = gbm.get_split_value_histogram(0)
3058
3059
    assert len(hist) == 20
    assert len(bins) == 21
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
    hist, bins = gbm.get_split_value_histogram(0, bins=999)
    assert len(hist) == 999
    assert len(bins) == 1000
    with pytest.raises(ValueError):
        gbm.get_split_value_histogram(0, bins=-1)
    with pytest.raises(ValueError):
        gbm.get_split_value_histogram(0, bins=0)
    hist, bins = gbm.get_split_value_histogram(0, bins=1)
    assert len(hist) == 1
    assert len(bins) == 2
    hist, bins = gbm.get_split_value_histogram(0, bins=2)
    assert len(hist) == 2
    assert len(bins) == 3
    hist, bins = gbm.get_split_value_histogram(0, bins=6)
    assert len(hist) == 6
    assert len(bins) == 7
    hist, bins = gbm.get_split_value_histogram(0, bins=7)
    assert len(hist) == 7
    assert len(bins) == 8
    hist_idx, bins_idx = gbm.get_split_value_histogram(0)
    hist_name, bins_name = gbm.get_split_value_histogram(gbm.feature_name()[0])
    np.testing.assert_array_equal(hist_idx, hist_name)
    np.testing.assert_allclose(bins_idx, bins_name)
    hist_idx, bins_idx = gbm.get_split_value_histogram(X.shape[-1] - 1)
    hist_name, bins_name = gbm.get_split_value_histogram(gbm.feature_name()[X.shape[-1] - 1])
    np.testing.assert_array_equal(hist_idx, hist_name)
    np.testing.assert_allclose(bins_idx, bins_name)
    # test bins string type
3088
3089
    hist_vals, bin_edges = gbm.get_split_value_histogram(0, bins="auto")
    hist = gbm.get_split_value_histogram(0, bins="auto", xgboost_style=True)
3090
3091
    if lgb.compat.PANDAS_INSTALLED:
        mask = hist_vals > 0
3092
3093
        np.testing.assert_array_equal(hist_vals[mask], hist["Count"].values)
        np.testing.assert_allclose(bin_edges[1:][mask], hist["SplitValue"].values)
3094
3095
3096
3097
    else:
        mask = hist_vals > 0
        np.testing.assert_array_equal(hist_vals[mask], hist[:, 1])
        np.testing.assert_allclose(bin_edges[1:][mask], hist[:, 0])
3098
3099
3100
    # test histogram is disabled for categorical features
    with pytest.raises(lgb.basic.LightGBMError):
        gbm.get_split_value_histogram(2)
3101
3102


3103
3104
3105
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Skip due to differences in implementation details of CUDA version"
)
3106
def test_early_stopping_for_only_first_metric():
3107
    def metrics_combination_train_regression(valid_sets, metric_list, assumed_iteration, first_metric_only, feval=None):
3108
        params = {
3109
3110
3111
3112
3113
3114
            "objective": "regression",
            "learning_rate": 1.1,
            "num_leaves": 10,
            "metric": metric_list,
            "verbose": -1,
            "seed": 123,
3115
        }
3116
3117
3118
3119
3120
3121
        gbm = lgb.train(
            params,
            lgb_train,
            num_boost_round=25,
            valid_sets=valid_sets,
            feval=feval,
3122
            callbacks=[lgb.early_stopping(stopping_rounds=5, first_metric_only=first_metric_only)],
3123
        )
3124
        assert assumed_iteration == gbm.best_iteration
3125

3126
3127
3128
    def metrics_combination_cv_regression(
        metric_list, assumed_iteration, first_metric_only, eval_train_metric, feval=None
    ):
3129
        params = {
3130
3131
3132
3133
3134
3135
3136
            "objective": "regression",
            "learning_rate": 0.9,
            "num_leaves": 10,
            "metric": metric_list,
            "verbose": -1,
            "seed": 123,
            "gpu_use_dp": True,
3137
        }
3138
3139
3140
3141
3142
3143
3144
        ret = lgb.cv(
            params,
            train_set=lgb_train,
            num_boost_round=25,
            stratified=False,
            feval=feval,
            callbacks=[lgb.early_stopping(stopping_rounds=5, first_metric_only=first_metric_only)],
3145
            eval_train_metric=eval_train_metric,
3146
        )
3147
3148
        assert assumed_iteration == len(ret[list(ret.keys())[0]])

3149
    X, y = make_synthetic_regression()
3150
3151
3152
3153
3154
3155
3156
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
    X_test1, X_test2, y_test1, y_test2 = train_test_split(X_test, y_test, test_size=0.5, random_state=73)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_valid1 = lgb.Dataset(X_test1, y_test1, reference=lgb_train)
    lgb_valid2 = lgb.Dataset(X_test2, y_test2, reference=lgb_train)

    iter_valid1_l1 = 3
3157
3158
    iter_valid1_l2 = 3
    iter_valid2_l1 = 3
3159
    iter_valid2_l2 = 15
3160
    assert len({iter_valid1_l1, iter_valid1_l2, iter_valid2_l1, iter_valid2_l2}) == 2
3161
3162
3163
3164
    iter_min_l1 = min([iter_valid1_l1, iter_valid2_l1])
    iter_min_l2 = min([iter_valid1_l2, iter_valid2_l2])
    iter_min_valid1 = min([iter_valid1_l1, iter_valid1_l2])

3165
3166
    iter_cv_l1 = 15
    iter_cv_l2 = 13
3167
    assert len({iter_cv_l1, iter_cv_l2}) == 2
3168
3169
3170
3171
3172
3173
3174
    iter_cv_min = min([iter_cv_l1, iter_cv_l2])

    # test for lgb.train
    metrics_combination_train_regression(lgb_valid1, [], iter_valid1_l2, False)
    metrics_combination_train_regression(lgb_valid1, [], iter_valid1_l2, True)
    metrics_combination_train_regression(lgb_valid1, None, iter_valid1_l2, False)
    metrics_combination_train_regression(lgb_valid1, None, iter_valid1_l2, True)
3175
3176
3177
3178
3179
3180
    metrics_combination_train_regression(lgb_valid1, "l2", iter_valid1_l2, True)
    metrics_combination_train_regression(lgb_valid1, "l1", iter_valid1_l1, True)
    metrics_combination_train_regression(lgb_valid1, ["l2", "l1"], iter_valid1_l2, True)
    metrics_combination_train_regression(lgb_valid1, ["l1", "l2"], iter_valid1_l1, True)
    metrics_combination_train_regression(lgb_valid1, ["l2", "l1"], iter_min_valid1, False)
    metrics_combination_train_regression(lgb_valid1, ["l1", "l2"], iter_min_valid1, False)
3181
3182

    # test feval for lgb.train
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
    metrics_combination_train_regression(
        lgb_valid1,
        "None",
        1,
        False,
        feval=lambda preds, train_data: [decreasing_metric(preds, train_data), constant_metric(preds, train_data)],
    )
    metrics_combination_train_regression(
        lgb_valid1,
        "None",
        25,
        True,
        feval=lambda preds, train_data: [decreasing_metric(preds, train_data), constant_metric(preds, train_data)],
    )
    metrics_combination_train_regression(
        lgb_valid1,
        "None",
        1,
        True,
        feval=lambda preds, train_data: [constant_metric(preds, train_data), decreasing_metric(preds, train_data)],
    )
3204
3205

    # test with two valid data for lgb.train
3206
3207
3208
3209
    metrics_combination_train_regression([lgb_valid1, lgb_valid2], ["l2", "l1"], iter_min_l2, True)
    metrics_combination_train_regression([lgb_valid2, lgb_valid1], ["l2", "l1"], iter_min_l2, True)
    metrics_combination_train_regression([lgb_valid1, lgb_valid2], ["l1", "l2"], iter_min_l1, True)
    metrics_combination_train_regression([lgb_valid2, lgb_valid1], ["l1", "l2"], iter_min_l1, True)
3210
3211
3212

    # test for lgb.cv
    metrics_combination_cv_regression(None, iter_cv_l2, True, False)
3213
3214
3215
3216
3217
3218
    metrics_combination_cv_regression("l2", iter_cv_l2, True, False)
    metrics_combination_cv_regression("l1", iter_cv_l1, True, False)
    metrics_combination_cv_regression(["l2", "l1"], iter_cv_l2, True, False)
    metrics_combination_cv_regression(["l1", "l2"], iter_cv_l1, True, False)
    metrics_combination_cv_regression(["l2", "l1"], iter_cv_min, False, False)
    metrics_combination_cv_regression(["l1", "l2"], iter_cv_min, False, False)
3219
    metrics_combination_cv_regression(None, iter_cv_l2, True, True)
3220
3221
3222
3223
3224
3225
    metrics_combination_cv_regression("l2", iter_cv_l2, True, True)
    metrics_combination_cv_regression("l1", iter_cv_l1, True, True)
    metrics_combination_cv_regression(["l2", "l1"], iter_cv_l2, True, True)
    metrics_combination_cv_regression(["l1", "l2"], iter_cv_l1, True, True)
    metrics_combination_cv_regression(["l2", "l1"], iter_cv_min, False, True)
    metrics_combination_cv_regression(["l1", "l2"], iter_cv_min, False, True)
3226
3227

    # test feval for lgb.cv
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
    metrics_combination_cv_regression(
        "None",
        1,
        False,
        False,
        feval=lambda preds, train_data: [decreasing_metric(preds, train_data), constant_metric(preds, train_data)],
    )
    metrics_combination_cv_regression(
        "None",
        25,
        True,
        False,
        feval=lambda preds, train_data: [decreasing_metric(preds, train_data), constant_metric(preds, train_data)],
    )
    metrics_combination_cv_regression(
        "None",
        1,
        True,
        False,
        feval=lambda preds, train_data: [constant_metric(preds, train_data), decreasing_metric(preds, train_data)],
    )
3249
3250
3251
3252
3253
3254


def test_node_level_subcol():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
3255
3256
3257
3258
3259
        "objective": "binary",
        "metric": "binary_logloss",
        "feature_fraction_bynode": 0.8,
        "feature_fraction": 1.0,
        "verbose": -1,
3260
3261
3262
3263
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    evals_result = {}
3264
    gbm = lgb.train(
3265
        params, lgb_train, num_boost_round=25, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
3266
    )
3267
3268
    ret = log_loss(y_test, gbm.predict(X_test))
    assert ret < 0.14
3269
3270
    assert evals_result["valid_0"]["binary_logloss"][-1] == pytest.approx(ret)
    params["feature_fraction"] = 0.5
3271
3272
3273
3274
3275
    gbm2 = lgb.train(params, lgb_train, num_boost_round=25)
    ret2 = log_loss(y_test, gbm2.predict(X_test))
    assert ret != ret2


3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
def test_forced_split_feature_indices(tmp_path):
    X, y = make_synthetic_regression()
    forced_split = {
        "feature": 0,
        "threshold": 0.5,
        "left": {"feature": X.shape[1], "threshold": 0.5},
    }
    tmp_split_file = tmp_path / "forced_split.json"
    with open(tmp_split_file, "w") as f:
        f.write(json.dumps(forced_split))
    lgb_train = lgb.Dataset(X, y)
3287
    params = {"objective": "regression", "forcedsplits_filename": tmp_split_file}
3288
    with pytest.raises(lgb.basic.LightGBMError, match="Forced splits file includes feature index"):
3289
        lgb.train(params, lgb_train)
3290
3291


3292
def test_forced_bins():
3293
    x = np.empty((100, 2))
3294
3295
3296
    x[:, 0] = np.arange(0, 1, 0.01)
    x[:, 1] = -np.arange(0, 1, 0.01)
    y = np.arange(0, 1, 0.01)
3297
3298
3299
3300
3301
3302
3303
3304
3305
    forcedbins_filename = Path(__file__).absolute().parents[2] / "examples" / "regression" / "forced_bins.json"
    params = {
        "objective": "regression_l1",
        "max_bin": 5,
        "forcedbins_filename": forcedbins_filename,
        "num_leaves": 2,
        "min_data_in_leaf": 1,
        "verbose": -1,
    }
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
    lgb_x = lgb.Dataset(x, label=y)
    est = lgb.train(params, lgb_x, num_boost_round=20)
    new_x = np.zeros((3, x.shape[1]))
    new_x[:, 0] = [0.31, 0.37, 0.41]
    predicted = est.predict(new_x)
    assert len(np.unique(predicted)) == 3
    new_x[:, 0] = [0, 0, 0]
    new_x[:, 1] = [-0.9, -0.6, -0.3]
    predicted = est.predict(new_x)
    assert len(np.unique(predicted)) == 1
3316
    params["forcedbins_filename"] = ""
3317
3318
3319
3320
    lgb_x = lgb.Dataset(x, label=y)
    est = lgb.train(params, lgb_x, num_boost_round=20)
    predicted = est.predict(new_x)
    assert len(np.unique(predicted)) == 3
3321
3322
    params["forcedbins_filename"] = (
        Path(__file__).absolute().parents[2] / "examples" / "regression" / "forced_bins2.json"
3323
    )
3324
    params["max_bin"] = 11
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
    lgb_x = lgb.Dataset(x[:, :1], label=y)
    est = lgb.train(params, lgb_x, num_boost_round=50)
    predicted = est.predict(x[1:, :1])
    _, counts = np.unique(predicted, return_counts=True)
    assert min(counts) >= 9
    assert max(counts) <= 11


def test_binning_same_sign():
    # test that binning works properly for features with only positive or only negative values
3335
    x = np.empty((99, 2))
3336
3337
3338
    x[:, 0] = np.arange(0.01, 1, 0.01)
    x[:, 1] = -np.arange(0.01, 1, 0.01)
    y = np.arange(0.01, 1, 0.01)
3339
3340
3341
3342
3343
3344
3345
3346
    params = {
        "objective": "regression_l1",
        "max_bin": 5,
        "num_leaves": 2,
        "min_data_in_leaf": 1,
        "verbose": -1,
        "seed": 0,
    }
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
    lgb_x = lgb.Dataset(x, label=y)
    est = lgb.train(params, lgb_x, num_boost_round=20)
    new_x = np.zeros((3, 2))
    new_x[:, 0] = [-1, 0, 1]
    predicted = est.predict(new_x)
    assert predicted[0] == pytest.approx(predicted[1])
    assert predicted[1] != pytest.approx(predicted[2])
    new_x = np.zeros((3, 2))
    new_x[:, 1] = [-1, 0, 1]
    predicted = est.predict(new_x)
    assert predicted[0] != pytest.approx(predicted[1])
    assert predicted[1] == pytest.approx(predicted[2])


3361
def test_dataset_update_params(rng):
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
    default_params = {
        "max_bin": 100,
        "max_bin_by_feature": [20, 10],
        "bin_construct_sample_cnt": 10000,
        "min_data_in_bin": 1,
        "use_missing": False,
        "zero_as_missing": False,
        "categorical_feature": [0],
        "feature_pre_filter": True,
        "pre_partition": False,
        "enable_bundle": True,
        "data_random_seed": 0,
        "is_enable_sparse": True,
        "header": True,
        "two_round": True,
        "label_column": 0,
        "weight_column": 0,
        "group_column": 0,
        "ignore_column": 0,
        "min_data_in_leaf": 10,
        "linear_tree": False,
        "precise_float_parser": True,
        "verbose": -1,
    }
    unchangeable_params = {
        "max_bin": 150,
        "max_bin_by_feature": [30, 5],
        "bin_construct_sample_cnt": 5000,
        "min_data_in_bin": 2,
        "use_missing": True,
        "zero_as_missing": True,
        "categorical_feature": [0, 1],
        "feature_pre_filter": False,
        "pre_partition": True,
        "enable_bundle": False,
        "data_random_seed": 1,
        "is_enable_sparse": False,
        "header": False,
        "two_round": False,
        "label_column": 1,
        "weight_column": 1,
        "group_column": 1,
        "ignore_column": 1,
        "forcedbins_filename": "/some/path/forcedbins.json",
        "min_data_in_leaf": 2,
        "linear_tree": True,
        "precise_float_parser": False,
    }
3410
3411
    X = rng.uniform(size=(100, 2))
    y = rng.uniform(size=(100,))
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439

    # decreasing without freeing raw data is allowed
    lgb_data = lgb.Dataset(X, y, params=default_params, free_raw_data=False).construct()
    default_params["min_data_in_leaf"] -= 1
    lgb.train(default_params, lgb_data, num_boost_round=3)

    # decreasing before lazy init is allowed
    lgb_data = lgb.Dataset(X, y, params=default_params)
    default_params["min_data_in_leaf"] -= 1
    lgb.train(default_params, lgb_data, num_boost_round=3)

    # increasing is allowed
    default_params["min_data_in_leaf"] += 2
    lgb.train(default_params, lgb_data, num_boost_round=3)

    # decreasing with disabled filter is allowed
    default_params["feature_pre_filter"] = False
    lgb_data = lgb.Dataset(X, y, params=default_params).construct()
    default_params["min_data_in_leaf"] -= 4
    lgb.train(default_params, lgb_data, num_boost_round=3)

    # decreasing with enabled filter is disallowed;
    # also changes of other params are disallowed
    default_params["feature_pre_filter"] = True
    lgb_data = lgb.Dataset(X, y, params=default_params).construct()
    for key, value in unchangeable_params.items():
        new_params = default_params.copy()
        new_params[key] = value
3440
3441
3442
3443
        if key != "forcedbins_filename":
            param_name = key
        else:
            param_name = "forced bins"
3444
3445
3446
3447
3448
        err_msg = (
            "Reducing `min_data_in_leaf` with `feature_pre_filter=true` may cause *"
            if key == "min_data_in_leaf"
            else f"Cannot change {param_name} *"
        )
3449
3450
3451
3452
        with np.testing.assert_raises_regex(lgb.basic.LightGBMError, err_msg):
            lgb.train(new_params, lgb_data, num_boost_round=3)


3453
def test_dataset_params_with_reference(rng):
3454
    default_params = {"max_bin": 100}
3455
3456
3457
3458
    X = rng.uniform(size=(100, 2))
    y = rng.uniform(size=(100,))
    X_val = rng.uniform(size=(100, 2))
    y_val = rng.uniform(size=(100,))
3459
3460
3461
3462
3463
3464
3465
3466
3467
    lgb_train = lgb.Dataset(X, y, params=default_params, free_raw_data=False).construct()
    lgb_val = lgb.Dataset(X_val, y_val, reference=lgb_train, free_raw_data=False).construct()
    assert lgb_train.get_params() == default_params
    assert lgb_val.get_params() == default_params
    lgb.train(default_params, lgb_train, valid_sets=[lgb_val])


def test_extra_trees():
    # check extra trees increases regularization
3468
    X, y = make_synthetic_regression()
3469
    lgb_x = lgb.Dataset(X, label=y)
3470
    params = {"objective": "regression", "num_leaves": 32, "verbose": -1, "extra_trees": False, "seed": 0}
3471
3472
3473
    est = lgb.train(params, lgb_x, num_boost_round=10)
    predicted = est.predict(X)
    err = mean_squared_error(y, predicted)
3474
    params["extra_trees"] = True
3475
3476
3477
3478
3479
3480
3481
3482
    est = lgb.train(params, lgb_x, num_boost_round=10)
    predicted_new = est.predict(X)
    err_new = mean_squared_error(y, predicted_new)
    assert err < err_new


def test_path_smoothing():
    # check path smoothing increases regularization
3483
    X, y = make_synthetic_regression()
3484
    lgb_x = lgb.Dataset(X, label=y)
3485
    params = {"objective": "regression", "num_leaves": 32, "verbose": -1, "seed": 0}
3486
3487
3488
    est = lgb.train(params, lgb_x, num_boost_round=10)
    predicted = est.predict(X)
    err = mean_squared_error(y, predicted)
3489
    params["path_smooth"] = 1
3490
3491
3492
3493
3494
3495
    est = lgb.train(params, lgb_x, num_boost_round=10)
    predicted_new = est.predict(X)
    err_new = mean_squared_error(y, predicted_new)
    assert err < err_new


3496
def test_trees_to_dataframe(rng):
3497
3498
3499
    pytest.importorskip("pandas")

    def _imptcs_to_numpy(X, impcts_dict):
3500
3501
        cols = [f"Column_{i}" for i in range(X.shape[1])]
        return [impcts_dict.get(col, 0.0) for col in cols]
3502
3503
3504
3505
3506
3507

    X, y = load_breast_cancer(return_X_y=True)
    data = lgb.Dataset(X, label=y)
    num_trees = 10
    bst = lgb.train({"objective": "binary", "verbose": -1}, data, num_trees)
    tree_df = bst.trees_to_dataframe()
3508
    split_dict = tree_df[~tree_df["split_gain"].isnull()].groupby("split_feature").size().to_dict()
3509

3510
    gains_dict = tree_df.groupby("split_feature")["split_gain"].sum().to_dict()
3511
3512
3513

    tree_split = _imptcs_to_numpy(X, split_dict)
    tree_gains = _imptcs_to_numpy(X, gains_dict)
3514
3515
3516
3517
    mod_split = bst.feature_importance("split")
    mod_gains = bst.feature_importance("gain")
    num_trees_from_df = tree_df["tree_index"].nunique()
    obs_counts_from_df = tree_df.loc[tree_df["node_depth"] == 1, "count"].values
3518
3519
3520
3521
3522
3523
3524
3525

    np.testing.assert_equal(tree_split, mod_split)
    np.testing.assert_allclose(tree_gains, mod_gains)
    assert num_trees_from_df == num_trees
    np.testing.assert_equal(obs_counts_from_df, len(y))

    # test edge case with one leaf
    X = np.ones((10, 2))
3526
    y = rng.uniform(size=(10,))
3527
3528
3529
3530
3531
    data = lgb.Dataset(X, label=y)
    bst = lgb.train({"objective": "binary", "verbose": -1}, data, num_trees)
    tree_df = bst.trees_to_dataframe()

    assert len(tree_df) == 1
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
    assert tree_df.loc[0, "tree_index"] == 0
    assert tree_df.loc[0, "node_depth"] == 1
    assert tree_df.loc[0, "node_index"] == "0-L0"
    assert tree_df.loc[0, "value"] is not None
    for col in (
        "left_child",
        "right_child",
        "parent_index",
        "split_feature",
        "split_gain",
        "threshold",
        "decision_type",
        "missing_direction",
        "missing_type",
        "weight",
        "count",
    ):
3549
3550
3551
3552
        assert tree_df.loc[0, col] is None


def test_interaction_constraints():
3553
    X, y = make_synthetic_regression(n_samples=200)
3554
3555
3556
    num_features = X.shape[1]
    train_data = lgb.Dataset(X, label=y)
    # check that constraint containing all features is equivalent to no constraint
3557
    params = {"verbose": -1, "seed": 0}
3558
3559
    est = lgb.train(params, train_data, num_boost_round=10)
    pred1 = est.predict(X)
3560
    est = lgb.train(dict(params, interaction_constraints=[list(range(num_features))]), train_data, num_boost_round=10)
3561
3562
3563
    pred2 = est.predict(X)
    np.testing.assert_allclose(pred1, pred2)
    # check that constraint partitioning the features reduces train accuracy
3564
    est = lgb.train(dict(params, interaction_constraints=[[0, 2], [1, 3]]), train_data, num_boost_round=10)
3565
3566
3567
    pred3 = est.predict(X)
    assert mean_squared_error(y, pred1) < mean_squared_error(y, pred3)
    # check that constraints consisting of single features reduce accuracy further
3568
3569
3570
    est = lgb.train(
        dict(params, interaction_constraints=[[i] for i in range(num_features)]), train_data, num_boost_round=10
    )
3571
3572
3573
3574
3575
3576
    pred4 = est.predict(X)
    assert mean_squared_error(y, pred3) < mean_squared_error(y, pred4)
    # test that interaction constraints work when not all features are used
    X = np.concatenate([np.zeros((X.shape[0], 1)), X], axis=1)
    num_features = X.shape[1]
    train_data = lgb.Dataset(X, label=y)
3577
3578
3579
3580
3581
    est = lgb.train(
        dict(params, interaction_constraints=[[0] + list(range(2, num_features)), [1] + list(range(2, num_features))]),
        train_data,
        num_boost_round=10,
    )
3582
3583


3584
def test_linear_trees_num_threads(rng_fixed_seed):
3585
3586
    # check that number of threads does not affect result
    x = np.arange(0, 1000, 0.1)
3587
    y = 2 * x + rng_fixed_seed.normal(loc=0, scale=0.1, size=(len(x),))
3588
3589
    x = x[:, np.newaxis]
    lgb_train = lgb.Dataset(x, label=y)
3590
    params = {"verbose": -1, "objective": "regression", "seed": 0, "linear_tree": True, "num_threads": 2}
3591
3592
3593
3594
3595
3596
3597
3598
    est = lgb.train(params, lgb_train, num_boost_round=100)
    pred1 = est.predict(x)
    params["num_threads"] = 4
    est = lgb.train(params, lgb_train, num_boost_round=100)
    pred2 = est.predict(x)
    np.testing.assert_allclose(pred1, pred2)


3599
def test_linear_trees(tmp_path, rng_fixed_seed):
3600
3601
    # check that setting linear_tree=True fits better than ordinary trees when data has linear relationship
    x = np.arange(0, 100, 0.1)
3602
    y = 2 * x + rng_fixed_seed.normal(0, 0.1, len(x))
3603
3604
    x = x[:, np.newaxis]
    lgb_train = lgb.Dataset(x, label=y)
3605
    params = {"verbose": -1, "metric": "mse", "seed": 0, "num_leaves": 2}
3606
3607
3608
3609
    est = lgb.train(params, lgb_train, num_boost_round=10)
    pred1 = est.predict(x)
    lgb_train = lgb.Dataset(x, label=y)
    res = {}
3610
    est = lgb.train(
3611
        dict(params, linear_tree=True),
3612
3613
3614
        lgb_train,
        num_boost_round=10,
        valid_sets=[lgb_train],
3615
3616
        valid_names=["train"],
        callbacks=[lgb.record_evaluation(res)],
3617
    )
3618
    pred2 = est.predict(x)
3619
    assert res["train"]["l2"][-1] == pytest.approx(mean_squared_error(y, pred2), abs=1e-1)
3620
3621
3622
3623
3624
3625
3626
3627
    assert mean_squared_error(y, pred2) < mean_squared_error(y, pred1)
    # test again with nans in data
    x[:10] = np.nan
    lgb_train = lgb.Dataset(x, label=y)
    est = lgb.train(params, lgb_train, num_boost_round=10)
    pred1 = est.predict(x)
    lgb_train = lgb.Dataset(x, label=y)
    res = {}
3628
    est = lgb.train(
3629
        dict(params, linear_tree=True),
3630
3631
3632
        lgb_train,
        num_boost_round=10,
        valid_sets=[lgb_train],
3633
3634
        valid_names=["train"],
        callbacks=[lgb.record_evaluation(res)],
3635
    )
3636
    pred2 = est.predict(x)
3637
    assert res["train"]["l2"][-1] == pytest.approx(mean_squared_error(y, pred2), abs=1e-1)
3638
3639
3640
    assert mean_squared_error(y, pred2) < mean_squared_error(y, pred1)
    # test again with bagging
    res = {}
3641
    est = lgb.train(
3642
        dict(params, linear_tree=True, subsample=0.8, bagging_freq=1),
3643
3644
3645
        lgb_train,
        num_boost_round=10,
        valid_sets=[lgb_train],
3646
3647
        valid_names=["train"],
        callbacks=[lgb.record_evaluation(res)],
3648
    )
3649
    pred = est.predict(x)
3650
    assert res["train"]["l2"][-1] == pytest.approx(mean_squared_error(y, pred), abs=1e-1)
3651
3652
3653
3654
3655
3656
    # test with a feature that has only one non-nan value
    x = np.concatenate([np.ones([x.shape[0], 1]), x], 1)
    x[500:, 1] = np.nan
    y[500:] += 10
    lgb_train = lgb.Dataset(x, label=y)
    res = {}
3657
    est = lgb.train(
3658
        dict(params, linear_tree=True, subsample=0.8, bagging_freq=1),
3659
3660
3661
        lgb_train,
        num_boost_round=10,
        valid_sets=[lgb_train],
3662
3663
        valid_names=["train"],
        callbacks=[lgb.record_evaluation(res)],
3664
    )
3665
    pred = est.predict(x)
3666
    assert res["train"]["l2"][-1] == pytest.approx(mean_squared_error(y, pred), abs=1e-1)
3667
3668
3669
3670
    # test with a categorical feature
    x[:250, 0] = 0
    y[:250] += 10
    lgb_train = lgb.Dataset(x, label=y)
3671
3672
3673
3674
3675
3676
    est = lgb.train(
        dict(params, linear_tree=True, subsample=0.8, bagging_freq=1),
        lgb_train,
        num_boost_round=10,
        categorical_feature=[0],
    )
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
    # test refit: same results on same data
    est2 = est.refit(x, label=y)
    p1 = est.predict(x)
    p2 = est2.predict(x)
    assert np.mean(np.abs(p1 - p2)) < 2

    # test refit with save and load
    temp_model = str(tmp_path / "temp_model.txt")
    est.save_model(temp_model)
    est2 = lgb.Booster(model_file=temp_model)
    est2 = est2.refit(x, label=y)
    p1 = est.predict(x)
    p2 = est2.predict(x)
    assert np.mean(np.abs(p1 - p2)) < 2
    # test refit: different results training on different data
    est3 = est.refit(x[:100, :], label=y[:100])
    p3 = est3.predict(x)
    assert np.mean(np.abs(p2 - p1)) > np.abs(np.max(p3 - p1))
    # test when num_leaves - 1 < num_features and when num_leaves - 1 > num_features
    X_train, _, y_train, _ = train_test_split(*load_breast_cancer(return_X_y=True), test_size=0.1, random_state=2)
3697
    params = {"linear_tree": True, "verbose": -1, "metric": "mse", "seed": 0}
3698
3699
3700
3701
3702
3703
    train_data = lgb.Dataset(X_train, label=y_train, params=dict(params, num_leaves=2))
    est = lgb.train(params, train_data, num_boost_round=10, categorical_feature=[0])
    train_data = lgb.Dataset(X_train, label=y_train, params=dict(params, num_leaves=60))
    est = lgb.train(params, train_data, num_boost_round=10, categorical_feature=[0])


3704
def test_save_and_load_linear(tmp_path):
3705
3706
3707
    X_train, X_test, y_train, y_test = train_test_split(
        *load_breast_cancer(return_X_y=True), test_size=0.1, random_state=2
    )
3708
    X_train = np.concatenate([np.ones((X_train.shape[0], 1)), X_train], 1)
3709
3710
3711
    X_train[: X_train.shape[0] // 2, 0] = 0
    y_train[: X_train.shape[0] // 2] = 1
    params = {"linear_tree": True}
3712
3713
3714
3715
    train_data_1 = lgb.Dataset(X_train, label=y_train, params=params)
    est_1 = lgb.train(params, train_data_1, num_boost_round=10, categorical_feature=[0])
    pred_1 = est_1.predict(X_train)

3716
    tmp_dataset = str(tmp_path / "temp_dataset.bin")
3717
3718
3719
3720
3721
3722
    train_data_1.save_binary(tmp_dataset)
    train_data_2 = lgb.Dataset(tmp_dataset)
    est_2 = lgb.train(params, train_data_2, num_boost_round=10)
    pred_2 = est_2.predict(X_train)
    np.testing.assert_allclose(pred_1, pred_2)

3723
    model_file = str(tmp_path / "model.txt")
3724
3725
3726
3727
3728
3729
    est_2.save_model(model_file)
    est_3 = lgb.Booster(model_file=model_file)
    pred_3 = est_3.predict(X_train)
    np.testing.assert_allclose(pred_2, pred_3)


3730
3731
3732
def test_linear_single_leaf():
    X_train, y_train = load_breast_cancer(return_X_y=True)
    train_data = lgb.Dataset(X_train, label=y_train)
3733
    params = {"objective": "binary", "linear_tree": True, "min_sum_hessian": 5000}
3734
3735
3736
3737
3738
    bst = lgb.train(params, train_data, num_boost_round=5)
    y_pred = bst.predict(X_train)
    assert log_loss(y_train, y_pred) < 0.661


3739
3740
3741
3742
3743
def test_predict_with_start_iteration():
    def inner_test(X, y, params, early_stopping_rounds):
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
        train_data = lgb.Dataset(X_train, label=y_train)
        valid_data = lgb.Dataset(X_test, label=y_test)
3744
        callbacks = [lgb.early_stopping(early_stopping_rounds)] if early_stopping_rounds is not None else []
3745
        booster = lgb.train(params, train_data, num_boost_round=50, valid_sets=[valid_data], callbacks=callbacks)
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762

        # test that the predict once with all iterations equals summed results with start_iteration and num_iteration
        all_pred = booster.predict(X, raw_score=True)
        all_pred_contrib = booster.predict(X, pred_contrib=True)
        steps = [10, 12]
        for step in steps:
            pred = np.zeros_like(all_pred)
            pred_contrib = np.zeros_like(all_pred_contrib)
            for start_iter in range(0, 50, step):
                pred += booster.predict(X, start_iteration=start_iter, num_iteration=step, raw_score=True)
                pred_contrib += booster.predict(X, start_iteration=start_iter, num_iteration=step, pred_contrib=True)
            np.testing.assert_allclose(all_pred, pred)
            np.testing.assert_allclose(all_pred_contrib, pred_contrib)
        # test the case where start_iteration <= 0, and num_iteration is None
        pred1 = booster.predict(X, start_iteration=-1)
        pred2 = booster.predict(X, num_iteration=booster.best_iteration)
        np.testing.assert_allclose(pred1, pred2)
3763

3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
        # test the case where start_iteration > 0, and num_iteration <= 0
        pred4 = booster.predict(X, start_iteration=10, num_iteration=-1)
        pred5 = booster.predict(X, start_iteration=10, num_iteration=90)
        pred6 = booster.predict(X, start_iteration=10, num_iteration=0)
        np.testing.assert_allclose(pred4, pred5)
        np.testing.assert_allclose(pred4, pred6)

        # test the case where start_iteration > 0, and num_iteration <= 0, with pred_leaf=True
        pred4 = booster.predict(X, start_iteration=10, num_iteration=-1, pred_leaf=True)
        pred5 = booster.predict(X, start_iteration=10, num_iteration=40, pred_leaf=True)
        pred6 = booster.predict(X, start_iteration=10, num_iteration=0, pred_leaf=True)
        np.testing.assert_allclose(pred4, pred5)
        np.testing.assert_allclose(pred4, pred6)

        # test the case where start_iteration > 0, and num_iteration <= 0, with pred_contrib=True
        pred4 = booster.predict(X, start_iteration=10, num_iteration=-1, pred_contrib=True)
        pred5 = booster.predict(X, start_iteration=10, num_iteration=40, pred_contrib=True)
        pred6 = booster.predict(X, start_iteration=10, num_iteration=0, pred_contrib=True)
        np.testing.assert_allclose(pred4, pred5)
        np.testing.assert_allclose(pred4, pred6)

    # test for regression
3786
    X, y = make_synthetic_regression()
3787
    params = {"objective": "regression", "verbose": -1, "metric": "l2", "learning_rate": 0.5}
3788
3789
3790
3791
3792
3793
3794
    # test both with and without early stopping
    inner_test(X, y, params, early_stopping_rounds=1)
    inner_test(X, y, params, early_stopping_rounds=5)
    inner_test(X, y, params, early_stopping_rounds=None)

    # test for multi-class
    X, y = load_iris(return_X_y=True)
3795
    params = {"objective": "multiclass", "num_class": 3, "verbose": -1, "metric": "multi_error"}
3796
3797
3798
3799
3800
3801
3802
    # test both with and without early stopping
    inner_test(X, y, params, early_stopping_rounds=1)
    inner_test(X, y, params, early_stopping_rounds=5)
    inner_test(X, y, params, early_stopping_rounds=None)

    # test for binary
    X, y = load_breast_cancer(return_X_y=True)
3803
    params = {"objective": "binary", "verbose": -1, "metric": "auc"}
3804
3805
3806
3807
3808
3809
3810
3811
3812
    # test both with and without early stopping
    inner_test(X, y, params, early_stopping_rounds=1)
    inner_test(X, y, params, early_stopping_rounds=5)
    inner_test(X, y, params, early_stopping_rounds=None)


def test_average_precision_metric():
    # test against sklearn average precision metric
    X, y = load_breast_cancer(return_X_y=True)
3813
    params = {"objective": "binary", "metric": "average_precision", "verbose": -1}
3814
3815
    res = {}
    lgb_X = lgb.Dataset(X, label=y)
3816
3817
    est = lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(res)])
    ap = res["training"]["average_precision"][-1]
3818
3819
3820
3821
3822
3823
3824
    pred = est.predict(X)
    sklearn_ap = average_precision_score(y, pred)
    assert ap == pytest.approx(sklearn_ap)
    # test that average precision is 1 where model predicts perfectly
    y = y.copy()
    y[:] = 1
    lgb_X = lgb.Dataset(X, label=y)
3825
3826
    lgb.train(params, lgb_X, num_boost_round=1, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(res)])
    assert res["training"]["average_precision"][-1] == pytest.approx(1)
3827
3828
3829
3830
3831
3832


def test_reset_params_works_with_metric_num_class_and_boosting():
    X, y = load_breast_cancer(return_X_y=True)
    dataset_params = {"max_bin": 150}
    booster_params = {
3833
3834
3835
3836
3837
3838
        "objective": "multiclass",
        "max_depth": 4,
        "bagging_fraction": 0.8,
        "metric": ["multi_logloss", "multi_error"],
        "boosting": "gbdt",
        "num_class": 5,
3839
3840
    }
    dtrain = lgb.Dataset(X, y, params=dataset_params)
3841
    bst = lgb.Booster(params=booster_params, train_set=dtrain)
3842
3843
3844
3845

    expected_params = dict(dataset_params, **booster_params)
    assert bst.params == expected_params

3846
    booster_params["bagging_fraction"] += 0.1
3847
3848
3849
3850
3851
    new_bst = bst.reset_parameter(booster_params)

    expected_params = dict(dataset_params, **booster_params)
    assert bst.params == expected_params
    assert new_bst.params == expected_params
3852
3853
3854
3855
3856


def test_dump_model():
    X, y = load_breast_cancer(return_X_y=True)
    train_data = lgb.Dataset(X, label=y)
3857
    params = {"objective": "binary", "verbose": -1}
3858
3859
3860
3861
3862
3863
3864
    bst = lgb.train(params, train_data, num_boost_round=5)
    dumped_model_str = str(bst.dump_model(5, 0))
    assert "leaf_features" not in dumped_model_str
    assert "leaf_coeff" not in dumped_model_str
    assert "leaf_const" not in dumped_model_str
    assert "leaf_value" in dumped_model_str
    assert "leaf_count" in dumped_model_str
3865
    params["linear_tree"] = True
3866
3867
3868
3869
3870
3871
3872
3873
    train_data = lgb.Dataset(X, label=y)
    bst = lgb.train(params, train_data, num_boost_round=5)
    dumped_model_str = str(bst.dump_model(5, 0))
    assert "leaf_features" in dumped_model_str
    assert "leaf_coeff" in dumped_model_str
    assert "leaf_const" in dumped_model_str
    assert "leaf_value" in dumped_model_str
    assert "leaf_count" in dumped_model_str
3874
3875
3876
3877


def test_dump_model_hook():
    def hook(obj):
3878
3879
3880
        if "leaf_value" in obj:
            obj["LV"] = obj["leaf_value"]
            del obj["leaf_value"]
3881
3882
3883
3884
        return obj

    X, y = load_breast_cancer(return_X_y=True)
    train_data = lgb.Dataset(X, label=y)
3885
    params = {"objective": "binary", "verbose": -1}
3886
3887
3888
3889
    bst = lgb.train(params, train_data, num_boost_round=5)
    dumped_model_str = str(bst.dump_model(5, 0, object_hook=hook))
    assert "leaf_value" not in dumped_model_str
    assert "LV" in dumped_model_str
3890
3891


3892
@pytest.mark.skipif(getenv("TASK", "") == "cuda", reason="Forced splits are not yet supported by CUDA version")
3893
def test_force_split_with_feature_fraction(tmp_path):
3894
    X, y = make_synthetic_regression()
3895
3896
3897
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)

3898
    forced_split = {"feature": 0, "threshold": 0.5, "right": {"feature": 2, "threshold": 10.0}}
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908

    tmp_split_file = tmp_path / "forced_split.json"
    with open(tmp_split_file, "w") as f:
        f.write(json.dumps(forced_split))

    params = {
        "objective": "regression",
        "feature_fraction": 0.6,
        "force_col_wise": True,
        "feature_fraction_seed": 1,
3909
        "forcedsplits_filename": tmp_split_file,
3910
3911
3912
3913
    }

    gbm = lgb.train(params, lgb_train)
    ret = mean_absolute_error(y_test, gbm.predict(X_test))
3914
    assert ret < 15.7
3915
3916
3917
3918
3919

    tree_info = gbm.dump_model()["tree_info"]
    assert len(tree_info) > 1
    for tree in tree_info:
        tree_structure = tree["tree_structure"]
3920
        assert tree_structure["split_feature"] == 0
3921
3922


3923
3924
3925
3926
3927
3928
def test_goss_boosting_and_strategy_equivalent():
    X, y = make_synthetic_regression(n_samples=10_000, n_features=10, n_informative=5, random_state=42)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    base_params = {
3929
3930
3931
3932
3933
3934
3935
        "metric": "l2",
        "verbose": -1,
        "bagging_seed": 0,
        "learning_rate": 0.05,
        "num_threads": 1,
        "force_row_wise": True,
        "gpu_use_dp": True,
3936
    }
3937
    params1 = {**base_params, "boosting": "goss"}
3938
    evals_result1 = {}
3939
3940
3941
3942
    lgb.train(
        params1, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result1)]
    )
    params2 = {**base_params, "data_sample_strategy": "goss"}
3943
    evals_result2 = {}
3944
3945
3946
3947
    lgb.train(
        params2, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result2)]
    )
    assert evals_result1["valid_0"]["l2"] == evals_result2["valid_0"]["l2"]
3948
3949
3950
3951
3952
3953
3954
3955
3956


def test_sample_strategy_with_boosting():
    X, y = make_synthetic_regression(n_samples=10_000, n_features=10, n_informative=5, random_state=42)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)

    base_params = {
3957
3958
3959
3960
3961
        "metric": "l2",
        "verbose": -1,
        "num_threads": 1,
        "force_row_wise": True,
        "gpu_use_dp": True,
3962
3963
    }

3964
    params1 = {**base_params, "boosting": "dart", "data_sample_strategy": "goss"}
3965
    evals_result = {}
3966
3967
3968
3969
    gbm = lgb.train(
        params1, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res1 = evals_result["valid_0"]["l2"][-1]
3970
3971
3972
3973
    test_res1 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res1 == pytest.approx(3149.393862, abs=1.0)
    assert eval_res1 == pytest.approx(test_res1)

3974
    params2 = {**base_params, "boosting": "gbdt", "data_sample_strategy": "goss"}
3975
    evals_result = {}
3976
3977
3978
3979
    gbm = lgb.train(
        params2, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res2 = evals_result["valid_0"]["l2"][-1]
3980
3981
3982
3983
    test_res2 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res2 == pytest.approx(2547.715968, abs=1.0)
    assert eval_res2 == pytest.approx(test_res2)

3984
    params3 = {**base_params, "boosting": "goss", "data_sample_strategy": "goss"}
3985
    evals_result = {}
3986
3987
3988
3989
    gbm = lgb.train(
        params3, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res3 = evals_result["valid_0"]["l2"][-1]
3990
3991
3992
3993
    test_res3 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res3 == pytest.approx(2547.715968, abs=1.0)
    assert eval_res3 == pytest.approx(test_res3)

3994
    params4 = {**base_params, "boosting": "rf", "data_sample_strategy": "goss"}
3995
    evals_result = {}
3996
3997
3998
3999
    gbm = lgb.train(
        params4, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res4 = evals_result["valid_0"]["l2"][-1]
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
    test_res4 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res4 == pytest.approx(2095.538735, abs=1.0)
    assert eval_res4 == pytest.approx(test_res4)

    assert test_res1 != test_res2
    assert eval_res1 != eval_res2
    assert test_res2 == test_res3
    assert eval_res2 == eval_res3
    assert eval_res1 != eval_res4
    assert test_res1 != test_res4
    assert eval_res2 != eval_res4
    assert test_res2 != test_res4

4013
4014
4015
4016
4017
4018
4019
    params5 = {
        **base_params,
        "boosting": "dart",
        "data_sample_strategy": "bagging",
        "bagging_freq": 1,
        "bagging_fraction": 0.5,
    }
4020
    evals_result = {}
4021
4022
4023
4024
    gbm = lgb.train(
        params5, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res5 = evals_result["valid_0"]["l2"][-1]
4025
4026
4027
4028
    test_res5 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res5 == pytest.approx(3134.866931, abs=1.0)
    assert eval_res5 == pytest.approx(test_res5)

4029
4030
4031
4032
4033
4034
4035
    params6 = {
        **base_params,
        "boosting": "gbdt",
        "data_sample_strategy": "bagging",
        "bagging_freq": 1,
        "bagging_fraction": 0.5,
    }
4036
    evals_result = {}
4037
4038
4039
4040
    gbm = lgb.train(
        params6, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res6 = evals_result["valid_0"]["l2"][-1]
4041
4042
4043
4044
4045
4046
    test_res6 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res6 == pytest.approx(2539.792378, abs=1.0)
    assert eval_res6 == pytest.approx(test_res6)
    assert test_res5 != test_res6
    assert eval_res5 != eval_res6

4047
4048
4049
4050
4051
4052
4053
    params7 = {
        **base_params,
        "boosting": "rf",
        "data_sample_strategy": "bagging",
        "bagging_freq": 1,
        "bagging_fraction": 0.5,
    }
4054
    evals_result = {}
4055
4056
4057
4058
    gbm = lgb.train(
        params7, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res7 = evals_result["valid_0"]["l2"][-1]
4059
4060
4061
4062
4063
4064
4065
4066
4067
    test_res7 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res7 == pytest.approx(1518.704481, abs=1.0)
    assert eval_res7 == pytest.approx(test_res7)
    assert test_res5 != test_res7
    assert eval_res5 != eval_res7
    assert test_res6 != test_res7
    assert eval_res6 != eval_res7


4068
4069
4070
4071
4072
def test_record_evaluation_with_train():
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, y)
    eval_result = {}
    callbacks = [lgb.record_evaluation(eval_result)]
4073
    params = {"objective": "l2", "num_leaves": 3}
4074
4075
    num_boost_round = 5
    bst = lgb.train(params, ds, num_boost_round=num_boost_round, valid_sets=[ds], callbacks=callbacks)
4076
    assert list(eval_result.keys()) == ["training"]
4077
4078
4079
4080
4081
    train_mses = []
    for i in range(num_boost_round):
        pred = bst.predict(X, num_iteration=i + 1)
        mse = mean_squared_error(y, pred)
        train_mses.append(mse)
4082
    np.testing.assert_allclose(eval_result["training"]["l2"], train_mses)
4083
4084


4085
@pytest.mark.parametrize("train_metric", [False, True])
4086
4087
4088
4089
4090
def test_record_evaluation_with_cv(train_metric):
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, y)
    eval_result = {}
    callbacks = [lgb.record_evaluation(eval_result)]
4091
4092
4093
4094
4095
4096
    metrics = ["l2", "rmse"]
    params = {"objective": "l2", "num_leaves": 3, "metric": metrics}
    cv_hist = lgb.cv(
        params, ds, num_boost_round=5, stratified=False, callbacks=callbacks, eval_train_metric=train_metric
    )
    expected_datasets = {"valid"}
4097
    if train_metric:
4098
        expected_datasets.add("train")
4099
4100
4101
    assert set(eval_result.keys()) == expected_datasets
    for dataset in expected_datasets:
        for metric in metrics:
4102
4103
4104
            for agg in ("mean", "stdv"):
                key = f"{dataset} {metric}-{agg}"
                np.testing.assert_allclose(cv_hist[key], eval_result[dataset][f"{metric}-{agg}"])
4105
4106


4107
def test_pandas_with_numpy_regular_dtypes(rng_fixed_seed):
4108
4109
4110
4111
    pd = pytest.importorskip("pandas")
    uints = ["uint8", "uint16", "uint32", "uint64"]
    ints = ["int8", "int16", "int32", "int64"]
    bool_and_floats = ["bool", "float16", "float32", "float64"]
4112
4113
4114

    n_samples = 100
    # data as float64
4115
4116
    df = pd.DataFrame(
        {
4117
4118
4119
4120
            "x1": rng_fixed_seed.integers(low=0, high=2, size=n_samples),
            "x2": rng_fixed_seed.integers(low=1, high=3, size=n_samples),
            "x3": 10 * rng_fixed_seed.integers(low=1, high=3, size=n_samples),
            "x4": 100 * rng_fixed_seed.integers(low=1, high=3, size=n_samples),
4121
4122
        }
    )
4123
    df = df.astype(np.float64)
4124
    y = df["x1"] * (df["x2"] + df["x3"] + df["x4"])
4125
    ds = lgb.Dataset(df, y)
4126
    params = {"objective": "l2", "num_leaves": 31, "min_child_samples": 1}
4127
4128
4129
4130
    bst = lgb.train(params, ds, num_boost_round=5)
    preds = bst.predict(df)

    # test all features were used
4131
    assert bst.trees_to_dataframe()["split_feature"].nunique() == df.shape[1]
4132
4133
4134
4135
4136
4137
    # test the score is better than predicting the mean
    baseline = np.full_like(y, y.mean())
    assert mean_squared_error(y, preds) < mean_squared_error(y, baseline)

    # test all predictions are equal using different input dtypes
    for target_dtypes in [uints, ints, bool_and_floats]:
4138
        df2 = df.astype({f"x{i}": dtype for i, dtype in enumerate(target_dtypes, start=1)})
4139
4140
4141
4142
4143
4144
4145
        assert df2.dtypes.tolist() == target_dtypes
        ds2 = lgb.Dataset(df2, y)
        bst2 = lgb.train(params, ds2, num_boost_round=5)
        preds2 = bst2.predict(df2)
        np.testing.assert_allclose(preds, preds2)


4146
def test_pandas_nullable_dtypes(rng_fixed_seed):
4147
4148
4149
    pd = pytest.importorskip("pandas")
    df = pd.DataFrame(
        {
4150
            "x1": rng_fixed_seed.integers(low=1, high=3, size=100),
4151
            "x2": np.linspace(-1, 1, 100),
4152
4153
            "x3": pd.arrays.SparseArray(rng_fixed_seed.integers(low=0, high=11, size=100)),
            "x4": rng_fixed_seed.uniform(size=(100,)) < 0.5,
4154
4155
        }
    )
4156
    # introduce some missing values
4157
4158
4159
    df.loc[1, "x1"] = np.nan
    df.loc[2, "x2"] = np.nan
    df.loc[3, "x4"] = np.nan
4160
    # the previous line turns x3 into object dtype in recent versions of pandas
4161
4162
    df["x4"] = df["x4"].astype(np.float64)
    y = df["x1"] * df["x2"] + df["x3"] * (1 + df["x4"])
4163
4164
4165
    y = y.fillna(0)

    # train with regular dtypes
4166
    params = {"objective": "l2", "num_leaves": 31, "min_child_samples": 1}
4167
4168
4169
4170
4171
4172
    ds = lgb.Dataset(df, y)
    bst = lgb.train(params, ds, num_boost_round=5)
    preds = bst.predict(df)

    # convert to nullable dtypes
    df2 = df.copy()
4173
4174
4175
    df2["x1"] = df2["x1"].astype("Int32")
    df2["x2"] = df2["x2"].astype("Float64")
    df2["x4"] = df2["x4"].astype("boolean")
4176
4177
4178
4179
4180
4181
4182
4183

    # test training succeeds
    ds_nullable_dtypes = lgb.Dataset(df2, y)
    bst_nullable_dtypes = lgb.train(params, ds_nullable_dtypes, num_boost_round=5)
    preds_nullable_dtypes = bst_nullable_dtypes.predict(df2)

    trees_df = bst_nullable_dtypes.trees_to_dataframe()
    # test all features were used
4184
    assert trees_df["split_feature"].nunique() == df.shape[1]
4185
4186
4187
4188
4189
4190
    # test the score is better than predicting the mean
    baseline = np.full_like(y, y.mean())
    assert mean_squared_error(y, preds) < mean_squared_error(y, baseline)

    # test equal predictions
    np.testing.assert_allclose(preds, preds_nullable_dtypes)
4191
4192
4193
4194
4195


def test_boost_from_average_with_single_leaf_trees():
    # test data are taken from bug report
    # https://github.com/microsoft/LightGBM/issues/4708
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
    X = np.array(
        [
            [1021.0589, 1018.9578],
            [1023.85754, 1018.7854],
            [1024.5468, 1018.88513],
            [1019.02954, 1018.88513],
            [1016.79926, 1018.88513],
            [1007.6, 1018.88513],
        ],
        dtype=np.float32,
    )
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
    y = np.array([1023.8, 1024.6, 1024.4, 1023.8, 1022.0, 1014.4], dtype=np.float32)
    params = {
        "extra_trees": True,
        "min_data_in_bin": 1,
        "extra_seed": 7,
        "objective": "regression",
        "verbose": -1,
        "boost_from_average": True,
        "min_data_in_leaf": 1,
    }
    train_set = lgb.Dataset(X, y)
    model = lgb.train(params=params, train_set=train_set, num_boost_round=10)

    preds = model.predict(X)
    mean_preds = np.mean(preds)
    assert y.min() <= mean_preds <= y.max()
4223
4224


4225
def test_cegb_split_buffer_clean(rng_fixed_seed):
4226
4227
4228
4229
4230
4231
4232
4233
    # modified from https://github.com/microsoft/LightGBM/issues/3679#issuecomment-938652811
    # and https://github.com/microsoft/LightGBM/pull/5087
    # test that the ``splits_per_leaf_`` of CEGB is cleaned before training a new tree
    # which is done in the fix #5164
    # without the fix:
    #    Check failed: (best_split_info.left_count) > (0)

    R, C = 1000, 100
4234
    data = rng_fixed_seed.standard_normal(size=(R, C))
4235
    for i in range(1, C):
4236
        data[i] += data[0] * rng_fixed_seed.standard_normal()
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246

    N = int(0.8 * len(data))
    train_data = data[:N]
    test_data = data[N:]
    train_y = np.sum(train_data, axis=1)
    test_y = np.sum(test_data, axis=1)

    train = lgb.Dataset(train_data, train_y, free_raw_data=True)

    params = {
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
        "boosting_type": "gbdt",
        "objective": "regression",
        "max_bin": 255,
        "num_leaves": 31,
        "seed": 0,
        "learning_rate": 0.1,
        "min_data_in_leaf": 0,
        "verbose": -1,
        "min_split_gain": 1000.0,
        "cegb_penalty_feature_coupled": 5 * np.arange(C),
        "cegb_penalty_split": 0.0002,
        "cegb_tradeoff": 10.0,
        "force_col_wise": True,
4260
4261
4262
4263
4264
4265
    }

    model = lgb.train(params, train, num_boost_round=10)
    predicts = model.predict(test_data)
    rmse = np.sqrt(mean_squared_error(test_y, predicts))
    assert rmse < 10.0
4266
4267


4268
4269
4270
4271
def test_verbosity_and_verbose(capsys):
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, y)
    params = {
4272
4273
4274
        "num_leaves": 3,
        "verbose": 1,
        "verbosity": 0,
4275
4276
    }
    lgb.train(params, ds, num_boost_round=1)
4277
    expected_msg = "[LightGBM] [Warning] verbosity is set=0, verbose=1 will be ignored. " "Current value: verbosity=0"
4278
4279
4280
4281
    stdout = capsys.readouterr().out
    assert expected_msg in stdout


4282
4283
@pytest.mark.parametrize("verbosity_param", lgb.basic._ConfigAliases.get("verbosity"))
@pytest.mark.parametrize("verbosity", [-1, 0])
4284
4285
4286
4287
def test_verbosity_can_suppress_alias_warnings(capsys, verbosity_param, verbosity):
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, y)
    params = {
4288
4289
4290
4291
        "num_leaves": 3,
        "subsample": 0.75,
        "bagging_fraction": 0.8,
        "force_col_wise": True,
4292
4293
4294
4295
        verbosity_param: verbosity,
    }
    lgb.train(params, ds, num_boost_round=1)
    expected_msg = (
4296
4297
        "[LightGBM] [Warning] bagging_fraction is set=0.8, subsample=0.75 will be ignored. "
        "Current value: bagging_fraction=0.8"
4298
4299
4300
4301
4302
    )
    stdout = capsys.readouterr().out
    if verbosity >= 0:
        assert expected_msg in stdout
    else:
4303
        assert re.search(r"\[LightGBM\]", stdout) is None
4304
4305


4306
@pytest.mark.skipif(not PANDAS_INSTALLED, reason="pandas is not installed")
4307
4308
def test_validate_features():
    X, y = make_synthetic_regression()
4309
    features = ["x1", "x2", "x3", "x4"]
4310
4311
    df = pd_DataFrame(X, columns=features)
    ds = lgb.Dataset(df, y)
4312
    bst = lgb.train({"num_leaves": 15, "verbose": -1}, ds, num_boost_round=10)
4313
4314
4315
    assert bst.feature_name() == features

    # try to predict with a different feature
4316
    df2 = df.rename(columns={"x3": "z"})
4317
4318
4319
4320
4321
    with pytest.raises(lgb.basic.LightGBMError, match="Expected 'x3' at position 2 but found 'z'"):
        bst.predict(df2, validate_features=True)

    # check that disabling the check doesn't raise the error
    bst.predict(df2, validate_features=False)
4322
4323
4324
4325
4326
4327
4328

    # try to refit with a different feature
    with pytest.raises(lgb.basic.LightGBMError, match="Expected 'x3' at position 2 but found 'z'"):
        bst.refit(df2, y, validate_features=True)

    # check that disabling the check doesn't raise the error
    bst.refit(df2, y, validate_features=False)
4329
4330


4331
4332
4333
4334
4335
4336
4337
def test_train_and_cv_raise_informative_error_for_train_set_of_wrong_type():
    with pytest.raises(TypeError, match=r"train\(\) only accepts Dataset object, train_set has type 'list'\."):
        lgb.train({}, train_set=[])
    with pytest.raises(TypeError, match=r"cv\(\) only accepts Dataset object, train_set has type 'list'\."):
        lgb.cv({}, train_set=[])


4338
@pytest.mark.parametrize("num_boost_round", [-7, -1, 0])
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
def test_train_and_cv_raise_informative_error_for_impossible_num_boost_round(num_boost_round):
    X, y = make_synthetic_regression(n_samples=100)
    error_msg = rf"num_boost_round must be greater than 0\. Got {num_boost_round}\."
    with pytest.raises(ValueError, match=error_msg):
        lgb.train({}, train_set=lgb.Dataset(X, y), num_boost_round=num_boost_round)
    with pytest.raises(ValueError, match=error_msg):
        lgb.cv({}, train_set=lgb.Dataset(X, y), num_boost_round=num_boost_round)


def test_train_raises_informative_error_if_any_valid_sets_are_not_dataset_objects():
    X, y = make_synthetic_regression(n_samples=100)
    X_valid = X * 2.0
4351
4352
4353
    with pytest.raises(
        TypeError, match=r"Every item in valid_sets must be a Dataset object\. Item 1 has type 'tuple'\."
    ):
4354
4355
4356
        lgb.train(
            params={},
            train_set=lgb.Dataset(X, y),
4357
            valid_sets=[lgb.Dataset(X_valid, y), ([1.0], [2.0]), [5.6, 5.7, 5.8]],
4358
4359
4360
        )


4361
4362
def test_train_raises_informative_error_for_params_of_wrong_type():
    X, y = make_synthetic_regression()
4363
    params = {"num_leaves": "too-many"}
4364
    dtrain = lgb.Dataset(X, label=y)
4365
    with pytest.raises(lgb.basic.LightGBMError, match='Parameter num_leaves should be of type int, got "too-many"'):
4366
        lgb.train(params, dtrain)
4367
4368
4369
4370
4371


def test_quantized_training():
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, label=y)
4372
    bst_params = {"num_leaves": 15, "verbose": -1, "seed": 0}
4373
4374
    bst = lgb.train(bst_params, ds, num_boost_round=10)
    rmse = np.sqrt(np.mean((bst.predict(X) - y) ** 2))
4375
4376
4377
4378
4379
4380
4381
    bst_params.update(
        {
            "use_quantized_grad": True,
            "num_grad_quant_bins": 30,
            "quant_train_renew_leaf": True,
        }
    )
4382
4383
4384
    quant_bst = lgb.train(bst_params, ds, num_boost_round=10)
    quant_rmse = np.sqrt(np.mean((quant_bst.predict(X) - y) ** 2))
    assert quant_rmse < rmse + 6.0