test_engine.py 171 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
# coding: utf-8
wxchan's avatar
wxchan committed
2
import copy
3
import itertools
4
import json
wxchan's avatar
wxchan committed
5
import math
6
import pickle
7
import platform
8
import random
9
import re
10
from os import getenv
11
from pathlib import Path
12
from shutil import copyfile
wxchan's avatar
wxchan committed
13
14

import numpy as np
15
import psutil
16
import pytest
17
from scipy.sparse import csr_matrix, isspmatrix_csc, isspmatrix_csr
18
from sklearn.datasets import load_svmlight_file, make_blobs, make_multilabel_classification
19
20
from sklearn.metrics import average_precision_score, log_loss, mean_absolute_error, mean_squared_error, roc_auc_score
from sklearn.model_selection import GroupKFold, TimeSeriesSplit, train_test_split
wxchan's avatar
wxchan committed
21

22
import lightgbm as lgb
23
from lightgbm.compat import PANDAS_INSTALLED, pd_DataFrame, pd_Series
24

25
26
27
28
29
30
31
32
33
34
35
36
37
from .utils import (
    SERIALIZERS,
    dummy_obj,
    load_breast_cancer,
    load_digits,
    load_iris,
    logistic_sigmoid,
    make_synthetic_regression,
    mse_obj,
    pickle_and_unpickle_object,
    sklearn_multiclass_custom_objective,
    softmax,
)
wxchan's avatar
wxchan committed
38

39
40
41
decreasing_generator = itertools.count(0, -1)


42
43
44
45
46
47
def logloss_obj(preds, train_data):
    y_true = train_data.get_label()
    y_pred = logistic_sigmoid(preds)
    grad = y_pred - y_true
    hess = y_pred * (1.0 - y_pred)
    return grad, hess
48
49


wxchan's avatar
wxchan committed
50
51
52
def multi_logloss(y_true, y_pred):
    return np.mean([-math.log(y_pred[i][y]) for i, y in enumerate(y_true)])

wxchan's avatar
wxchan committed
53

Belinda Trotta's avatar
Belinda Trotta committed
54
55
56
57
58
59
60
def top_k_error(y_true, y_pred, k):
    if k == y_pred.shape[1]:
        return 0
    max_rest = np.max(-np.partition(-y_pred, k)[:, k:], axis=1)
    return 1 - np.mean((y_pred[np.arange(len(y_true)), y_true] > max_rest))


61
def constant_metric(preds, train_data):
62
    return ("error", 0.0, False)
63
64
65


def decreasing_metric(preds, train_data):
66
    return ("decreasing_metric", next(decreasing_generator), False)
67
68


69
70
71
72
def categorize(continuous_x):
    return np.digitize(continuous_x, bins=np.arange(0, 1, 0.01))


73
74
75
76
def test_binary():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
77
78
79
80
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
        "num_iteration": 50,  # test num_iteration in dict here
81
82
83
84
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    evals_result = {}
85
    gbm = lgb.train(
86
        params, lgb_train, num_boost_round=20, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
87
    )
88
89
    ret = log_loss(y_test, gbm.predict(X_test))
    assert ret < 0.14
90
91
    assert len(evals_result["valid_0"]["binary_logloss"]) == 50
    assert evals_result["valid_0"]["binary_logloss"][-1] == pytest.approx(ret)
92
93
94
95
96
97


def test_rf():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
98
99
100
101
102
103
104
105
        "boosting_type": "rf",
        "objective": "binary",
        "bagging_freq": 1,
        "bagging_fraction": 0.5,
        "feature_fraction": 0.5,
        "num_leaves": 50,
        "metric": "binary_logloss",
        "verbose": -1,
106
107
108
109
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    evals_result = {}
110
    gbm = lgb.train(
111
        params, lgb_train, num_boost_round=50, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
112
    )
113
114
    ret = log_loss(y_test, gbm.predict(X_test))
    assert ret < 0.19
115
    assert evals_result["valid_0"]["binary_logloss"][-1] == pytest.approx(ret)
116
117


118
@pytest.mark.parametrize("objective", ["regression", "regression_l1", "huber", "fair", "poisson", "quantile"])
119
def test_regression(objective):
120
121
    X, y = make_synthetic_regression()
    y = np.abs(y)
122
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
123
    params = {"objective": objective, "metric": "l2", "verbose": -1}
124
125
126
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    evals_result = {}
127
    gbm = lgb.train(
128
        params, lgb_train, num_boost_round=50, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
129
    )
130
    ret = mean_squared_error(y_test, gbm.predict(X_test))
131
    if objective == "huber":
132
        assert ret < 430
133
    elif objective == "fair":
134
        assert ret < 296
135
    elif objective == "poisson":
136
        assert ret < 193
137
    elif objective == "quantile":
138
        assert ret < 1311
139
    else:
140
        assert ret < 343
141
    assert evals_result["valid_0"]["l2"][-1] == pytest.approx(ret)
142
143
144
145
146
147
148
149
150
151
152
153


def test_missing_value_handle():
    X_train = np.zeros((100, 1))
    y_train = np.zeros(100)
    trues = random.sample(range(100), 20)
    for idx in trues:
        X_train[idx, 0] = np.nan
        y_train[idx] = 1
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

154
    params = {"metric": "l2", "verbose": -1, "boost_from_average": False}
155
    evals_result = {}
156
    gbm = lgb.train(
157
        params, lgb_train, num_boost_round=20, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
158
    )
159
160
    ret = mean_squared_error(y_train, gbm.predict(X_train))
    assert ret < 0.005
161
    assert evals_result["valid_0"]["l2"][-1] == pytest.approx(ret)
162
163
164
165
166
167
168
169
170
171
172
173


def test_missing_value_handle_more_na():
    X_train = np.ones((100, 1))
    y_train = np.ones(100)
    trues = random.sample(range(100), 80)
    for idx in trues:
        X_train[idx, 0] = np.nan
        y_train[idx] = 0
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

174
    params = {"metric": "l2", "verbose": -1, "boost_from_average": False}
175
    evals_result = {}
176
    gbm = lgb.train(
177
        params, lgb_train, num_boost_round=20, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
178
    )
179
180
    ret = mean_squared_error(y_train, gbm.predict(X_train))
    assert ret < 0.005
181
    assert evals_result["valid_0"]["l2"][-1] == pytest.approx(ret)
182
183
184
185
186
187
188
189
190
191
192
193


def test_missing_value_handle_na():
    x = [0, 1, 2, 3, 4, 5, 6, 7, np.nan]
    y = [1, 1, 1, 1, 0, 0, 0, 0, 1]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
194
195
196
197
198
199
200
201
202
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "zero_as_missing": False,
203
204
    }
    evals_result = {}
205
    gbm = lgb.train(
206
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
207
    )
208
209
210
211
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
212
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
213
214
215
216
217
218
219
220
221
222
223
224


def test_missing_value_handle_zero():
    x = [0, 1, 2, 3, 4, 5, 6, 7, np.nan]
    y = [0, 1, 1, 1, 0, 0, 0, 0, 0]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
225
226
227
228
229
230
231
232
233
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "zero_as_missing": True,
234
235
    }
    evals_result = {}
236
    gbm = lgb.train(
237
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
238
    )
239
240
241
242
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
243
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
244
245
246
247
248
249
250
251
252
253
254
255


def test_missing_value_handle_none():
    x = [0, 1, 2, 3, 4, 5, 6, 7, np.nan]
    y = [0, 1, 1, 1, 0, 0, 0, 0, 0]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
256
257
258
259
260
261
262
263
264
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "use_missing": False,
265
266
    }
    evals_result = {}
267
    gbm = lgb.train(
268
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
269
    )
270
271
272
273
274
    pred = gbm.predict(X_train)
    assert pred[0] == pytest.approx(pred[1])
    assert pred[-1] == pytest.approx(pred[0])
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.83
275
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
276
277


278
279
280
281
282
283
284
285
286
287
288
289
290
291
@pytest.mark.parametrize(
    "use_quantized_grad",
    [
        pytest.param(
            True,
            marks=pytest.mark.skipif(
                getenv("TASK", "") == "cuda",
                reason="Skip because quantized training with categorical features is not supported for cuda version",
            ),
        ),
        False,
    ],
)
def test_categorical_handle(use_quantized_grad):
292
293
294
295
296
297
298
299
300
    x = [0, 1, 2, 3, 4, 5, 6, 7]
    y = [0, 1, 0, 1, 0, 1, 0, 1]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
301
302
303
304
305
306
307
308
309
310
311
312
313
314
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "min_data_per_group": 1,
        "cat_smooth": 1,
        "cat_l2": 0,
        "max_cat_to_onehot": 1,
        "zero_as_missing": True,
        "categorical_column": 0,
315
        "use_quantized_grad": use_quantized_grad,
316
317
    }
    evals_result = {}
318
    gbm = lgb.train(
319
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
320
    )
321
322
323
324
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
325
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
326
327


328
329
330
331
332
333
334
335
336
337
338
339
340
341
@pytest.mark.parametrize(
    "use_quantized_grad",
    [
        pytest.param(
            True,
            marks=pytest.mark.skipif(
                getenv("TASK", "") == "cuda",
                reason="Skip because quantized training with categorical features is not supported for cuda version",
            ),
        ),
        False,
    ],
)
def test_categorical_handle_na(use_quantized_grad):
342
343
344
345
346
347
348
349
350
    x = [0, np.nan, 0, np.nan, 0, np.nan]
    y = [0, 1, 0, 1, 0, 1]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
351
352
353
354
355
356
357
358
359
360
361
362
363
364
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "min_data_per_group": 1,
        "cat_smooth": 1,
        "cat_l2": 0,
        "max_cat_to_onehot": 1,
        "zero_as_missing": False,
        "categorical_column": 0,
365
        "use_quantized_grad": use_quantized_grad,
366
367
    }
    evals_result = {}
368
    gbm = lgb.train(
369
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
370
    )
371
372
373
374
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
375
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
376
377


378
379
380
381
382
383
384
385
386
387
388
389
390
391
@pytest.mark.parametrize(
    "use_quantized_grad",
    [
        pytest.param(
            True,
            marks=pytest.mark.skipif(
                getenv("TASK", "") == "cuda",
                reason="Skip because quantized training with categorical features is not supported for cuda version",
            ),
        ),
        False,
    ],
)
def test_categorical_non_zero_inputs(use_quantized_grad):
392
393
394
395
396
397
398
399
400
    x = [1, 1, 1, 1, 1, 1, 2, 2]
    y = [1, 1, 1, 1, 1, 1, 0, 0]

    X_train = np.array(x).reshape(len(x), 1)
    y_train = np.array(y)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_train, y_train)

    params = {
401
402
403
404
405
406
407
408
409
410
411
412
413
414
        "objective": "regression",
        "metric": "auc",
        "verbose": -1,
        "boost_from_average": False,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "min_data_per_group": 1,
        "cat_smooth": 1,
        "cat_l2": 0,
        "max_cat_to_onehot": 1,
        "zero_as_missing": False,
        "categorical_column": 0,
415
        "use_quantized_grad": use_quantized_grad,
416
417
    }
    evals_result = {}
418
    gbm = lgb.train(
419
        params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
420
    )
421
422
423
424
    pred = gbm.predict(X_train)
    np.testing.assert_allclose(pred, y)
    ret = roc_auc_score(y_train, pred)
    assert ret > 0.999
425
    assert evals_result["valid_0"]["auc"][-1] == pytest.approx(ret)
426
427
428
429
430


def test_multiclass():
    X, y = load_digits(n_class=10, return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
431
    params = {"objective": "multiclass", "metric": "multi_logloss", "num_class": 10, "verbose": -1}
432
433
434
    lgb_train = lgb.Dataset(X_train, y_train, params=params)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, params=params)
    evals_result = {}
435
    gbm = lgb.train(
436
        params, lgb_train, num_boost_round=50, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
437
    )
438
439
    ret = multi_logloss(y_test, gbm.predict(X_test))
    assert ret < 0.16
440
    assert evals_result["valid_0"]["multi_logloss"][-1] == pytest.approx(ret)
441
442
443
444
445
446


def test_multiclass_rf():
    X, y = load_digits(n_class=10, return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
447
448
449
450
451
452
453
454
455
456
457
        "boosting_type": "rf",
        "objective": "multiclass",
        "metric": "multi_logloss",
        "bagging_freq": 1,
        "bagging_fraction": 0.6,
        "feature_fraction": 0.6,
        "num_class": 10,
        "num_leaves": 50,
        "min_data": 1,
        "verbose": -1,
        "gpu_use_dp": True,
458
459
460
461
    }
    lgb_train = lgb.Dataset(X_train, y_train, params=params)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, params=params)
    evals_result = {}
462
    gbm = lgb.train(
463
        params, lgb_train, num_boost_round=50, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
464
    )
465
466
    ret = multi_logloss(y_test, gbm.predict(X_test))
    assert ret < 0.23
467
    assert evals_result["valid_0"]["multi_logloss"][-1] == pytest.approx(ret)
468
469
470
471
472


def test_multiclass_prediction_early_stopping():
    X, y = load_digits(n_class=10, return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
473
    params = {"objective": "multiclass", "metric": "multi_logloss", "num_class": 10, "verbose": -1}
474
    lgb_train = lgb.Dataset(X_train, y_train, params=params)
475
    gbm = lgb.train(params, lgb_train, num_boost_round=50)
476

477
    pred_parameter = {"pred_early_stop": True, "pred_early_stop_freq": 5, "pred_early_stop_margin": 1.5}
478
479
480
481
    ret = multi_logloss(y_test, gbm.predict(X_test, **pred_parameter))
    assert ret < 0.8
    assert ret > 0.6  # loss will be higher than when evaluating the full model

482
    pred_parameter["pred_early_stop_margin"] = 5.5
483
484
485
486
487
488
    ret = multi_logloss(y_test, gbm.predict(X_test, **pred_parameter))
    assert ret < 0.2


def test_multi_class_error():
    X, y = load_digits(n_class=10, return_X_y=True)
489
    params = {"objective": "multiclass", "num_classes": 10, "metric": "multi_error", "num_leaves": 4, "verbose": -1}
490
491
492
493
    lgb_data = lgb.Dataset(X, label=y)
    est = lgb.train(params, lgb_data, num_boost_round=10)
    predict_default = est.predict(X)
    results = {}
494
    est = lgb.train(
495
        dict(params, multi_error_top_k=1),
496
497
498
        lgb_data,
        num_boost_round=10,
        valid_sets=[lgb_data],
499
        callbacks=[lgb.record_evaluation(results)],
500
    )
501
502
503
504
505
    predict_1 = est.predict(X)
    # check that default gives same result as k = 1
    np.testing.assert_allclose(predict_1, predict_default)
    # check against independent calculation for k = 1
    err = top_k_error(y, predict_1, 1)
506
    assert results["training"]["multi_error"][-1] == pytest.approx(err)
507
508
    # check against independent calculation for k = 2
    results = {}
509
    est = lgb.train(
510
        dict(params, multi_error_top_k=2),
511
512
513
        lgb_data,
        num_boost_round=10,
        valid_sets=[lgb_data],
514
        callbacks=[lgb.record_evaluation(results)],
515
    )
516
517
    predict_2 = est.predict(X)
    err = top_k_error(y, predict_2, 2)
518
    assert results["training"]["multi_error@2"][-1] == pytest.approx(err)
519
520
    # check against independent calculation for k = 10
    results = {}
521
    est = lgb.train(
522
        dict(params, multi_error_top_k=10),
523
524
525
        lgb_data,
        num_boost_round=10,
        valid_sets=[lgb_data],
526
        callbacks=[lgb.record_evaluation(results)],
527
    )
528
529
    predict_3 = est.predict(X)
    err = top_k_error(y, predict_3, 10)
530
    assert results["training"]["multi_error@10"][-1] == pytest.approx(err)
531
532
533
534
    # check cases where predictions are equal
    X = np.array([[0, 0], [0, 0]])
    y = np.array([0, 1])
    lgb_data = lgb.Dataset(X, label=y)
535
    params["num_classes"] = 2
536
    results = {}
537
538
    lgb.train(params, lgb_data, num_boost_round=10, valid_sets=[lgb_data], callbacks=[lgb.record_evaluation(results)])
    assert results["training"]["multi_error"][-1] == pytest.approx(1)
539
    results = {}
540
    lgb.train(
541
        dict(params, multi_error_top_k=2),
542
543
544
        lgb_data,
        num_boost_round=10,
        valid_sets=[lgb_data],
545
        callbacks=[lgb.record_evaluation(results)],
546
    )
547
    assert results["training"]["multi_error@2"][-1] == pytest.approx(0)
548
549


550
551
552
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Skip due to differences in implementation details of CUDA version"
)
553
554
555
556
557
558
def test_auc_mu():
    # should give same result as binary auc for 2 classes
    X, y = load_digits(n_class=10, return_X_y=True)
    y_new = np.zeros((len(y)))
    y_new[y != 0] = 1
    lgb_X = lgb.Dataset(X, label=y_new)
559
    params = {"objective": "multiclass", "metric": "auc_mu", "verbose": -1, "num_classes": 2, "seed": 0}
560
    results_auc_mu = {}
561
562
    lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_auc_mu)])
    params = {"objective": "binary", "metric": "auc", "verbose": -1, "seed": 0}
563
    results_auc = {}
564
565
    lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_auc)])
    np.testing.assert_allclose(results_auc_mu["training"]["auc_mu"], results_auc["training"]["auc"])
566
567
    # test the case where all predictions are equal
    lgb_X = lgb.Dataset(X[:10], label=y_new[:10])
568
569
570
571
572
573
574
575
    params = {
        "objective": "multiclass",
        "metric": "auc_mu",
        "verbose": -1,
        "num_classes": 2,
        "min_data_in_leaf": 20,
        "seed": 0,
    }
576
    results_auc_mu = {}
577
578
    lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_auc_mu)])
    assert results_auc_mu["training"]["auc_mu"][-1] == pytest.approx(0.5)
579
580
581
582
583
584
    # test that weighted data gives different auc_mu
    lgb_X = lgb.Dataset(X, label=y)
    lgb_X_weighted = lgb.Dataset(X, label=y, weight=np.abs(np.random.normal(size=y.shape)))
    results_unweighted = {}
    results_weighted = {}
    params = dict(params, num_classes=10, num_leaves=5)
585
    lgb.train(
586
        params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_unweighted)]
587
588
589
590
591
592
    )
    lgb.train(
        params,
        lgb_X_weighted,
        num_boost_round=10,
        valid_sets=[lgb_X_weighted],
593
        callbacks=[lgb.record_evaluation(results_weighted)],
594
    )
595
596
    assert results_weighted["training"]["auc_mu"][-1] < 1
    assert results_unweighted["training"]["auc_mu"][-1] != results_weighted["training"]["auc_mu"][-1]
597
598
    # test that equal data weights give same auc_mu as unweighted data
    lgb_X_weighted = lgb.Dataset(X, label=y, weight=np.ones(y.shape) * 0.5)
599
600
601
602
603
    lgb.train(
        params,
        lgb_X_weighted,
        num_boost_round=10,
        valid_sets=[lgb_X_weighted],
604
605
606
607
        callbacks=[lgb.record_evaluation(results_weighted)],
    )
    assert results_unweighted["training"]["auc_mu"][-1] == pytest.approx(
        results_weighted["training"]["auc_mu"][-1], abs=1e-5
608
    )
609
610
611
612
    # should give 1 when accuracy = 1
    X = X[:10, :]
    y = y[:10]
    lgb_X = lgb.Dataset(X, label=y)
613
    params = {"objective": "multiclass", "metric": "auc_mu", "num_classes": 10, "min_data_in_leaf": 1, "verbose": -1}
614
    results = {}
615
616
    lgb.train(params, lgb_X, num_boost_round=100, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results)])
    assert results["training"]["auc_mu"][-1] == pytest.approx(1)
617
    # test loading class weights
618
    Xy = np.loadtxt(
619
        str(Path(__file__).absolute().parents[2] / "examples" / "multiclass_classification" / "multiclass.train")
620
    )
621
622
623
    y = Xy[:, 0]
    X = Xy[:, 1:]
    lgb_X = lgb.Dataset(X, label=y)
624
625
626
627
628
629
630
631
    params = {
        "objective": "multiclass",
        "metric": "auc_mu",
        "auc_mu_weights": [0, 2, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0],
        "num_classes": 5,
        "verbose": -1,
        "seed": 0,
    }
632
    results_weight = {}
633
634
    lgb.train(params, lgb_X, num_boost_round=5, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_weight)])
    params["auc_mu_weights"] = []
635
    results_no_weight = {}
636
    lgb.train(
637
        params, lgb_X, num_boost_round=5, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(results_no_weight)]
638
    )
639
    assert results_weight["training"]["auc_mu"][-1] != results_no_weight["training"]["auc_mu"][-1]
640
641


642
def test_ranking_prediction_early_stopping():
643
644
645
646
647
    rank_example_dir = Path(__file__).absolute().parents[2] / "examples" / "lambdarank"
    X_train, y_train = load_svmlight_file(str(rank_example_dir / "rank.train"))
    q_train = np.loadtxt(str(rank_example_dir / "rank.train.query"))
    X_test, _ = load_svmlight_file(str(rank_example_dir / "rank.test"))
    params = {"objective": "rank_xendcg", "verbose": -1}
648
649
650
    lgb_train = lgb.Dataset(X_train, y_train, group=q_train, params=params)
    gbm = lgb.train(params, lgb_train, num_boost_round=50)

651
    pred_parameter = {"pred_early_stop": True, "pred_early_stop_freq": 5, "pred_early_stop_margin": 1.5}
652
653
654
655
656
657
658
659
    ret_early = gbm.predict(X_test, **pred_parameter)

    pred_parameter["pred_early_stop_margin"] = 5.5
    ret_early_more_strict = gbm.predict(X_test, **pred_parameter)
    with pytest.raises(AssertionError):
        np.testing.assert_allclose(ret_early, ret_early_more_strict)


660
661
662
663
664
665
666
# Simulates position bias for a given ranking dataset.
# The ouput dataset is identical to the input one with the exception for the relevance labels.
# The new labels are generated according to an instance of a cascade user model:
# for each query, the user is simulated to be traversing the list of documents ranked by a baseline ranker
# (in our example it is simply the ordering by some feature correlated with relevance, e.g., 34)
# and clicks on that document (new_label=1) with some probability 'pclick' depending on its true relevance;
# at each position the user may stop the traversal with some probability pstop. For the non-clicked documents,
667
# new_label=0. Thus the generated new labels are biased towards the baseline ranker.
668
669
670
671
672
673
674
675
676
677
678
679
680
681
# The positions of the documents in the ranked lists produced by the baseline, are returned.
def simulate_position_bias(file_dataset_in, file_query_in, file_dataset_out, baseline_feature):
    # a mapping of a document's true relevance (defined on a 5-grade scale) into the probability of clicking it
    def get_pclick(label):
        if label == 0:
            return 0.4
        elif label == 1:
            return 0.6
        elif label == 2:
            return 0.7
        elif label == 3:
            return 0.8
        else:
            return 0.9
682

683
684
    # an instantiation of a cascade model where the user stops with probability 0.2 after observing each document
    pstop = 0.2
685

686
687
    f_dataset_in = open(file_dataset_in, "r")
    f_dataset_out = open(file_dataset_out, "w")
688
689
690
    random.seed(10)
    positions_all = []
    for line in open(file_query_in):
691
        docs_num = int(line)
692
        lines = []
693
        index_values = []
694
695
696
697
698
699
        positions = [0] * docs_num
        for index in range(docs_num):
            features = f_dataset_in.readline().split()
            lines.append(features)
            val = 0.0
            for feature_val in features:
700
                feature_val_split = feature_val.split(":")
701
702
703
704
                if int(feature_val_split[0]) == baseline_feature:
                    val = float(feature_val_split[1])
            index_values.append([index, val])
        index_values.sort(key=lambda x: -x[1])
705
        stop = False
706
707
708
709
710
711
712
        for pos in range(docs_num):
            index = index_values[pos][0]
            new_label = 0
            if not stop:
                label = int(lines[index][0])
                pclick = get_pclick(label)
                if random.random() < pclick:
713
                    new_label = 1
714
715
716
717
                stop = random.random() < pstop
            lines[index][0] = str(new_label)
            positions[index] = pos
        for features in lines:
718
            f_dataset_out.write(" ".join(features) + "\n")
719
720
721
722
723
        positions_all.extend(positions)
    f_dataset_out.close()
    return positions_all


724
725
726
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Positions in learning to rank is not supported in CUDA version yet"
)
727
def test_ranking_with_position_information_with_file(tmp_path):
728
    rank_example_dir = Path(__file__).absolute().parents[2] / "examples" / "lambdarank"
729
    params = {
730
731
732
733
734
735
736
737
        "objective": "lambdarank",
        "verbose": -1,
        "eval_at": [3],
        "metric": "ndcg",
        "bagging_freq": 1,
        "bagging_fraction": 0.9,
        "min_data_in_leaf": 50,
        "min_sum_hessian_in_leaf": 5.0,
738
739
740
    }

    # simulate position bias for the train dataset and put the train dataset with biased labels to temp directory
741
742
743
744
745
746
747
748
749
    positions = simulate_position_bias(
        str(rank_example_dir / "rank.train"),
        str(rank_example_dir / "rank.train.query"),
        str(tmp_path / "rank.train"),
        baseline_feature=34,
    )
    copyfile(str(rank_example_dir / "rank.train.query"), str(tmp_path / "rank.train.query"))
    copyfile(str(rank_example_dir / "rank.test"), str(tmp_path / "rank.test"))
    copyfile(str(rank_example_dir / "rank.test.query"), str(tmp_path / "rank.test.query"))
750

751
752
753
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
    gbm_baseline = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
754

755
    f_positions_out = open(str(tmp_path / "rank.train.position"), "w")
756
    for pos in positions:
757
        f_positions_out.write(str(pos) + "\n")
758
759
    f_positions_out.close()

760
761
762
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
    gbm_unbiased_with_file = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
763

764
    # the performance of the unbiased LambdaMART should outperform the plain LambdaMART on the dataset with position bias
765
    assert gbm_baseline.best_score["valid_0"]["ndcg@3"] + 0.03 <= gbm_unbiased_with_file.best_score["valid_0"]["ndcg@3"]
766
767

    # add extra row to position file
768
769
    with open(str(tmp_path / "rank.train.position"), "a") as file:
        file.write("pos_1000\n")
770
        file.close()
771
772
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
773
    with pytest.raises(lgb.basic.LightGBMError, match=r"Positions size \(3006\) doesn't match data size"):
774
        lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
775
776


777
778
779
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Positions in learning to rank is not supported in CUDA version yet"
)
780
def test_ranking_with_position_information_with_dataset_constructor(tmp_path):
781
    rank_example_dir = Path(__file__).absolute().parents[2] / "examples" / "lambdarank"
782
    params = {
783
784
785
786
787
788
789
790
791
792
793
        "objective": "lambdarank",
        "verbose": -1,
        "eval_at": [3],
        "metric": "ndcg",
        "bagging_freq": 1,
        "bagging_fraction": 0.9,
        "min_data_in_leaf": 50,
        "min_sum_hessian_in_leaf": 5.0,
        "num_threads": 1,
        "deterministic": True,
        "seed": 0,
794
795
796
    }

    # simulate position bias for the train dataset and put the train dataset with biased labels to temp directory
797
798
799
800
801
802
803
804
805
    positions = simulate_position_bias(
        str(rank_example_dir / "rank.train"),
        str(rank_example_dir / "rank.train.query"),
        str(tmp_path / "rank.train"),
        baseline_feature=34,
    )
    copyfile(str(rank_example_dir / "rank.train.query"), str(tmp_path / "rank.train.query"))
    copyfile(str(rank_example_dir / "rank.test"), str(tmp_path / "rank.test"))
    copyfile(str(rank_example_dir / "rank.test.query"), str(tmp_path / "rank.test.query"))
806

807
808
809
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
    gbm_baseline = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
810
811
812
813

    positions = np.array(positions)

    # test setting positions through Dataset constructor with numpy array
814
815
816
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params, position=positions)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
    gbm_unbiased = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
817
818

    # the performance of the unbiased LambdaMART should outperform the plain LambdaMART on the dataset with position bias
819
    assert gbm_baseline.best_score["valid_0"]["ndcg@3"] + 0.03 <= gbm_unbiased.best_score["valid_0"]["ndcg@3"]
820
821
822

    if PANDAS_INSTALLED:
        # test setting positions through Dataset constructor with pandas Series
823
824
825
826
827
828
        lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params, position=pd_Series(positions))
        lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
        gbm_unbiased_pandas_series = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
        assert (
            gbm_unbiased.best_score["valid_0"]["ndcg@3"] == gbm_unbiased_pandas_series.best_score["valid_0"]["ndcg@3"]
        )
829
830

    # test setting positions through set_position
831
832
    lgb_train = lgb.Dataset(str(tmp_path / "rank.train"), params=params)
    lgb_valid = [lgb_train.create_valid(str(tmp_path / "rank.test"))]
833
    lgb_train.set_position(positions)
834
835
    gbm_unbiased_set_position = lgb.train(params, lgb_train, valid_sets=lgb_valid, num_boost_round=50)
    assert gbm_unbiased.best_score["valid_0"]["ndcg@3"] == gbm_unbiased_set_position.best_score["valid_0"]["ndcg@3"]
836
837
838
839
840
841

    # test get_position works
    positions_from_get = lgb_train.get_position()
    np.testing.assert_array_equal(positions_from_get, positions)


842
843
def test_early_stopping():
    X, y = load_breast_cancer(return_X_y=True)
844
    params = {"objective": "binary", "metric": "binary_logloss", "verbose": -1}
845
846
847
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
848
    valid_set_name = "valid_set"
849
    # no early stopping
850
851
852
853
854
855
856
857
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=10,
        valid_sets=lgb_eval,
        valid_names=valid_set_name,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
    )
858
859
    assert gbm.best_iteration == 10
    assert valid_set_name in gbm.best_score
860
    assert "binary_logloss" in gbm.best_score[valid_set_name]
861
    # early stopping occurs
862
863
864
865
866
867
868
869
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=40,
        valid_sets=lgb_eval,
        valid_names=valid_set_name,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
    )
870
871
    assert gbm.best_iteration <= 39
    assert valid_set_name in gbm.best_score
872
    assert "binary_logloss" in gbm.best_score[valid_set_name]
873
874


875
@pytest.mark.parametrize("use_valid", [True, False])
876
877
878
879
880
881
882
883
884
def test_early_stopping_ignores_training_set(use_valid):
    x = np.linspace(-1, 1, 100)
    X = x.reshape(-1, 1)
    y = x**2
    X_train, X_valid = X[:80], X[80:]
    y_train, y_valid = y[:80], y[80:]
    train_ds = lgb.Dataset(X_train, y_train)
    valid_ds = lgb.Dataset(X_valid, y_valid)
    valid_sets = [train_ds]
885
    valid_names = ["train"]
886
887
    if use_valid:
        valid_sets.append(valid_ds)
888
        valid_names.append("valid")
889
890
891
892
    eval_result = {}

    def train_fn():
        return lgb.train(
893
            {"num_leaves": 5},
894
895
896
897
            train_ds,
            num_boost_round=2,
            valid_sets=valid_sets,
            valid_names=valid_names,
898
            callbacks=[lgb.early_stopping(1), lgb.record_evaluation(eval_result)],
899
        )
900

901
902
903
    if use_valid:
        bst = train_fn()
        assert bst.best_iteration == 1
904
905
        assert eval_result["train"]["l2"][1] < eval_result["train"]["l2"][0]  # train improved
        assert eval_result["valid"]["l2"][1] > eval_result["valid"]["l2"][0]  # valid didn't
906
    else:
907
        with pytest.warns(UserWarning, match="Only training set found, disabling early stopping."):
908
909
910
911
912
            bst = train_fn()
        assert bst.current_iteration() == 2
        assert bst.best_iteration == 0


913
@pytest.mark.parametrize("first_metric_only", [True, False])
914
915
916
917
def test_early_stopping_via_global_params(first_metric_only):
    X, y = load_breast_cancer(return_X_y=True)
    num_trees = 5
    params = {
918
919
920
921
922
923
        "num_trees": num_trees,
        "objective": "binary",
        "metric": "None",
        "verbose": -1,
        "early_stopping_round": 2,
        "first_metric_only": first_metric_only,
924
925
926
927
    }
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
928
929
930
931
    valid_set_name = "valid_set"
    gbm = lgb.train(
        params, lgb_train, feval=[decreasing_metric, constant_metric], valid_sets=lgb_eval, valid_names=valid_set_name
    )
932
933
934
935
936
    if first_metric_only:
        assert gbm.best_iteration == num_trees
    else:
        assert gbm.best_iteration == 1
    assert valid_set_name in gbm.best_score
937
938
    assert "decreasing_metric" in gbm.best_score[valid_set_name]
    assert "error" in gbm.best_score[valid_set_name]
939
940


941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
@pytest.mark.parametrize("early_stopping_round", [-10, -1, 0, None, "None"])
def test_early_stopping_is_not_enabled_for_non_positive_stopping_rounds(early_stopping_round):
    X, y = load_breast_cancer(return_X_y=True)
    num_trees = 5
    params = {
        "num_trees": num_trees,
        "objective": "binary",
        "metric": "None",
        "verbose": -1,
        "early_stopping_round": early_stopping_round,
        "first_metric_only": True,
    }
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    valid_set_name = "valid_set"

    if early_stopping_round is None:
        gbm = lgb.train(
            params,
            lgb_train,
            feval=[constant_metric],
            valid_sets=lgb_eval,
            valid_names=valid_set_name,
        )
        assert "early_stopping_round" not in gbm.params
        assert gbm.num_trees() == num_trees
    elif early_stopping_round == "None":
        with pytest.raises(TypeError, match="early_stopping_round should be an integer. Got 'str'"):
            gbm = lgb.train(
                params,
                lgb_train,
                feval=[constant_metric],
                valid_sets=lgb_eval,
                valid_names=valid_set_name,
            )
    elif early_stopping_round <= 0:
        gbm = lgb.train(
            params,
            lgb_train,
            feval=[constant_metric],
            valid_sets=lgb_eval,
            valid_names=valid_set_name,
        )
        assert gbm.params["early_stopping_round"] == early_stopping_round
        assert gbm.num_trees() == num_trees


989
990
991
@pytest.mark.parametrize("first_only", [True, False])
@pytest.mark.parametrize("single_metric", [True, False])
@pytest.mark.parametrize("greater_is_better", [True, False])
992
993
994
995
def test_early_stopping_min_delta(first_only, single_metric, greater_is_better):
    if single_metric and not first_only:
        pytest.skip("first_metric_only doesn't affect single metric.")
    metric2min_delta = {
996
997
998
999
        "auc": 0.001,
        "binary_logloss": 0.01,
        "average_precision": 0.001,
        "mape": 0.01,
1000
1001
1002
    }
    if single_metric:
        if greater_is_better:
1003
            metric = "auc"
1004
        else:
1005
            metric = "binary_logloss"
1006
1007
1008
    else:
        if first_only:
            if greater_is_better:
1009
                metric = ["auc", "binary_logloss"]
1010
            else:
1011
                metric = ["binary_logloss", "auc"]
1012
1013
        else:
            if greater_is_better:
1014
                metric = ["auc", "average_precision"]
1015
            else:
1016
                metric = ["binary_logloss", "mape"]
1017
1018
1019
1020
1021
1022

    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=0)
    train_ds = lgb.Dataset(X_train, y_train)
    valid_ds = lgb.Dataset(X_valid, y_valid, reference=train_ds)

1023
    params = {"objective": "binary", "metric": metric, "verbose": -1}
1024
1025
1026
1027
1028
1029
    if isinstance(metric, str):
        min_delta = metric2min_delta[metric]
    elif first_only:
        min_delta = metric2min_delta[metric[0]]
    else:
        min_delta = [metric2min_delta[m] for m in metric]
1030
1031
1032
1033
1034
    train_kwargs = {
        "params": params,
        "train_set": train_ds,
        "num_boost_round": 50,
        "valid_sets": [train_ds, valid_ds],
1035
        "valid_names": ["training", "valid"],
1036
    }
1037
1038
1039

    # regular early stopping
    evals_result = {}
1040
    train_kwargs["callbacks"] = [
1041
        lgb.callback.early_stopping(10, first_only, verbose=False),
1042
        lgb.record_evaluation(evals_result),
1043
1044
    ]
    bst = lgb.train(**train_kwargs)
1045
    scores = np.vstack(list(evals_result["valid"].values())).T
1046
1047
1048

    # positive min_delta
    delta_result = {}
1049
    train_kwargs["callbacks"] = [
1050
        lgb.callback.early_stopping(10, first_only, verbose=False, min_delta=min_delta),
1051
        lgb.record_evaluation(delta_result),
1052
1053
    ]
    delta_bst = lgb.train(**train_kwargs)
1054
    delta_scores = np.vstack(list(delta_result["valid"].values())).T
1055
1056
1057
1058
1059
1060

    if first_only:
        scores = scores[:, 0]
        delta_scores = delta_scores[:, 0]

    assert delta_bst.num_trees() < bst.num_trees()
1061
    np.testing.assert_allclose(scores[: len(delta_scores)], delta_scores)
1062
1063
1064
1065
1066
1067
1068
1069
    last_score = delta_scores[-1]
    best_score = delta_scores[delta_bst.num_trees() - 1]
    if greater_is_better:
        assert np.less_equal(last_score, best_score + min_delta).any()
    else:
        assert np.greater_equal(last_score, best_score - min_delta).any()


1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
@pytest.mark.parametrize("early_stopping_min_delta", [1e3, 0.0])
def test_early_stopping_min_delta_via_global_params(early_stopping_min_delta):
    X, y = load_breast_cancer(return_X_y=True)
    num_trees = 5
    params = {
        "num_trees": num_trees,
        "num_leaves": 5,
        "objective": "binary",
        "metric": "None",
        "verbose": -1,
        "early_stopping_round": 2,
        "early_stopping_min_delta": early_stopping_min_delta,
    }
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    gbm = lgb.train(params, lgb_train, feval=decreasing_metric, valid_sets=lgb_eval)
    if early_stopping_min_delta == 0:
        assert gbm.best_iteration == num_trees
    else:
        assert gbm.best_iteration == 1


1093
1094
1095
1096
1097
1098
def test_early_stopping_can_be_triggered_via_custom_callback():
    X, y = make_synthetic_regression()

    def _early_stop_after_seventh_iteration(env):
        if env.iteration == 6:
            exc = lgb.EarlyStopException(
1099
                best_iteration=6, best_score=[("some_validation_set", "some_metric", 0.708, True)]
1100
1101
1102
1103
            )
            raise exc

    bst = lgb.train(
1104
        params={"objective": "regression", "verbose": -1, "num_leaves": 2},
1105
1106
        train_set=lgb.Dataset(X, label=y),
        num_boost_round=23,
1107
        callbacks=[_early_stop_after_seventh_iteration],
1108
1109
1110
1111
1112
1113
1114
    )
    assert bst.num_trees() == 7
    assert bst.best_score["some_validation_set"]["some_metric"] == 0.708
    assert bst.best_iteration == 7
    assert bst.current_iteration() == 7


1115
def test_continue_train():
1116
    X, y = make_synthetic_regression()
1117
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
1118
    params = {"objective": "regression", "metric": "l1", "verbose": -1}
1119
1120
1121
    lgb_train = lgb.Dataset(X_train, y_train, free_raw_data=False)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, free_raw_data=False)
    init_gbm = lgb.train(params, lgb_train, num_boost_round=20)
1122
    model_name = "model.txt"
1123
1124
    init_gbm.save_model(model_name)
    evals_result = {}
1125
1126
1127
1128
1129
1130
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=30,
        valid_sets=lgb_eval,
        # test custom eval metrics
1131
        feval=(lambda p, d: ("custom_mae", mean_absolute_error(p, d.get_label()), False)),
1132
        callbacks=[lgb.record_evaluation(evals_result)],
1133
        init_model="model.txt",
1134
    )
1135
    ret = mean_absolute_error(y_test, gbm.predict(X_test))
1136
    assert ret < 13.6
1137
1138
    assert evals_result["valid_0"]["l1"][-1] == pytest.approx(ret)
    np.testing.assert_allclose(evals_result["valid_0"]["l1"], evals_result["valid_0"]["custom_mae"])
1139
1140
1141


def test_continue_train_reused_dataset():
1142
    X, y = make_synthetic_regression()
1143
    params = {"objective": "regression", "verbose": -1}
1144
1145
1146
1147
1148
1149
1150
1151
1152
    lgb_train = lgb.Dataset(X, y, free_raw_data=False)
    init_gbm = lgb.train(params, lgb_train, num_boost_round=5)
    init_gbm_2 = lgb.train(params, lgb_train, num_boost_round=5, init_model=init_gbm)
    init_gbm_3 = lgb.train(params, lgb_train, num_boost_round=5, init_model=init_gbm_2)
    gbm = lgb.train(params, lgb_train, num_boost_round=5, init_model=init_gbm_3)
    assert gbm.current_iteration() == 20


def test_continue_train_dart():
1153
    X, y = make_synthetic_regression()
1154
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
1155
    params = {"boosting_type": "dart", "objective": "regression", "metric": "l1", "verbose": -1}
1156
1157
1158
1159
    lgb_train = lgb.Dataset(X_train, y_train, free_raw_data=False)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, free_raw_data=False)
    init_gbm = lgb.train(params, lgb_train, num_boost_round=50)
    evals_result = {}
1160
1161
1162
1163
1164
1165
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=50,
        valid_sets=lgb_eval,
        callbacks=[lgb.record_evaluation(evals_result)],
1166
        init_model=init_gbm,
1167
    )
1168
    ret = mean_absolute_error(y_test, gbm.predict(X_test))
1169
    assert ret < 13.6
1170
    assert evals_result["valid_0"]["l1"][-1] == pytest.approx(ret)
1171
1172
1173
1174
1175


def test_continue_train_multiclass():
    X, y = load_iris(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
1176
    params = {"objective": "multiclass", "metric": "multi_logloss", "num_class": 3, "verbose": -1}
1177
1178
1179
1180
    lgb_train = lgb.Dataset(X_train, y_train, params=params, free_raw_data=False)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, params=params, free_raw_data=False)
    init_gbm = lgb.train(params, lgb_train, num_boost_round=20)
    evals_result = {}
1181
1182
1183
1184
1185
1186
    gbm = lgb.train(
        params,
        lgb_train,
        num_boost_round=30,
        valid_sets=lgb_eval,
        callbacks=[lgb.record_evaluation(evals_result)],
1187
        init_model=init_gbm,
1188
    )
1189
1190
    ret = multi_logloss(y_test, gbm.predict(X_test))
    assert ret < 0.1
1191
    assert evals_result["valid_0"]["multi_logloss"][-1] == pytest.approx(ret)
1192
1193
1194


def test_cv():
1195
    X_train, y_train = make_synthetic_regression()
1196
    params = {"verbose": -1}
1197
1198
    lgb_train = lgb.Dataset(X_train, y_train)
    # shuffle = False, override metric in params
1199
1200
1201
1202
1203
1204
1205
    params_with_metric = {"metric": "l2", "verbose": -1}
    cv_res = lgb.cv(
        params_with_metric, lgb_train, num_boost_round=10, nfold=3, stratified=False, shuffle=False, metrics="l1"
    )
    assert "valid l1-mean" in cv_res
    assert "valid l2-mean" not in cv_res
    assert len(cv_res["valid l1-mean"]) == 10
1206
    # shuffle = True, callbacks
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
    cv_res = lgb.cv(
        params,
        lgb_train,
        num_boost_round=10,
        nfold=3,
        stratified=False,
        shuffle=True,
        metrics="l1",
        callbacks=[lgb.reset_parameter(learning_rate=lambda i: 0.1 - 0.001 * i)],
    )
    assert "valid l1-mean" in cv_res
    assert len(cv_res["valid l1-mean"]) == 10
1219
    # enable display training loss
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
    cv_res = lgb.cv(
        params_with_metric,
        lgb_train,
        num_boost_round=10,
        nfold=3,
        stratified=False,
        shuffle=False,
        metrics="l1",
        eval_train_metric=True,
    )
    assert "train l1-mean" in cv_res
    assert "valid l1-mean" in cv_res
    assert "train l2-mean" not in cv_res
    assert "valid l2-mean" not in cv_res
    assert len(cv_res["train l1-mean"]) == 10
    assert len(cv_res["valid l1-mean"]) == 10
1236
1237
1238
    # self defined folds
    tss = TimeSeriesSplit(3)
    folds = tss.split(X_train)
1239
1240
    cv_res_gen = lgb.cv(params_with_metric, lgb_train, num_boost_round=10, folds=folds)
    cv_res_obj = lgb.cv(params_with_metric, lgb_train, num_boost_round=10, folds=tss)
1241
    np.testing.assert_allclose(cv_res_gen["valid l2-mean"], cv_res_obj["valid l2-mean"])
Andrew Ziem's avatar
Andrew Ziem committed
1242
    # LambdaRank
1243
1244
1245
1246
    rank_example_dir = Path(__file__).absolute().parents[2] / "examples" / "lambdarank"
    X_train, y_train = load_svmlight_file(str(rank_example_dir / "rank.train"))
    q_train = np.loadtxt(str(rank_example_dir / "rank.train.query"))
    params_lambdarank = {"objective": "lambdarank", "verbose": -1, "eval_at": 3}
1247
1248
    lgb_train = lgb.Dataset(X_train, y_train, group=q_train)
    # ... with l2 metric
1249
    cv_res_lambda = lgb.cv(params_lambdarank, lgb_train, num_boost_round=10, nfold=3, metrics="l2")
1250
    assert len(cv_res_lambda) == 2
1251
    assert not np.isnan(cv_res_lambda["valid l2-mean"]).any()
1252
    # ... with NDCG (default) metric
1253
    cv_res_lambda = lgb.cv(params_lambdarank, lgb_train, num_boost_round=10, nfold=3)
1254
    assert len(cv_res_lambda) == 2
1255
    assert not np.isnan(cv_res_lambda["valid ndcg@3-mean"]).any()
1256
    # self defined folds with lambdarank
1257
1258
    cv_res_lambda_obj = lgb.cv(params_lambdarank, lgb_train, num_boost_round=10, folds=GroupKFold(n_splits=3))
    np.testing.assert_allclose(cv_res_lambda["valid ndcg@3-mean"], cv_res_lambda_obj["valid ndcg@3-mean"])
1259
1260


1261
1262
def test_cv_works_with_init_model(tmp_path):
    X, y = make_synthetic_regression()
1263
    params = {"objective": "regression", "verbose": -1}
1264
1265
    num_train_rounds = 2
    lgb_train = lgb.Dataset(X, y, free_raw_data=False)
1266
    bst = lgb.train(params=params, train_set=lgb_train, num_boost_round=num_train_rounds)
1267
    preds_raw = bst.predict(X, raw_score=True)
1268
    model_path_txt = str(tmp_path / "lgb.model")
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
    bst.save_model(model_path_txt)

    num_cv_rounds = 5
    cv_kwargs = {
        "num_boost_round": num_cv_rounds,
        "nfold": 3,
        "stratified": False,
        "shuffle": False,
        "seed": 708,
        "return_cvbooster": True,
1279
        "params": params,
1280
1281
1282
    }

    # init_model from an in-memory Booster
1283
    cv_res = lgb.cv(train_set=lgb_train, init_model=bst, **cv_kwargs)
1284
1285
1286
    cv_bst_w_in_mem_init_model = cv_res["cvbooster"]
    assert cv_bst_w_in_mem_init_model.current_iteration() == [num_train_rounds + num_cv_rounds] * 3
    for booster in cv_bst_w_in_mem_init_model.boosters:
1287
        np.testing.assert_allclose(preds_raw, booster.predict(X, raw_score=True, num_iteration=num_train_rounds))
1288
1289

    # init_model from a text file
1290
    cv_res = lgb.cv(train_set=lgb_train, init_model=model_path_txt, **cv_kwargs)
1291
1292
1293
    cv_bst_w_file_init_model = cv_res["cvbooster"]
    assert cv_bst_w_file_init_model.current_iteration() == [num_train_rounds + num_cv_rounds] * 3
    for booster in cv_bst_w_file_init_model.boosters:
1294
        np.testing.assert_allclose(preds_raw, booster.predict(X, raw_score=True, num_iteration=num_train_rounds))
1295
1296
1297
1298

    # predictions should be identical
    for i in range(3):
        np.testing.assert_allclose(
1299
            cv_bst_w_in_mem_init_model.boosters[i].predict(X), cv_bst_w_file_init_model.boosters[i].predict(X)
1300
1301
1302
        )


1303
1304
1305
1306
def test_cvbooster():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1307
1308
1309
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
1310
    }
1311
    nfold = 3
1312
1313
    lgb_train = lgb.Dataset(X_train, y_train)
    # with early stopping
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
    cv_res = lgb.cv(
        params,
        lgb_train,
        num_boost_round=25,
        nfold=nfold,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
        return_cvbooster=True,
    )
    assert "cvbooster" in cv_res
    cvb = cv_res["cvbooster"]
1324
1325
    assert isinstance(cvb, lgb.CVBooster)
    assert isinstance(cvb.boosters, list)
1326
    assert len(cvb.boosters) == nfold
1327
1328
1329
    assert all(isinstance(bst, lgb.Booster) for bst in cvb.boosters)
    assert cvb.best_iteration > 0
    # predict by each fold booster
1330
    preds = cvb.predict(X_test)
1331
    assert isinstance(preds, list)
1332
1333
1334
1335
1336
1337
    assert len(preds) == nfold
    # check that each booster predicted using the best iteration
    for fold_preds, bst in zip(preds, cvb.boosters):
        assert bst.best_iteration == cvb.best_iteration
        expected = bst.predict(X_test, num_iteration=cvb.best_iteration)
        np.testing.assert_allclose(fold_preds, expected)
1338
1339
1340
1341
1342
    # fold averaging
    avg_pred = np.mean(preds, axis=0)
    ret = log_loss(y_test, avg_pred)
    assert ret < 0.13
    # without early stopping
1343
1344
    cv_res = lgb.cv(params, lgb_train, num_boost_round=20, nfold=3, return_cvbooster=True)
    cvb = cv_res["cvbooster"]
1345
1346
1347
1348
1349
1350
1351
    assert cvb.best_iteration == -1
    preds = cvb.predict(X_test)
    avg_pred = np.mean(preds, axis=0)
    ret = log_loss(y_test, avg_pred)
    assert ret < 0.15


1352
1353
1354
1355
def test_cvbooster_save_load(tmp_path):
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, _ = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1356
1357
1358
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
1359
1360
1361
1362
    }
    nfold = 3
    lgb_train = lgb.Dataset(X_train, y_train)

1363
1364
1365
1366
1367
1368
1369
1370
1371
    cv_res = lgb.cv(
        params,
        lgb_train,
        num_boost_round=10,
        nfold=nfold,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
        return_cvbooster=True,
    )
    cvbooster = cv_res["cvbooster"]
1372
1373
1374
    preds = cvbooster.predict(X_test)
    best_iteration = cvbooster.best_iteration

1375
    model_path_txt = str(tmp_path / "lgb.model")
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387

    cvbooster.save_model(model_path_txt)
    model_string = cvbooster.model_to_string()
    del cvbooster

    cvbooster_from_txt_file = lgb.CVBooster(model_file=model_path_txt)
    cvbooster_from_string = lgb.CVBooster().model_from_string(model_string)
    for cvbooster_loaded in [cvbooster_from_txt_file, cvbooster_from_string]:
        assert best_iteration == cvbooster_loaded.best_iteration
        np.testing.assert_array_equal(preds, cvbooster_loaded.predict(X_test))


1388
@pytest.mark.parametrize("serializer", SERIALIZERS)
1389
1390
1391
1392
def test_cvbooster_picklable(serializer):
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, _ = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1393
1394
1395
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
1396
1397
1398
1399
    }
    nfold = 3
    lgb_train = lgb.Dataset(X_train, y_train)

1400
1401
1402
1403
1404
1405
1406
1407
1408
    cv_res = lgb.cv(
        params,
        lgb_train,
        num_boost_round=10,
        nfold=nfold,
        callbacks=[lgb.early_stopping(stopping_rounds=5)],
        return_cvbooster=True,
    )
    cvbooster = cv_res["cvbooster"]
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
    preds = cvbooster.predict(X_test)
    best_iteration = cvbooster.best_iteration

    cvbooster_from_disk = pickle_and_unpickle_object(obj=cvbooster, serializer=serializer)
    del cvbooster

    assert best_iteration == cvbooster_from_disk.best_iteration

    preds_from_disk = cvbooster_from_disk.predict(X_test)
    np.testing.assert_array_equal(preds, preds_from_disk)


1421
def test_feature_name():
1422
    X_train, y_train = make_synthetic_regression()
1423
1424
    params = {"verbose": -1}
    feature_names = [f"f_{i}" for i in range(X_train.shape[-1])]
1425
1426
    lgb_train = lgb.Dataset(X_train, y_train, feature_name=feature_names)
    gbm = lgb.train(params, lgb_train, num_boost_round=5)
1427
1428
    assert feature_names == gbm.feature_name()
    # test feature_names with whitespaces
1429
    feature_names_with_space = [f"f {i}" for i in range(X_train.shape[-1])]
1430
1431
    lgb_train.set_feature_name(feature_names_with_space)
    gbm = lgb.train(params, lgb_train, num_boost_round=5)
1432
1433
1434
1435
1436
1437
1438
    assert feature_names == gbm.feature_name()


def test_feature_name_with_non_ascii():
    X_train = np.random.normal(size=(100, 4))
    y_train = np.random.random(100)
    # This has non-ascii strings.
1439
1440
    feature_names = ["F_零", "F_一", "F_二", "F_三"]
    params = {"verbose": -1}
1441
    lgb_train = lgb.Dataset(X_train, y_train, feature_name=feature_names)
1442

1443
    gbm = lgb.train(params, lgb_train, num_boost_round=5)
1444
    assert feature_names == gbm.feature_name()
1445
    gbm.save_model("lgb.model")
1446

1447
    gbm2 = lgb.Booster(model_file="lgb.model")
1448
1449
1450
    assert feature_names == gbm2.feature_name()


1451
def test_parameters_are_loaded_from_model_file(tmp_path, capsys):
1452
1453
1454
1455
    X = np.hstack([np.random.rand(100, 1), np.random.randint(0, 5, (100, 2))])
    y = np.random.rand(100)
    ds = lgb.Dataset(X, y)
    params = {
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
        "bagging_fraction": 0.8,
        "bagging_freq": 2,
        "boosting": "rf",
        "feature_contri": [0.5, 0.5, 0.5],
        "feature_fraction": 0.7,
        "boost_from_average": False,
        "interaction_constraints": [[0, 1], [0]],
        "metric": ["l2", "rmse"],
        "num_leaves": 5,
        "num_threads": 1,
1466
    }
1467
    model_file = tmp_path / "model.txt"
1468
1469
    orig_bst = lgb.train(params, ds, num_boost_round=1, categorical_feature=[1, 2])
    orig_bst.save_model(model_file)
1470
    with model_file.open("rt") as f:
1471
        model_contents = f.readlines()
1472
1473
1474
    params_start = model_contents.index("parameters:\n")
    model_contents.insert(params_start + 1, "[max_conflict_rate: 0]\n")
    with model_file.open("wt") as f:
1475
        f.writelines(model_contents)
1476
    bst = lgb.Booster(model_file=model_file)
1477
1478
1479
    expected_msg = "[LightGBM] [Warning] Ignoring unrecognized parameter 'max_conflict_rate' found in model string."
    stdout = capsys.readouterr().out
    assert expected_msg in stdout
1480
1481
    set_params = {k: bst.params[k] for k in params.keys()}
    assert set_params == params
1482
    assert bst.params["categorical_feature"] == [1, 2]
1483
1484

    # check that passing parameters to the constructor raises warning and ignores them
1485
1486
    with pytest.warns(UserWarning, match="Ignoring params argument"):
        bst2 = lgb.Booster(params={"num_leaves": 7}, model_file=model_file)
1487
1488
    assert bst.params == bst2.params

1489
1490
1491
1492
1493
    # check inference isn't affected by unknown parameter
    orig_preds = orig_bst.predict(X)
    preds = bst.predict(X)
    np.testing.assert_allclose(preds, orig_preds)

1494

1495
1496
def test_save_load_copy_pickle():
    def train_and_predict(init_model=None, return_model=False):
1497
        X, y = make_synthetic_regression()
1498
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
1499
        params = {"objective": "regression", "metric": "l2", "verbose": -1}
1500
        lgb_train = lgb.Dataset(X_train, y_train)
1501
1502
1503
1504
1505
1506
        gbm_template = lgb.train(params, lgb_train, num_boost_round=10, init_model=init_model)
        return gbm_template if return_model else mean_squared_error(y_test, gbm_template.predict(X_test))

    gbm = train_and_predict(return_model=True)
    ret_origin = train_and_predict(init_model=gbm)
    other_ret = []
1507
1508
    gbm.save_model("lgb.model")
    with open("lgb.model") as f:  # check all params are logged into model file correctly
1509
        assert f.read().find("[num_iterations: 10]") != -1
1510
1511
    other_ret.append(train_and_predict(init_model="lgb.model"))
    gbm_load = lgb.Booster(model_file="lgb.model")
1512
1513
1514
    other_ret.append(train_and_predict(init_model=gbm_load))
    other_ret.append(train_and_predict(init_model=copy.copy(gbm)))
    other_ret.append(train_and_predict(init_model=copy.deepcopy(gbm)))
1515
    with open("lgb.pkl", "wb") as f:
1516
        pickle.dump(gbm, f)
1517
    with open("lgb.pkl", "rb") as f:
1518
1519
1520
1521
1522
1523
1524
1525
        gbm_pickle = pickle.load(f)
    other_ret.append(train_and_predict(init_model=gbm_pickle))
    gbm_pickles = pickle.loads(pickle.dumps(gbm))
    other_ret.append(train_and_predict(init_model=gbm_pickles))
    for ret in other_ret:
        assert ret_origin == pytest.approx(ret)


1526
1527
1528
def test_all_expected_params_are_written_out_to_model_text(tmp_path):
    X, y = make_synthetic_regression()
    params = {
1529
1530
1531
1532
1533
1534
        "objective": "mape",
        "metric": ["l2", "mae"],
        "seed": 708,
        "data_sample_strategy": "bagging",
        "sub_row": 0.8234,
        "verbose": -1,
1535
1536
    }
    dtrain = lgb.Dataset(data=X, label=y)
1537
    gbm = lgb.train(params=params, train_set=dtrain, num_boost_round=3)
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582

    model_txt_from_memory = gbm.model_to_string()
    model_file = tmp_path / "out.model"
    gbm.save_model(filename=model_file)
    with open(model_file, "r") as f:
        model_txt_from_file = f.read()

    assert model_txt_from_memory == model_txt_from_file

    # entries whose values should reflect params passed to lgb.train()
    non_default_param_entries = [
        "[objective: mape]",
        # 'l1' was passed in with alias 'mae'
        "[metric: l2,l1]",
        "[data_sample_strategy: bagging]",
        "[seed: 708]",
        # NOTE: this was passed in with alias 'sub_row'
        "[bagging_fraction: 0.8234]",
        "[num_iterations: 3]",
    ]

    # entries with default values of params
    default_param_entries = [
        "[boosting: gbdt]",
        "[tree_learner: serial]",
        "[data: ]",
        "[valid: ]",
        "[learning_rate: 0.1]",
        "[num_leaves: 31]",
        "[num_threads: 0]",
        "[deterministic: 0]",
        "[histogram_pool_size: -1]",
        "[max_depth: -1]",
        "[min_data_in_leaf: 20]",
        "[min_sum_hessian_in_leaf: 0.001]",
        "[pos_bagging_fraction: 1]",
        "[neg_bagging_fraction: 1]",
        "[bagging_freq: 0]",
        "[bagging_seed: 15415]",
        "[feature_fraction: 1]",
        "[feature_fraction_bynode: 1]",
        "[feature_fraction_seed: 32671]",
        "[extra_trees: 0]",
        "[extra_seed: 6642]",
        "[early_stopping_round: 0]",
1583
        "[early_stopping_min_delta: 0]",
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
        "[first_metric_only: 0]",
        "[max_delta_step: 0]",
        "[lambda_l1: 0]",
        "[lambda_l2: 0]",
        "[linear_lambda: 0]",
        "[min_gain_to_split: 0]",
        "[drop_rate: 0.1]",
        "[max_drop: 50]",
        "[skip_drop: 0.5]",
        "[xgboost_dart_mode: 0]",
        "[uniform_drop: 0]",
        "[drop_seed: 20623]",
        "[top_rate: 0.2]",
        "[other_rate: 0.1]",
        "[min_data_per_group: 100]",
        "[max_cat_threshold: 32]",
        "[cat_l2: 10]",
        "[cat_smooth: 10]",
        "[max_cat_to_onehot: 4]",
        "[top_k: 20]",
        "[monotone_constraints: ]",
        "[monotone_constraints_method: basic]",
        "[monotone_penalty: 0]",
        "[feature_contri: ]",
        "[forcedsplits_filename: ]",
        "[refit_decay_rate: 0.9]",
        "[cegb_tradeoff: 1]",
        "[cegb_penalty_split: 0]",
        "[cegb_penalty_feature_lazy: ]",
        "[cegb_penalty_feature_coupled: ]",
        "[path_smooth: 0]",
        "[interaction_constraints: ]",
        "[verbosity: -1]",
        "[saved_feature_importance_type: 0]",
        "[use_quantized_grad: 0]",
        "[num_grad_quant_bins: 4]",
        "[quant_train_renew_leaf: 0]",
        "[stochastic_rounding: 1]",
        "[linear_tree: 0]",
        "[max_bin: 255]",
        "[max_bin_by_feature: ]",
        "[min_data_in_bin: 3]",
        "[bin_construct_sample_cnt: 200000]",
        "[data_random_seed: 2350]",
        "[is_enable_sparse: 1]",
        "[enable_bundle: 1]",
        "[use_missing: 1]",
        "[zero_as_missing: 0]",
        "[feature_pre_filter: 1]",
        "[pre_partition: 0]",
        "[two_round: 0]",
        "[header: 0]",
        "[label_column: ]",
        "[weight_column: ]",
        "[group_column: ]",
        "[ignore_column: ]",
        "[categorical_feature: ]",
        "[forcedbins_filename: ]",
        "[precise_float_parser: 0]",
        "[parser_config_file: ]",
        "[objective_seed: 4309]",
        "[num_class: 1]",
        "[is_unbalance: 0]",
        "[scale_pos_weight: 1]",
        "[sigmoid: 1]",
        "[boost_from_average: 1]",
        "[reg_sqrt: 0]",
        "[alpha: 0.9]",
        "[fair_c: 1]",
        "[poisson_max_delta_step: 0.7]",
        "[tweedie_variance_power: 1.5]",
        "[lambdarank_truncation_level: 30]",
        "[lambdarank_norm: 1]",
        "[label_gain: ]",
        "[lambdarank_position_bias_regularization: 0]",
        "[eval_at: ]",
        "[multi_error_top_k: 1]",
        "[auc_mu_weights: ]",
        "[num_machines: 1]",
        "[local_listen_port: 12400]",
        "[time_out: 120]",
        "[machine_list_filename: ]",
        "[machines: ]",
        "[gpu_platform_id: -1]",
        "[gpu_device_id: -1]",
        "[num_gpu: 1]",
    ]
    all_param_entries = non_default_param_entries + default_param_entries

    # add device-specific entries
    #
    # passed-in force_col_wise / force_row_wise parameters are ignored on CUDA and GPU builds...
    # https://github.com/microsoft/LightGBM/blob/1d7ee63686272bceffd522284127573b511df6be/src/io/config.cpp#L375-L377
1677
1678
1679
1680
    if getenv("TASK", "") == "cuda":
        device_entries = ["[force_col_wise: 0]", "[force_row_wise: 1]", "[device_type: cuda]", "[gpu_use_dp: 1]"]
    elif getenv("TASK", "") == "gpu":
        device_entries = ["[force_col_wise: 1]", "[force_row_wise: 0]", "[device_type: gpu]", "[gpu_use_dp: 0]"]
1681
    else:
1682
        device_entries = ["[force_col_wise: 0]", "[force_row_wise: 0]", "[device_type: cpu]", "[gpu_use_dp: 0]"]
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704

    all_param_entries += device_entries

    # check that model text has all expected param entries
    for param_str in all_param_entries:
        assert param_str in model_txt_from_file
        assert param_str in model_txt_from_memory

    # since Booster.model_to_string() is used when pickling, check that parameters all
    # roundtrip pickling successfully too
    gbm_pkl = pickle_and_unpickle_object(gbm, serializer="joblib")
    model_txt_from_memory = gbm_pkl.model_to_string()
    model_file = tmp_path / "out-pkl.model"
    gbm_pkl.save_model(filename=model_file)
    with open(model_file, "r") as f:
        model_txt_from_file = f.read()

    for param_str in all_param_entries:
        assert param_str in model_txt_from_file
        assert param_str in model_txt_from_memory


1705
1706
1707
def test_pandas_categorical():
    pd = pytest.importorskip("pandas")
    np.random.seed(42)  # sometimes there is no difference how cols are treated (cat or not cat)
1708
1709
1710
1711
1712
1713
1714
1715
1716
    X = pd.DataFrame(
        {
            "A": np.random.permutation(["a", "b", "c", "d"] * 75),  # str
            "B": np.random.permutation([1, 2, 3] * 100),  # int
            "C": np.random.permutation([0.1, 0.2, -0.1, -0.1, 0.2] * 60),  # float
            "D": np.random.permutation([True, False] * 150),  # bool
            "E": pd.Categorical(np.random.permutation(["z", "y", "x", "w", "v"] * 60), ordered=True),
        }
    )  # str and ordered categorical
1717
    y = np.random.permutation([0, 1] * 150)
1718
1719
1720
1721
1722
1723
1724
1725
1726
    X_test = pd.DataFrame(
        {
            "A": np.random.permutation(["a", "b", "e"] * 20),  # unseen category
            "B": np.random.permutation([1, 3] * 30),
            "C": np.random.permutation([0.1, -0.1, 0.2, 0.2] * 15),
            "D": np.random.permutation([True, False] * 30),
            "E": pd.Categorical(np.random.permutation(["z", "y"] * 30), ordered=True),
        }
    )
1727
1728
1729
    np.random.seed()  # reset seed
    cat_cols_actual = ["A", "B", "C", "D"]
    cat_cols_to_store = cat_cols_actual + ["E"]
1730
1731
    X[cat_cols_actual] = X[cat_cols_actual].astype("category")
    X_test[cat_cols_actual] = X_test[cat_cols_actual].astype("category")
1732
    cat_values = [X[col].cat.categories.tolist() for col in cat_cols_to_store]
1733
    params = {"objective": "binary", "metric": "binary_logloss", "verbose": -1}
1734
1735
1736
    lgb_train = lgb.Dataset(X, y)
    gbm0 = lgb.train(params, lgb_train, num_boost_round=10)
    pred0 = gbm0.predict(X_test)
1737
    assert lgb_train.categorical_feature == "auto"
1738
1739
1740
1741
1742
    lgb_train = lgb.Dataset(X, pd.DataFrame(y))  # also test that label can be one-column pd.DataFrame
    gbm1 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=[0])
    pred1 = gbm1.predict(X_test)
    assert lgb_train.categorical_feature == [0]
    lgb_train = lgb.Dataset(X, pd.Series(y))  # also test that label can be pd.Series
1743
    gbm2 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=["A"])
1744
    pred2 = gbm2.predict(X_test)
1745
    assert lgb_train.categorical_feature == ["A"]
1746
    lgb_train = lgb.Dataset(X, y)
1747
    gbm3 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=["A", "B", "C", "D"])
1748
    pred3 = gbm3.predict(X_test)
1749
1750
1751
    assert lgb_train.categorical_feature == ["A", "B", "C", "D"]
    gbm3.save_model("categorical.model")
    gbm4 = lgb.Booster(model_file="categorical.model")
1752
1753
    pred4 = gbm4.predict(X_test)
    model_str = gbm4.model_to_string()
1754
    gbm4.model_from_string(model_str)
1755
1756
1757
1758
    pred5 = gbm4.predict(X_test)
    gbm5 = lgb.Booster(model_str=model_str)
    pred6 = gbm5.predict(X_test)
    lgb_train = lgb.Dataset(X, y)
1759
    gbm6 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=["A", "B", "C", "D", "E"])
1760
    pred7 = gbm6.predict(X_test)
1761
    assert lgb_train.categorical_feature == ["A", "B", "C", "D", "E"]
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
    lgb_train = lgb.Dataset(X, y)
    gbm7 = lgb.train(params, lgb_train, num_boost_round=10, categorical_feature=[])
    pred8 = gbm7.predict(X_test)
    assert lgb_train.categorical_feature == []
    with pytest.raises(AssertionError):
        np.testing.assert_allclose(pred0, pred1)
    with pytest.raises(AssertionError):
        np.testing.assert_allclose(pred0, pred2)
    np.testing.assert_allclose(pred1, pred2)
    np.testing.assert_allclose(pred0, pred3)
    np.testing.assert_allclose(pred0, pred4)
    np.testing.assert_allclose(pred0, pred5)
    np.testing.assert_allclose(pred0, pred6)
    with pytest.raises(AssertionError):
        np.testing.assert_allclose(pred0, pred7)  # ordered cat features aren't treated as cat features by default
    with pytest.raises(AssertionError):
1778
        np.testing.assert_allclose(pred0, pred8)
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
    assert gbm0.pandas_categorical == cat_values
    assert gbm1.pandas_categorical == cat_values
    assert gbm2.pandas_categorical == cat_values
    assert gbm3.pandas_categorical == cat_values
    assert gbm4.pandas_categorical == cat_values
    assert gbm5.pandas_categorical == cat_values
    assert gbm6.pandas_categorical == cat_values
    assert gbm7.pandas_categorical == cat_values


def test_pandas_sparse():
    pd = pytest.importorskip("pandas")
1791
1792
1793
1794
1795
1796
1797
    X = pd.DataFrame(
        {
            "A": pd.arrays.SparseArray(np.random.permutation([0, 1, 2] * 100)),
            "B": pd.arrays.SparseArray(np.random.permutation([0.0, 0.1, 0.2, -0.1, 0.2] * 60)),
            "C": pd.arrays.SparseArray(np.random.permutation([True, False] * 150)),
        }
    )
1798
    y = pd.Series(pd.arrays.SparseArray(np.random.permutation([0, 1] * 150)))
1799
1800
1801
1802
1803
1804
1805
    X_test = pd.DataFrame(
        {
            "A": pd.arrays.SparseArray(np.random.permutation([0, 2] * 30)),
            "B": pd.arrays.SparseArray(np.random.permutation([0.0, 0.1, 0.2, -0.1] * 15)),
            "C": pd.arrays.SparseArray(np.random.permutation([True, False] * 30)),
        }
    )
1806
1807
    for dtype in pd.concat([X.dtypes, X_test.dtypes, pd.Series(y.dtypes)]):
        assert pd.api.types.is_sparse(dtype)
1808
    params = {"objective": "binary", "verbose": -1}
1809
1810
1811
    lgb_train = lgb.Dataset(X, y)
    gbm = lgb.train(params, lgb_train, num_boost_round=10)
    pred_sparse = gbm.predict(X_test, raw_score=True)
1812
    if hasattr(X_test, "sparse"):
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
        pred_dense = gbm.predict(X_test.sparse.to_dense(), raw_score=True)
    else:
        pred_dense = gbm.predict(X_test.to_dense(), raw_score=True)
    np.testing.assert_allclose(pred_sparse, pred_dense)


def test_reference_chain():
    X = np.random.normal(size=(100, 2))
    y = np.random.normal(size=100)
    tmp_dat = lgb.Dataset(X, y)
    # take subsets and train
    tmp_dat_train = tmp_dat.subset(np.arange(80))
    tmp_dat_val = tmp_dat.subset(np.arange(80, 100)).subset(np.arange(18))
1826
    params = {"objective": "regression_l2", "metric": "rmse"}
1827
    evals_result = {}
1828
1829
1830
1831
1832
    lgb.train(
        params,
        tmp_dat_train,
        num_boost_round=20,
        valid_sets=[tmp_dat_train, tmp_dat_val],
1833
        callbacks=[lgb.record_evaluation(evals_result)],
1834
    )
1835
1836
    assert len(evals_result["training"]["rmse"]) == 20
    assert len(evals_result["valid_1"]["rmse"]) == 20
1837
1838
1839
1840
1841
1842


def test_contribs():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1843
1844
1845
        "objective": "binary",
        "metric": "binary_logloss",
        "verbose": -1,
1846
1847
1848
1849
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)

1850
1851
1852
1853
    assert (
        np.linalg.norm(gbm.predict(X_test, raw_score=True) - np.sum(gbm.predict(X_test, pred_contrib=True), axis=1))
        < 1e-4
    )
1854
1855
1856
1857
1858
1859


def test_contribs_sparse():
    n_features = 20
    n_samples = 100
    # generate CSR sparse dataset
1860
1861
1862
    X, y = make_multilabel_classification(
        n_samples=n_samples, sparse=True, n_features=n_features, n_classes=1, n_labels=2
    )
1863
1864
1865
    y = y.flatten()
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1866
1867
        "objective": "binary",
        "verbose": -1,
1868
1869
1870
1871
1872
1873
1874
1875
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)
    contribs_csr = gbm.predict(X_test, pred_contrib=True)
    assert isspmatrix_csr(contribs_csr)
    # convert data to dense and get back same contribs
    contribs_dense = gbm.predict(X_test.toarray(), pred_contrib=True)
    # validate the values are the same
1876
    if platform.machine() == "aarch64":
1877
1878
1879
        np.testing.assert_allclose(contribs_csr.toarray(), contribs_dense, rtol=1, atol=1e-12)
    else:
        np.testing.assert_allclose(contribs_csr.toarray(), contribs_dense)
1880
    assert np.linalg.norm(gbm.predict(X_test, raw_score=True) - np.sum(contribs_dense, axis=1)) < 1e-4
1881
1882
1883
1884
1885
    # validate using CSC matrix
    X_test_csc = X_test.tocsc()
    contribs_csc = gbm.predict(X_test_csc, pred_contrib=True)
    assert isspmatrix_csc(contribs_csc)
    # validate the values are the same
1886
    if platform.machine() == "aarch64":
1887
1888
1889
        np.testing.assert_allclose(contribs_csc.toarray(), contribs_dense, rtol=1, atol=1e-12)
    else:
        np.testing.assert_allclose(contribs_csc.toarray(), contribs_dense)
1890
1891
1892
1893
1894
1895
1896


def test_contribs_sparse_multiclass():
    n_features = 20
    n_samples = 100
    n_labels = 4
    # generate CSR sparse dataset
1897
1898
1899
    X, y = make_multilabel_classification(
        n_samples=n_samples, sparse=True, n_features=n_features, n_classes=1, n_labels=n_labels
    )
1900
1901
1902
    y = y.flatten()
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
1903
1904
1905
        "objective": "multiclass",
        "num_class": n_labels,
        "verbose": -1,
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)
    contribs_csr = gbm.predict(X_test, pred_contrib=True)
    assert isinstance(contribs_csr, list)
    for perclass_contribs_csr in contribs_csr:
        assert isspmatrix_csr(perclass_contribs_csr)
    # convert data to dense and get back same contribs
    contribs_dense = gbm.predict(X_test.toarray(), pred_contrib=True)
    # validate the values are the same
1916
    contribs_csr_array = np.swapaxes(np.array([sparse_array.toarray() for sparse_array in contribs_csr]), 0, 1)
1917
1918
1919
1920
    contribs_csr_arr_re = contribs_csr_array.reshape(
        (contribs_csr_array.shape[0], contribs_csr_array.shape[1] * contribs_csr_array.shape[2])
    )
    if platform.machine() == "aarch64":
1921
1922
1923
        np.testing.assert_allclose(contribs_csr_arr_re, contribs_dense, rtol=1, atol=1e-12)
    else:
        np.testing.assert_allclose(contribs_csr_arr_re, contribs_dense)
1924
1925
1926
1927
1928
1929
1930
1931
1932
    contribs_dense_re = contribs_dense.reshape(contribs_csr_array.shape)
    assert np.linalg.norm(gbm.predict(X_test, raw_score=True) - np.sum(contribs_dense_re, axis=2)) < 1e-4
    # validate using CSC matrix
    X_test_csc = X_test.tocsc()
    contribs_csc = gbm.predict(X_test_csc, pred_contrib=True)
    assert isinstance(contribs_csc, list)
    for perclass_contribs_csc in contribs_csc:
        assert isspmatrix_csc(perclass_contribs_csc)
    # validate the values are the same
1933
    contribs_csc_array = np.swapaxes(np.array([sparse_array.toarray() for sparse_array in contribs_csc]), 0, 1)
1934
1935
1936
1937
    contribs_csc_array = contribs_csc_array.reshape(
        (contribs_csc_array.shape[0], contribs_csc_array.shape[1] * contribs_csc_array.shape[2])
    )
    if platform.machine() == "aarch64":
1938
1939
1940
        np.testing.assert_allclose(contribs_csc_array, contribs_dense, rtol=1, atol=1e-12)
    else:
        np.testing.assert_allclose(contribs_csc_array, contribs_dense)
1941
1942


1943
@pytest.mark.skipif(psutil.virtual_memory().available / 1024 / 1024 / 1024 < 3, reason="not enough RAM")
1944
def test_int32_max_sparse_contribs():
1945
    params = {"objective": "binary"}
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
    train_features = np.random.rand(100, 1000)
    train_targets = [0] * 50 + [1] * 50
    lgb_train = lgb.Dataset(train_features, train_targets)
    gbm = lgb.train(params, lgb_train, num_boost_round=2)
    csr_input_shape = (3000000, 1000)
    test_features = csr_matrix(csr_input_shape)
    for i in range(0, csr_input_shape[0], csr_input_shape[0] // 6):
        for j in range(0, 1000, 100):
            test_features[i, j] = random.random()
    y_pred_csr = gbm.predict(test_features, pred_contrib=True)
    # Note there is an extra column added to the output for the expected value
    csr_output_shape = (csr_input_shape[0], csr_input_shape[1] + 1)
    assert y_pred_csr.shape == csr_output_shape
    y_pred_csc = gbm.predict(test_features.tocsc(), pred_contrib=True)
    # Note output CSC shape should be same as CSR output shape
    assert y_pred_csc.shape == csr_output_shape


def test_sliced_data():
    def train_and_get_predictions(features, labels):
        dataset = lgb.Dataset(features, label=labels)
        lgb_params = {
1968
1969
1970
            "application": "binary",
            "verbose": -1,
            "min_data": 5,
1971
        }
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
        gbm = lgb.train(
            params=lgb_params,
            train_set=dataset,
            num_boost_round=10,
        )
        return gbm.predict(features)

    num_samples = 100
    features = np.random.rand(num_samples, 5)
    positive_samples = int(num_samples * 0.25)
1982
1983
1984
    labels = np.append(
        np.ones(positive_samples, dtype=np.float32), np.zeros(num_samples - positive_samples, dtype=np.float32)
    )
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
    # test sliced labels
    origin_pred = train_and_get_predictions(features, labels)
    stacked_labels = np.column_stack((labels, np.ones(num_samples, dtype=np.float32)))
    sliced_labels = stacked_labels[:, 0]
    sliced_pred = train_and_get_predictions(features, sliced_labels)
    np.testing.assert_allclose(origin_pred, sliced_pred)
    # append some columns
    stacked_features = np.column_stack((np.ones(num_samples, dtype=np.float32), features))
    stacked_features = np.column_stack((np.ones(num_samples, dtype=np.float32), stacked_features))
    stacked_features = np.column_stack((stacked_features, np.ones(num_samples, dtype=np.float32)))
    stacked_features = np.column_stack((stacked_features, np.ones(num_samples, dtype=np.float32)))
    # append some rows
    stacked_features = np.concatenate((np.ones(9, dtype=np.float32).reshape((1, 9)), stacked_features), axis=0)
    stacked_features = np.concatenate((np.ones(9, dtype=np.float32).reshape((1, 9)), stacked_features), axis=0)
    stacked_features = np.concatenate((stacked_features, np.ones(9, dtype=np.float32).reshape((1, 9))), axis=0)
    stacked_features = np.concatenate((stacked_features, np.ones(9, dtype=np.float32).reshape((1, 9))), axis=0)
    # test sliced 2d matrix
    sliced_features = stacked_features[2:102, 2:7]
    assert np.all(sliced_features == features)
    sliced_pred = train_and_get_predictions(sliced_features, sliced_labels)
    np.testing.assert_allclose(origin_pred, sliced_pred)
    # test sliced CSR
    stacked_csr = csr_matrix(stacked_features)
    sliced_csr = stacked_csr[2:102, 2:7]
    assert np.all(sliced_csr == features)
    sliced_pred = train_and_get_predictions(sliced_csr, sliced_labels)
    np.testing.assert_allclose(origin_pred, sliced_pred)


def test_init_with_subset():
    data = np.random.random((50, 2))
    y = [1] * 25 + [0] * 25
    lgb_train = lgb.Dataset(data, y, free_raw_data=False)
    subset_index_1 = np.random.choice(np.arange(50), 30, replace=False)
    subset_data_1 = lgb_train.subset(subset_index_1)
    subset_index_2 = np.random.choice(np.arange(50), 20, replace=False)
    subset_data_2 = lgb_train.subset(subset_index_2)
2022
2023
2024
    params = {"objective": "binary", "verbose": -1}
    init_gbm = lgb.train(params=params, train_set=subset_data_1, num_boost_round=10, keep_training_booster=True)
    lgb.train(params=params, train_set=subset_data_2, num_boost_round=10, init_model=init_gbm)
2025
2026
2027
2028
    assert lgb_train.get_data().shape[0] == 50
    assert subset_data_1.get_data().shape[0] == 30
    assert subset_data_2.get_data().shape[0] == 20
    lgb_train.save_binary("lgb_train_data.bin")
2029
    lgb_train_from_file = lgb.Dataset("lgb_train_data.bin", free_raw_data=False)
2030
2031
    subset_data_3 = lgb_train_from_file.subset(subset_index_1)
    subset_data_4 = lgb_train_from_file.subset(subset_index_2)
2032
    init_gbm_2 = lgb.train(params=params, train_set=subset_data_3, num_boost_round=10, keep_training_booster=True)
2033
    with np.testing.assert_raises_regex(lgb.basic.LightGBMError, "Unknown format of training data"):
2034
        lgb.train(params=params, train_set=subset_data_4, num_boost_round=10, init_model=init_gbm_2)
2035
2036
2037
2038
2039
    assert lgb_train_from_file.get_data() == "lgb_train_data.bin"
    assert subset_data_3.get_data() == "lgb_train_data.bin"
    assert subset_data_4.get_data() == "lgb_train_data.bin"


2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
def test_training_on_constructed_subset_without_params():
    X = np.random.random((100, 10))
    y = np.random.random(100)
    lgb_data = lgb.Dataset(X, y)
    subset_indices = [1, 2, 3, 4]
    subset = lgb_data.subset(subset_indices).construct()
    bst = lgb.train({}, subset, num_boost_round=1)
    assert subset.get_params() == {}
    assert subset.num_data() == len(subset_indices)
    assert bst.current_iteration() == 1


2052
2053
2054
2055
2056
2057
def generate_trainset_for_monotone_constraints_tests(x3_to_category=True):
    number_of_dpoints = 3000
    x1_positively_correlated_with_y = np.random.random(size=number_of_dpoints)
    x2_negatively_correlated_with_y = np.random.random(size=number_of_dpoints)
    x3_negatively_correlated_with_y = np.random.random(size=number_of_dpoints)
    x = np.column_stack(
2058
2059
        (
            x1_positively_correlated_with_y,
2060
            x2_negatively_correlated_with_y,
2061
2062
2063
            categorize(x3_negatively_correlated_with_y) if x3_to_category else x3_negatively_correlated_with_y,
        )
    )
2064
2065

    zs = np.random.normal(loc=0.0, scale=0.01, size=number_of_dpoints)
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
    scales = 10.0 * (np.random.random(6) + 0.5)
    y = (
        scales[0] * x1_positively_correlated_with_y
        + np.sin(scales[1] * np.pi * x1_positively_correlated_with_y)
        - scales[2] * x2_negatively_correlated_with_y
        - np.cos(scales[3] * np.pi * x2_negatively_correlated_with_y)
        - scales[4] * x3_negatively_correlated_with_y
        - np.cos(scales[5] * np.pi * x3_negatively_correlated_with_y)
        + zs
    )
2076
2077
2078
    categorical_features = []
    if x3_to_category:
        categorical_features = [2]
2079
    return lgb.Dataset(x, label=y, categorical_feature=categorical_features, free_raw_data=False)
2080
2081


2082
@pytest.mark.skipif(getenv("TASK", "") == "cuda", reason="Monotone constraints are not yet supported by CUDA version")
2083
2084
@pytest.mark.parametrize("test_with_categorical_variable", [True, False])
def test_monotone_constraints(test_with_categorical_variable):
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
    def is_increasing(y):
        return (np.diff(y) >= 0.0).all()

    def is_decreasing(y):
        return (np.diff(y) <= 0.0).all()

    def is_non_monotone(y):
        return (np.diff(y) < 0.0).any() and (np.diff(y) > 0.0).any()

    def is_correctly_constrained(learner, x3_to_category=True):
        iterations = 10
        n = 1000
        variable_x = np.linspace(0, 1, n).reshape((n, 1))
        fixed_xs_values = np.linspace(0, 1, n)
        for i in range(iterations):
            fixed_x = fixed_xs_values[i] * np.ones((n, 1))
            monotonically_increasing_x = np.column_stack((variable_x, fixed_x, fixed_x))
            monotonically_increasing_y = learner.predict(monotonically_increasing_x)
            monotonically_decreasing_x = np.column_stack((fixed_x, variable_x, fixed_x))
            monotonically_decreasing_y = learner.predict(monotonically_decreasing_x)
2105
2106
2107
2108
2109
2110
2111
            non_monotone_x = np.column_stack(
                (
                    fixed_x,
                    fixed_x,
                    categorize(variable_x) if x3_to_category else variable_x,
                )
            )
2112
            non_monotone_y = learner.predict(non_monotone_x)
2113
2114
2115
2116
2117
            if not (
                is_increasing(monotonically_increasing_y)
                and is_decreasing(monotonically_decreasing_y)
                and is_non_monotone(non_monotone_y)
            ):
2118
                return False
2119
        return True
2120

2121
2122
2123
2124
2125
2126
2127
2128
    def are_interactions_enforced(gbm, feature_sets):
        def parse_tree_features(gbm):
            # trees start at position 1.
            tree_str = gbm.model_to_string().split("Tree")[1:]
            feature_sets = []
            for tree in tree_str:
                # split_features are in 4th line.
                features = tree.splitlines()[3].split("=")[1].split(" ")
2129
                features = {f"Column_{f}" for f in features}
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
                feature_sets.append(features)
            return np.array(feature_sets)

        def has_interaction(treef):
            n = 0
            for fs in feature_sets:
                if len(treef.intersection(fs)) > 0:
                    n += 1
            return n > 1

        tree_features = parse_tree_features(gbm)
2141
        has_interaction_flag = np.array([has_interaction(treef) for treef in tree_features])
2142
2143
2144

        return not has_interaction_flag.any()

2145
    trainset = generate_trainset_for_monotone_constraints_tests(test_with_categorical_variable)
2146
    for test_with_interaction_constraints in [True, False]:
2147
2148
2149
2150
        error_msg = (
            "Model not correctly constrained "
            f"(test_with_interaction_constraints={test_with_interaction_constraints})"
        )
2151
        for monotone_constraints_method in ["basic", "intermediate", "advanced"]:
2152
            params = {
2153
2154
2155
                "min_data": 20,
                "num_leaves": 20,
                "monotone_constraints": [1, -1, 0],
2156
                "monotone_constraints_method": monotone_constraints_method,
2157
                "use_missing": False,
2158
            }
2159
2160
            if test_with_interaction_constraints:
                params["interaction_constraints"] = [[0], [1], [2]]
2161
            constrained_model = lgb.train(params, trainset)
2162
            assert is_correctly_constrained(constrained_model, test_with_categorical_variable), error_msg
2163
2164
2165
            if test_with_interaction_constraints:
                feature_sets = [["Column_0"], ["Column_1"], "Column_2"]
                assert are_interactions_enforced(constrained_model, feature_sets)
2166
2167


2168
@pytest.mark.skipif(getenv("TASK", "") == "cuda", reason="Monotone constraints are not yet supported by CUDA version")
2169
2170
2171
2172
2173
2174
2175
2176
def test_monotone_penalty():
    def are_first_splits_non_monotone(tree, n, monotone_constraints):
        if n <= 0:
            return True
        if "leaf_value" in tree:
            return True
        if monotone_constraints[tree["split_feature"]] != 0:
            return False
2177
2178
2179
        return are_first_splits_non_monotone(
            tree["left_child"], n - 1, monotone_constraints
        ) and are_first_splits_non_monotone(tree["right_child"], n - 1, monotone_constraints)
2180
2181
2182
2183
2184
2185

    def are_there_monotone_splits(tree, monotone_constraints):
        if "leaf_value" in tree:
            return False
        if monotone_constraints[tree["split_feature"]] != 0:
            return True
2186
2187
2188
        return are_there_monotone_splits(tree["left_child"], monotone_constraints) or are_there_monotone_splits(
            tree["right_child"], monotone_constraints
        )
2189
2190
2191
2192
2193
2194

    max_depth = 5
    monotone_constraints = [1, -1, 0]
    penalization_parameter = 2.0
    trainset = generate_trainset_for_monotone_constraints_tests(x3_to_category=False)
    for monotone_constraints_method in ["basic", "intermediate", "advanced"]:
2195
        params = {
2196
2197
2198
            "max_depth": max_depth,
            "monotone_constraints": monotone_constraints,
            "monotone_penalty": penalization_parameter,
2199
            "monotone_constraints_method": monotone_constraints_method,
2200
        }
2201
2202
2203
        constrained_model = lgb.train(params, trainset, 10)
        dumped_model = constrained_model.dump_model()["tree_info"]
        for tree in dumped_model:
2204
2205
2206
            assert are_first_splits_non_monotone(
                tree["tree_structure"], int(penalization_parameter), monotone_constraints
            )
2207
2208
2209
2210
            assert are_there_monotone_splits(tree["tree_structure"], monotone_constraints)


# test if a penalty as high as the depth indeed prohibits all monotone splits
2211
@pytest.mark.skipif(getenv("TASK", "") == "cuda", reason="Monotone constraints are not yet supported by CUDA version")
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
def test_monotone_penalty_max():
    max_depth = 5
    monotone_constraints = [1, -1, 0]
    penalization_parameter = max_depth
    trainset_constrained_model = generate_trainset_for_monotone_constraints_tests(x3_to_category=False)
    x = trainset_constrained_model.data
    y = trainset_constrained_model.label
    x3_negatively_correlated_with_y = x[:, 2]
    trainset_unconstrained_model = lgb.Dataset(x3_negatively_correlated_with_y.reshape(-1, 1), label=y)
    params_constrained_model = {
2222
2223
        "monotone_constraints": monotone_constraints,
        "monotone_penalty": penalization_parameter,
2224
2225
2226
2227
2228
2229
2230
2231
2232
        "max_depth": max_depth,
        "gpu_use_dp": True,
    }
    params_unconstrained_model = {
        "max_depth": max_depth,
        "gpu_use_dp": True,
    }

    unconstrained_model = lgb.train(params_unconstrained_model, trainset_unconstrained_model, 10)
2233
    unconstrained_model_predictions = unconstrained_model.predict(x3_negatively_correlated_with_y.reshape(-1, 1))
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250

    for monotone_constraints_method in ["basic", "intermediate", "advanced"]:
        params_constrained_model["monotone_constraints_method"] = monotone_constraints_method
        # The penalization is so high that the first 2 features should not be used here
        constrained_model = lgb.train(params_constrained_model, trainset_constrained_model, 10)

        # Check that a very high penalization is the same as not using the features at all
        np.testing.assert_array_equal(constrained_model.predict(x), unconstrained_model_predictions)


def test_max_bin_by_feature():
    col1 = np.arange(0, 100)[:, np.newaxis]
    col2 = np.zeros((100, 1))
    col2[20:] = 1
    X = np.concatenate([col1, col2], axis=1)
    y = np.arange(0, 100)
    params = {
2251
2252
2253
2254
2255
2256
2257
        "objective": "regression_l2",
        "verbose": -1,
        "num_leaves": 100,
        "min_data_in_leaf": 1,
        "min_sum_hessian_in_leaf": 0,
        "min_data_in_bin": 1,
        "max_bin_by_feature": [100, 2],
2258
2259
2260
2261
    }
    lgb_data = lgb.Dataset(X, label=y)
    est = lgb.train(params, lgb_data, num_boost_round=1)
    assert len(np.unique(est.predict(X))) == 100
2262
    params["max_bin_by_feature"] = [2, 100]
2263
2264
2265
2266
2267
2268
2269
2270
    lgb_data = lgb.Dataset(X, label=y)
    est = lgb.train(params, lgb_data, num_boost_round=1)
    assert len(np.unique(est.predict(X))) == 3


def test_small_max_bin():
    np.random.seed(0)
    y = np.random.choice([0, 1], 100)
2271
    x = np.ones((100, 1))
2272
2273
    x[:30, 0] = -1
    x[60:, 0] = 2
2274
    params = {"objective": "binary", "seed": 0, "min_data_in_leaf": 1, "verbose": -1, "max_bin": 2}
2275
2276
2277
    lgb_x = lgb.Dataset(x, label=y)
    lgb.train(params, lgb_x, num_boost_round=5)
    x[0, 0] = np.nan
2278
    params["max_bin"] = 3
2279
2280
2281
2282
2283
2284
2285
2286
    lgb_x = lgb.Dataset(x, label=y)
    lgb.train(params, lgb_x, num_boost_round=5)
    np.random.seed()  # reset seed


def test_refit():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
2287
    params = {"objective": "binary", "metric": "binary_logloss", "verbose": -1, "min_data": 10}
2288
2289
2290
2291
2292
2293
2294
2295
    lgb_train = lgb.Dataset(X_train, y_train)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)
    err_pred = log_loss(y_test, gbm.predict(X_test))
    new_gbm = gbm.refit(X_test, y_test)
    new_err_pred = log_loss(y_test, new_gbm.predict(X_test))
    assert err_pred > new_err_pred


2296
2297
2298
2299
def test_refit_dataset_params():
    # check refit accepts dataset_params
    X, y = load_breast_cancer(return_X_y=True)
    lgb_train = lgb.Dataset(X, y, init_score=np.zeros(y.size))
2300
    train_params = {"objective": "binary", "verbose": -1, "seed": 123}
2301
2302
2303
2304
    gbm = lgb.train(train_params, lgb_train, num_boost_round=10)
    non_weight_err_pred = log_loss(y, gbm.predict(X))
    refit_weight = np.random.rand(y.shape[0])
    dataset_params = {
2305
2306
2307
        "max_bin": 260,
        "min_data_in_bin": 5,
        "data_random_seed": 123,
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
    }
    new_gbm = gbm.refit(
        data=X,
        label=y,
        weight=refit_weight,
        dataset_params=dataset_params,
        decay_rate=0.0,
    )
    weight_err_pred = log_loss(y, new_gbm.predict(X))
    train_set_params = new_gbm.train_set.get_params()
    stored_weights = new_gbm.train_set.get_weight()
    assert weight_err_pred != non_weight_err_pred
    assert train_set_params["max_bin"] == 260
    assert train_set_params["min_data_in_bin"] == 5
    assert train_set_params["data_random_seed"] == 123
    np.testing.assert_allclose(stored_weights, refit_weight)


2326
@pytest.mark.parametrize("boosting_type", ["rf", "dart"])
2327
2328
2329
def test_mape_for_specific_boosting_types(boosting_type):
    X, y = make_synthetic_regression()
    y = abs(y)
2330
    params = {
2331
2332
2333
2334
2335
2336
2337
        "boosting_type": boosting_type,
        "objective": "mape",
        "verbose": -1,
        "bagging_freq": 1,
        "bagging_fraction": 0.8,
        "feature_fraction": 0.8,
        "boost_from_average": True,
2338
2339
2340
2341
2342
    }
    lgb_train = lgb.Dataset(X, y)
    gbm = lgb.train(params, lgb_train, num_boost_round=20)
    pred = gbm.predict(X)
    pred_mean = pred.mean()
2343
2344
2345
    # the following checks that dart and rf with mape can predict outside the 0-1 range
    # https://github.com/microsoft/LightGBM/issues/1579
    assert pred_mean > 8
2346
2347
2348
2349
2350
2351


def check_constant_features(y_true, expected_pred, more_params):
    X_train = np.ones((len(y_true), 1))
    y_train = np.array(y_true)
    params = {
2352
2353
2354
2355
2356
2357
2358
2359
        "objective": "regression",
        "num_class": 1,
        "verbose": -1,
        "min_data": 1,
        "num_leaves": 2,
        "learning_rate": 1,
        "min_data_in_bin": 1,
        "boost_from_average": True,
2360
2361
2362
2363
2364
2365
2366
2367
2368
    }
    params.update(more_params)
    lgb_train = lgb.Dataset(X_train, y_train, params=params)
    gbm = lgb.train(params, lgb_train, num_boost_round=2)
    pred = gbm.predict(X_train)
    assert np.allclose(pred, expected_pred)


def test_constant_features_regression():
2369
    params = {"objective": "regression"}
2370
2371
2372
2373
2374
2375
    check_constant_features([0.0, 10.0, 0.0, 10.0], 5.0, params)
    check_constant_features([0.0, 1.0, 2.0, 3.0], 1.5, params)
    check_constant_features([-1.0, 1.0, -2.0, 2.0], 0.0, params)


def test_constant_features_binary():
2376
    params = {"objective": "binary"}
2377
2378
2379
2380
2381
    check_constant_features([0.0, 10.0, 0.0, 10.0], 0.5, params)
    check_constant_features([0.0, 1.0, 2.0, 3.0], 0.75, params)


def test_constant_features_multiclass():
2382
    params = {"objective": "multiclass", "num_class": 3}
2383
2384
2385
2386
2387
    check_constant_features([0.0, 1.0, 2.0, 0.0], [0.5, 0.25, 0.25], params)
    check_constant_features([0.0, 1.0, 2.0, 1.0], [0.25, 0.5, 0.25], params)


def test_constant_features_multiclassova():
2388
    params = {"objective": "multiclassova", "num_class": 3}
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
    check_constant_features([0.0, 1.0, 2.0, 0.0], [0.5, 0.25, 0.25], params)
    check_constant_features([0.0, 1.0, 2.0, 1.0], [0.25, 0.5, 0.25], params)


def test_fpreproc():
    def preprocess_data(dtrain, dtest, params):
        train_data = dtrain.construct().get_data()
        test_data = dtest.construct().get_data()
        train_data[:, 0] += 1
        test_data[:, 0] += 1
        dtrain.label[-5:] = 3
        dtest.label[-5:] = 3
        dtrain = lgb.Dataset(train_data, dtrain.label)
        dtest = lgb.Dataset(test_data, dtest.label, reference=dtrain)
2403
        params["num_class"] = 4
2404
2405
2406
2407
        return dtrain, dtest, params

    X, y = load_iris(return_X_y=True)
    dataset = lgb.Dataset(X, y, free_raw_data=False)
2408
    params = {"objective": "multiclass", "num_class": 3, "verbose": -1}
2409
    results = lgb.cv(params, dataset, num_boost_round=10, fpreproc=preprocess_data)
2410
2411
    assert "valid multi_logloss-mean" in results
    assert len(results["valid multi_logloss-mean"]) == 10
2412
2413
2414
2415
2416


def test_metrics():
    X, y = load_digits(n_class=2, return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
2417
2418
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_valid = lgb.Dataset(X_test, y_test, reference=lgb_train)
2419
2420

    evals_result = {}
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
    params_dummy_obj_verbose = {"verbose": -1, "objective": dummy_obj}
    params_obj_verbose = {"objective": "binary", "verbose": -1}
    params_obj_metric_log_verbose = {"objective": "binary", "metric": "binary_logloss", "verbose": -1}
    params_obj_metric_err_verbose = {"objective": "binary", "metric": "binary_error", "verbose": -1}
    params_obj_metric_inv_verbose = {"objective": "binary", "metric": "invalid_metric", "verbose": -1}
    params_obj_metric_quant_verbose = {"objective": "regression", "metric": "quantile", "verbose": 2}
    params_obj_metric_multi_verbose = {
        "objective": "binary",
        "metric": ["binary_logloss", "binary_error"],
        "verbose": -1,
    }
    params_obj_metric_none_verbose = {"objective": "binary", "metric": "None", "verbose": -1}
    params_dummy_obj_metric_log_verbose = {"objective": dummy_obj, "metric": "binary_logloss", "verbose": -1}
    params_dummy_obj_metric_err_verbose = {"objective": dummy_obj, "metric": "binary_error", "verbose": -1}
    params_dummy_obj_metric_inv_verbose = {"objective": dummy_obj, "metric_types": "invalid_metric", "verbose": -1}
    params_dummy_obj_metric_multi_verbose = {
        "objective": dummy_obj,
        "metric": ["binary_logloss", "binary_error"],
        "verbose": -1,
    }
    params_dummy_obj_metric_none_verbose = {"objective": dummy_obj, "metric": "None", "verbose": -1}
2442
2443

    def get_cv_result(params=params_obj_verbose, **kwargs):
2444
        return lgb.cv(params, lgb_train, num_boost_round=2, **kwargs)
2445
2446

    def train_booster(params=params_obj_verbose, **kwargs):
2447
2448
2449
2450
2451
2452
        lgb.train(
            params,
            lgb_train,
            num_boost_round=2,
            valid_sets=[lgb_valid],
            callbacks=[lgb.record_evaluation(evals_result)],
2453
            **kwargs,
2454
        )
2455

2456
    # no custom objective, no feval
2457
2458
2459
    # default metric
    res = get_cv_result()
    assert len(res) == 2
2460
    assert "valid binary_logloss-mean" in res
2461
2462
2463
2464

    # non-default metric in params
    res = get_cv_result(params=params_obj_metric_err_verbose)
    assert len(res) == 2
2465
    assert "valid binary_error-mean" in res
2466
2467

    # default metric in args
2468
    res = get_cv_result(metrics="binary_logloss")
2469
    assert len(res) == 2
2470
    assert "valid binary_logloss-mean" in res
2471
2472

    # non-default metric in args
2473
    res = get_cv_result(metrics="binary_error")
2474
    assert len(res) == 2
2475
    assert "valid binary_error-mean" in res
2476
2477

    # metric in args overwrites one in params
2478
    res = get_cv_result(params=params_obj_metric_inv_verbose, metrics="binary_error")
2479
    assert len(res) == 2
2480
    assert "valid binary_error-mean" in res
2481

2482
2483
2484
    # metric in args overwrites one in params
    res = get_cv_result(params=params_obj_metric_quant_verbose)
    assert len(res) == 2
2485
    assert "valid quantile-mean" in res
2486

2487
2488
2489
    # multiple metrics in params
    res = get_cv_result(params=params_obj_metric_multi_verbose)
    assert len(res) == 4
2490
2491
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
2492
2493

    # multiple metrics in args
2494
    res = get_cv_result(metrics=["binary_logloss", "binary_error"])
2495
    assert len(res) == 4
2496
2497
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
2498
2499

    # remove default metric by 'None' in list
2500
    res = get_cv_result(metrics=["None"])
2501
2502
2503
    assert len(res) == 0

    # remove default metric by 'None' aliases
2504
    for na_alias in ("None", "na", "null", "custom"):
2505
2506
2507
        res = get_cv_result(metrics=na_alias)
        assert len(res) == 0

2508
    # custom objective, no feval
2509
    # no default metric
2510
    res = get_cv_result(params=params_dummy_obj_verbose)
2511
2512
2513
    assert len(res) == 0

    # metric in params
2514
    res = get_cv_result(params=params_dummy_obj_metric_err_verbose)
2515
    assert len(res) == 2
2516
    assert "valid binary_error-mean" in res
2517
2518

    # metric in args
2519
    res = get_cv_result(params=params_dummy_obj_verbose, metrics="binary_error")
2520
    assert len(res) == 2
2521
    assert "valid binary_error-mean" in res
2522
2523

    # metric in args overwrites its' alias in params
2524
    res = get_cv_result(params=params_dummy_obj_metric_inv_verbose, metrics="binary_error")
2525
    assert len(res) == 2
2526
    assert "valid binary_error-mean" in res
2527
2528

    # multiple metrics in params
2529
    res = get_cv_result(params=params_dummy_obj_metric_multi_verbose)
2530
    assert len(res) == 4
2531
2532
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
2533
2534

    # multiple metrics in args
2535
    res = get_cv_result(params=params_dummy_obj_verbose, metrics=["binary_logloss", "binary_error"])
2536
    assert len(res) == 4
2537
2538
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
2539

2540
    # no custom objective, feval
2541
2542
2543
    # default metric with custom one
    res = get_cv_result(feval=constant_metric)
    assert len(res) == 4
2544
2545
    assert "valid binary_logloss-mean" in res
    assert "valid error-mean" in res
2546
2547
2548
2549

    # non-default metric in params with custom one
    res = get_cv_result(params=params_obj_metric_err_verbose, feval=constant_metric)
    assert len(res) == 4
2550
2551
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2552
2553

    # default metric in args with custom one
2554
    res = get_cv_result(metrics="binary_logloss", feval=constant_metric)
2555
    assert len(res) == 4
2556
2557
    assert "valid binary_logloss-mean" in res
    assert "valid error-mean" in res
2558
2559

    # non-default metric in args with custom one
2560
    res = get_cv_result(metrics="binary_error", feval=constant_metric)
2561
    assert len(res) == 4
2562
2563
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2564
2565

    # metric in args overwrites one in params, custom one is evaluated too
2566
    res = get_cv_result(params=params_obj_metric_inv_verbose, metrics="binary_error", feval=constant_metric)
2567
    assert len(res) == 4
2568
2569
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2570
2571
2572
2573

    # multiple metrics in params with custom one
    res = get_cv_result(params=params_obj_metric_multi_verbose, feval=constant_metric)
    assert len(res) == 6
2574
2575
2576
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2577
2578

    # multiple metrics in args with custom one
2579
    res = get_cv_result(metrics=["binary_logloss", "binary_error"], feval=constant_metric)
2580
    assert len(res) == 6
2581
2582
2583
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2584
2585

    # custom metric is evaluated despite 'None' is passed
2586
    res = get_cv_result(metrics=["None"], feval=constant_metric)
2587
    assert len(res) == 2
2588
    assert "valid error-mean" in res
2589

2590
    # custom objective, feval
2591
    # no default metric, only custom one
2592
    res = get_cv_result(params=params_dummy_obj_verbose, feval=constant_metric)
2593
    assert len(res) == 2
2594
    assert "valid error-mean" in res
2595
2596

    # metric in params with custom one
2597
    res = get_cv_result(params=params_dummy_obj_metric_err_verbose, feval=constant_metric)
2598
    assert len(res) == 4
2599
2600
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2601
2602

    # metric in args with custom one
2603
    res = get_cv_result(params=params_dummy_obj_verbose, feval=constant_metric, metrics="binary_error")
2604
    assert len(res) == 4
2605
2606
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2607
2608

    # metric in args overwrites one in params, custom one is evaluated too
2609
    res = get_cv_result(params=params_dummy_obj_metric_inv_verbose, feval=constant_metric, metrics="binary_error")
2610
    assert len(res) == 4
2611
2612
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2613
2614

    # multiple metrics in params with custom one
2615
    res = get_cv_result(params=params_dummy_obj_metric_multi_verbose, feval=constant_metric)
2616
    assert len(res) == 6
2617
2618
2619
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2620
2621

    # multiple metrics in args with custom one
2622
2623
2624
    res = get_cv_result(
        params=params_dummy_obj_verbose, feval=constant_metric, metrics=["binary_logloss", "binary_error"]
    )
2625
    assert len(res) == 6
2626
2627
2628
    assert "valid binary_logloss-mean" in res
    assert "valid binary_error-mean" in res
    assert "valid error-mean" in res
2629
2630

    # custom metric is evaluated despite 'None' is passed
2631
    res = get_cv_result(params=params_dummy_obj_metric_none_verbose, feval=constant_metric)
2632
    assert len(res) == 2
2633
    assert "valid error-mean" in res
2634

2635
    # no custom objective, no feval
2636
2637
    # default metric
    train_booster()
2638
2639
    assert len(evals_result["valid_0"]) == 1
    assert "binary_logloss" in evals_result["valid_0"]
2640
2641
2642

    # default metric in params
    train_booster(params=params_obj_metric_log_verbose)
2643
2644
    assert len(evals_result["valid_0"]) == 1
    assert "binary_logloss" in evals_result["valid_0"]
2645
2646
2647

    # non-default metric in params
    train_booster(params=params_obj_metric_err_verbose)
2648
2649
    assert len(evals_result["valid_0"]) == 1
    assert "binary_error" in evals_result["valid_0"]
2650
2651
2652

    # multiple metrics in params
    train_booster(params=params_obj_metric_multi_verbose)
2653
2654
2655
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "binary_error" in evals_result["valid_0"]
2656
2657

    # remove default metric by 'None' aliases
2658
2659
    for na_alias in ("None", "na", "null", "custom"):
        params = {"objective": "binary", "metric": na_alias, "verbose": -1}
2660
2661
2662
        train_booster(params=params)
        assert len(evals_result) == 0

2663
    # custom objective, no feval
2664
    # no default metric
2665
    train_booster(params=params_dummy_obj_verbose)
2666
2667
2668
    assert len(evals_result) == 0

    # metric in params
2669
    train_booster(params=params_dummy_obj_metric_log_verbose)
2670
2671
    assert len(evals_result["valid_0"]) == 1
    assert "binary_logloss" in evals_result["valid_0"]
2672
2673

    # multiple metrics in params
2674
    train_booster(params=params_dummy_obj_metric_multi_verbose)
2675
2676
2677
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "binary_error" in evals_result["valid_0"]
2678

2679
    # no custom objective, feval
2680
2681
    # default metric with custom one
    train_booster(feval=constant_metric)
2682
2683
2684
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2685
2686
2687

    # default metric in params with custom one
    train_booster(params=params_obj_metric_log_verbose, feval=constant_metric)
2688
2689
2690
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2691
2692
2693

    # non-default metric in params with custom one
    train_booster(params=params_obj_metric_err_verbose, feval=constant_metric)
2694
2695
2696
    assert len(evals_result["valid_0"]) == 2
    assert "binary_error" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2697
2698
2699

    # multiple metrics in params with custom one
    train_booster(params=params_obj_metric_multi_verbose, feval=constant_metric)
2700
2701
2702
2703
    assert len(evals_result["valid_0"]) == 3
    assert "binary_logloss" in evals_result["valid_0"]
    assert "binary_error" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2704
2705
2706
2707

    # custom metric is evaluated despite 'None' is passed
    train_booster(params=params_obj_metric_none_verbose, feval=constant_metric)
    assert len(evals_result) == 1
2708
    assert "error" in evals_result["valid_0"]
2709

2710
    # custom objective, feval
2711
    # no default metric, only custom one
2712
    train_booster(params=params_dummy_obj_verbose, feval=constant_metric)
2713
2714
    assert len(evals_result["valid_0"]) == 1
    assert "error" in evals_result["valid_0"]
2715
2716

    # metric in params with custom one
2717
    train_booster(params=params_dummy_obj_metric_log_verbose, feval=constant_metric)
2718
2719
2720
    assert len(evals_result["valid_0"]) == 2
    assert "binary_logloss" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2721
2722

    # multiple metrics in params with custom one
2723
    train_booster(params=params_dummy_obj_metric_multi_verbose, feval=constant_metric)
2724
2725
2726
2727
    assert len(evals_result["valid_0"]) == 3
    assert "binary_logloss" in evals_result["valid_0"]
    assert "binary_error" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
2728
2729

    # custom metric is evaluated despite 'None' is passed
2730
    train_booster(params=params_dummy_obj_metric_none_verbose, feval=constant_metric)
2731
    assert len(evals_result) == 1
2732
    assert "error" in evals_result["valid_0"]
2733
2734

    X, y = load_digits(n_class=3, return_X_y=True)
2735
    lgb_train = lgb.Dataset(X, y)
2736

2737
    obj_multi_aliases = ["multiclass", "softmax", "multiclassova", "multiclass_ova", "ova", "ovr"]
2738
    for obj_multi_alias in obj_multi_aliases:
2739
        # Custom objective replaces multiclass
2740
2741
2742
2743
2744
        params_obj_class_3_verbose = {"objective": obj_multi_alias, "num_class": 3, "verbose": -1}
        params_dummy_obj_class_3_verbose = {"objective": dummy_obj, "num_class": 3, "verbose": -1}
        params_dummy_obj_class_1_verbose = {"objective": dummy_obj, "num_class": 1, "verbose": -1}
        params_obj_verbose = {"objective": obj_multi_alias, "verbose": -1}
        params_dummy_obj_verbose = {"objective": dummy_obj, "verbose": -1}
2745
2746
2747
        # multiclass default metric
        res = get_cv_result(params_obj_class_3_verbose)
        assert len(res) == 2
2748
        assert "valid multi_logloss-mean" in res
2749
2750
2751
        # multiclass default metric with custom one
        res = get_cv_result(params_obj_class_3_verbose, feval=constant_metric)
        assert len(res) == 4
2752
2753
        assert "valid multi_logloss-mean" in res
        assert "valid error-mean" in res
2754
        # multiclass metric alias with custom one for custom objective
2755
        res = get_cv_result(params_dummy_obj_class_3_verbose, feval=constant_metric)
2756
        assert len(res) == 2
2757
        assert "valid error-mean" in res
2758
        # no metric for invalid class_num
2759
        res = get_cv_result(params_dummy_obj_class_1_verbose)
2760
2761
        assert len(res) == 0
        # custom metric for invalid class_num
2762
        res = get_cv_result(params_dummy_obj_class_1_verbose, feval=constant_metric)
2763
        assert len(res) == 2
2764
        assert "valid error-mean" in res
2765
2766
        # multiclass metric alias with custom one with invalid class_num
        with pytest.raises(lgb.basic.LightGBMError):
2767
            get_cv_result(params_dummy_obj_class_1_verbose, metrics=obj_multi_alias, feval=constant_metric)
2768
2769
2770
        # multiclass default metric without num_class
        with pytest.raises(lgb.basic.LightGBMError):
            get_cv_result(params_obj_verbose)
2771
        for metric_multi_alias in obj_multi_aliases + ["multi_logloss"]:
2772
2773
2774
            # multiclass metric alias
            res = get_cv_result(params_obj_class_3_verbose, metrics=metric_multi_alias)
            assert len(res) == 2
2775
            assert "valid multi_logloss-mean" in res
2776
        # multiclass metric
2777
        res = get_cv_result(params_obj_class_3_verbose, metrics="multi_error")
2778
        assert len(res) == 2
2779
        assert "valid multi_error-mean" in res
2780
2781
        # non-valid metric for multiclass objective
        with pytest.raises(lgb.basic.LightGBMError):
2782
2783
            get_cv_result(params_obj_class_3_verbose, metrics="binary_logloss")
    params_class_3_verbose = {"num_class": 3, "verbose": -1}
2784
2785
2786
2787
    # non-default num_class for default objective
    with pytest.raises(lgb.basic.LightGBMError):
        get_cv_result(params_class_3_verbose)
    # no metric with non-default num_class for custom objective
2788
    res = get_cv_result(params_dummy_obj_class_3_verbose)
2789
    assert len(res) == 0
2790
    for metric_multi_alias in obj_multi_aliases + ["multi_logloss"]:
2791
        # multiclass metric alias for custom objective
2792
        res = get_cv_result(params_dummy_obj_class_3_verbose, metrics=metric_multi_alias)
2793
        assert len(res) == 2
2794
        assert "valid multi_logloss-mean" in res
2795
    # multiclass metric for custom objective
2796
    res = get_cv_result(params_dummy_obj_class_3_verbose, metrics="multi_error")
2797
    assert len(res) == 2
2798
    assert "valid multi_error-mean" in res
2799
2800
    # binary metric with non-default num_class for custom objective
    with pytest.raises(lgb.basic.LightGBMError):
2801
        get_cv_result(params_dummy_obj_class_3_verbose, metrics="binary_error")
2802
2803
2804
2805
2806


def test_multiple_feval_train():
    X, y = load_breast_cancer(return_X_y=True)

2807
    params = {"verbose": -1, "objective": "binary", "metric": "binary_logloss"}
2808
2809
2810

    X_train, X_validation, y_train, y_validation = train_test_split(X, y, test_size=0.2)

2811
2812
    train_dataset = lgb.Dataset(data=X_train, label=y_train)
    validation_dataset = lgb.Dataset(data=X_validation, label=y_validation, reference=train_dataset)
2813
2814
2815
2816
2817
2818
2819
    evals_result = {}
    lgb.train(
        params=params,
        train_set=train_dataset,
        valid_sets=validation_dataset,
        num_boost_round=5,
        feval=[constant_metric, decreasing_metric],
2820
        callbacks=[lgb.record_evaluation(evals_result)],
2821
    )
2822

2823
2824
2825
2826
    assert len(evals_result["valid_0"]) == 3
    assert "binary_logloss" in evals_result["valid_0"]
    assert "error" in evals_result["valid_0"]
    assert "decreasing_metric" in evals_result["valid_0"]
2827
2828


2829
2830
def test_objective_callable_train_binary_classification():
    X, y = load_breast_cancer(return_X_y=True)
2831
    params = {"verbose": -1, "objective": logloss_obj, "learning_rate": 0.01}
2832
    train_dataset = lgb.Dataset(X, y)
2833
    booster = lgb.train(params=params, train_set=train_dataset, num_boost_round=20)
2834
2835
2836
    y_pred = logistic_sigmoid(booster.predict(X))
    logloss_error = log_loss(y, y_pred)
    rocauc_error = roc_auc_score(y, y_pred)
2837
    assert booster.params["objective"] == "none"
2838
2839
    assert logloss_error == pytest.approx(0.547907)
    assert rocauc_error == pytest.approx(0.995944)
2840
2841
2842
2843


def test_objective_callable_train_regression():
    X, y = make_synthetic_regression()
2844
    params = {"verbose": -1, "objective": mse_obj}
2845
    lgb_train = lgb.Dataset(X, y)
2846
    booster = lgb.train(params, lgb_train, num_boost_round=20)
2847
2848
    y_pred = booster.predict(X)
    mse_error = mean_squared_error(y, y_pred)
2849
    assert booster.params["objective"] == "none"
2850
    assert mse_error == pytest.approx(286.724194)
2851
2852
2853
2854


def test_objective_callable_cv_binary_classification():
    X, y = load_breast_cancer(return_X_y=True)
2855
    params = {"verbose": -1, "objective": logloss_obj, "learning_rate": 0.01}
2856
    train_dataset = lgb.Dataset(X, y)
2857
2858
2859
2860
    cv_res = lgb.cv(params, train_dataset, num_boost_round=20, nfold=3, return_cvbooster=True)
    cv_booster = cv_res["cvbooster"].boosters
    cv_logloss_errors = [log_loss(y, logistic_sigmoid(cb.predict(X))) < 0.56 for cb in cv_booster]
    cv_objs = [cb.params["objective"] == "none" for cb in cv_booster]
2861
2862
2863
2864
2865
2866
2867
    assert all(cv_objs)
    assert all(cv_logloss_errors)


def test_objective_callable_cv_regression():
    X, y = make_synthetic_regression()
    lgb_train = lgb.Dataset(X, y)
2868
2869
2870
2871
2872
    params = {"verbose": -1, "objective": mse_obj}
    cv_res = lgb.cv(params, lgb_train, num_boost_round=20, nfold=3, stratified=False, return_cvbooster=True)
    cv_booster = cv_res["cvbooster"].boosters
    cv_mse_errors = [mean_squared_error(y, cb.predict(X)) < 463 for cb in cv_booster]
    cv_objs = [cb.params["objective"] == "none" for cb in cv_booster]
2873
2874
2875
2876
    assert all(cv_objs)
    assert all(cv_mse_errors)


2877
2878
2879
def test_multiple_feval_cv():
    X, y = load_breast_cancer(return_X_y=True)

2880
    params = {"verbose": -1, "objective": "binary", "metric": "binary_logloss"}
2881

2882
    train_dataset = lgb.Dataset(data=X, label=y)
2883
2884

    cv_results = lgb.cv(
2885
2886
        params=params, train_set=train_dataset, num_boost_round=5, feval=[constant_metric, decreasing_metric]
    )
2887
2888
2889

    # Expect three metrics but mean and stdv for each metric
    assert len(cv_results) == 6
2890
2891
2892
2893
2894
2895
    assert "valid binary_logloss-mean" in cv_results
    assert "valid error-mean" in cv_results
    assert "valid decreasing_metric-mean" in cv_results
    assert "valid binary_logloss-stdv" in cv_results
    assert "valid error-stdv" in cv_results
    assert "valid decreasing_metric-stdv" in cv_results
2896
2897


2898
2899
2900
2901
2902
2903
def test_default_objective_and_metric():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
    train_dataset = lgb.Dataset(data=X_train, label=y_train)
    validation_dataset = lgb.Dataset(data=X_test, label=y_test, reference=train_dataset)
    evals_result = {}
2904
    params = {"verbose": -1}
2905
2906
2907
2908
2909
    lgb.train(
        params=params,
        train_set=train_dataset,
        valid_sets=validation_dataset,
        num_boost_round=5,
2910
        callbacks=[lgb.record_evaluation(evals_result)],
2911
2912
    )

2913
2914
2915
2916
    assert "valid_0" in evals_result
    assert len(evals_result["valid_0"]) == 1
    assert "l2" in evals_result["valid_0"]
    assert len(evals_result["valid_0"]["l2"]) == 5
2917
2918


2919
@pytest.mark.parametrize("use_weight", [True, False])
2920
def test_multiclass_custom_objective(use_weight):
2921
2922
    def custom_obj(y_pred, ds):
        y_true = ds.get_label()
2923
2924
2925
        weight = ds.get_weight()
        grad, hess = sklearn_multiclass_custom_objective(y_true, y_pred, weight)
        return grad, hess
2926
2927
2928

    centers = [[-4, -4], [4, 4], [-4, 4]]
    X, y = make_blobs(n_samples=1_000, centers=centers, random_state=42)
2929
    weight = np.full_like(y, 2)
2930
    ds = lgb.Dataset(X, y)
2931
2932
    if use_weight:
        ds.set_weight(weight)
2933
    params = {"objective": "multiclass", "num_class": 3, "num_leaves": 7}
2934
2935
2936
    builtin_obj_bst = lgb.train(params, ds, num_boost_round=10)
    builtin_obj_preds = builtin_obj_bst.predict(X)

2937
    params["objective"] = custom_obj
2938
    custom_obj_bst = lgb.train(params, ds, num_boost_round=10)
2939
2940
2941
2942
2943
    custom_obj_preds = softmax(custom_obj_bst.predict(X))

    np.testing.assert_allclose(builtin_obj_preds, custom_obj_preds, rtol=0.01)


2944
@pytest.mark.parametrize("use_weight", [True, False])
2945
def test_multiclass_custom_eval(use_weight):
2946
2947
    def custom_eval(y_pred, ds):
        y_true = ds.get_label()
2948
2949
        weight = ds.get_weight()  # weight is None when not set
        loss = log_loss(y_true, y_pred, sample_weight=weight)
2950
        return "custom_logloss", loss, False
2951
2952
2953

    centers = [[-4, -4], [4, 4], [-4, 4]]
    X, y = make_blobs(n_samples=1_000, centers=centers, random_state=42)
2954
2955
2956
2957
    weight = np.full_like(y, 2)
    X_train, X_valid, y_train, y_valid, weight_train, weight_valid = train_test_split(
        X, y, weight, test_size=0.2, random_state=0
    )
2958
2959
    train_ds = lgb.Dataset(X_train, y_train)
    valid_ds = lgb.Dataset(X_valid, y_valid, reference=train_ds)
2960
2961
2962
    if use_weight:
        train_ds.set_weight(weight_train)
        valid_ds.set_weight(weight_valid)
2963
    params = {"objective": "multiclass", "num_class": 3, "num_leaves": 7}
2964
2965
2966
2967
2968
2969
    eval_result = {}
    bst = lgb.train(
        params,
        train_ds,
        num_boost_round=10,
        valid_sets=[train_ds, valid_ds],
2970
        valid_names=["train", "valid"],
2971
2972
2973
2974
2975
        feval=custom_eval,
        callbacks=[lgb.record_evaluation(eval_result)],
        keep_training_booster=True,
    )

2976
2977
    for key, ds in zip(["train", "valid"], [train_ds, valid_ds]):
        np.testing.assert_allclose(eval_result[key]["multi_logloss"], eval_result[key]["custom_logloss"])
2978
        _, metric, value, _ = bst.eval(ds, key, feval=custom_eval)[1]  # first element is multi_logloss
2979
        assert metric == "custom_logloss"
2980
2981
2982
        np.testing.assert_allclose(value, eval_result[key][metric][-1])


2983
@pytest.mark.skipif(psutil.virtual_memory().available / 1024 / 1024 / 1024 < 3, reason="not enough RAM")
2984
def test_model_size():
2985
    X, y = make_synthetic_regression()
2986
    data = lgb.Dataset(X, y)
2987
    bst = lgb.train({"verbose": -1}, data, num_boost_round=2)
2988
2989
    y_pred = bst.predict(X)
    model_str = bst.model_to_string()
2990
    one_tree = model_str[model_str.find("Tree=1") : model_str.find("end of trees")]
2991
    one_tree_size = len(one_tree)
2992
    one_tree = one_tree.replace("Tree=1", "Tree={}")
2993
2994
2995
    multiplier = 100
    total_trees = multiplier + 2
    try:
2996
2997
        before_tree_sizes = model_str[: model_str.find("tree_sizes")]
        trees = model_str[model_str.find("Tree=0") : model_str.find("end of trees")]
2998
        more_trees = (one_tree * multiplier).format(*range(2, total_trees))
2999
        after_trees = model_str[model_str.find("end of trees") :]
3000
3001
        num_end_spaces = 2**31 - one_tree_size * total_trees
        new_model_str = f"{before_tree_sizes}\n\n{trees}{more_trees}{after_trees}{'':{num_end_spaces}}"
3002
        assert len(new_model_str) > 2**31
3003
        bst.model_from_string(new_model_str)
3004
3005
3006
3007
        assert bst.num_trees() == total_trees
        y_pred_new = bst.predict(X, num_iteration=2)
        np.testing.assert_allclose(y_pred, y_pred_new)
    except MemoryError:
3008
        pytest.skipTest("not enough RAM")
3009
3010


3011
3012
3013
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Skip due to differences in implementation details of CUDA version"
)
3014
def test_get_split_value_histogram():
3015
3016
3017
3018
    X, y = make_synthetic_regression()
    X = np.repeat(X, 3, axis=0)
    y = np.repeat(y, 3, axis=0)
    X[:, 2] = np.random.default_rng(0).integers(0, 20, size=X.shape[0])
3019
    lgb_train = lgb.Dataset(X, y, categorical_feature=[2])
3020
    gbm = lgb.train({"verbose": -1}, lgb_train, num_boost_round=20)
3021
    # test XGBoost-style return value
3022
    params = {"feature": 0, "xgboost_style": True}
3023
3024
    assert gbm.get_split_value_histogram(**params).shape == (12, 2)
    assert gbm.get_split_value_histogram(bins=999, **params).shape == (12, 2)
3025
3026
3027
3028
    assert gbm.get_split_value_histogram(bins=-1, **params).shape == (1, 2)
    assert gbm.get_split_value_histogram(bins=0, **params).shape == (1, 2)
    assert gbm.get_split_value_histogram(bins=1, **params).shape == (1, 2)
    assert gbm.get_split_value_histogram(bins=2, **params).shape == (2, 2)
3029
3030
    assert gbm.get_split_value_histogram(bins=6, **params).shape == (6, 2)
    assert gbm.get_split_value_histogram(bins=7, **params).shape == (7, 2)
3031
3032
3033
    if lgb.compat.PANDAS_INSTALLED:
        np.testing.assert_allclose(
            gbm.get_split_value_histogram(0, xgboost_style=True).values,
3034
            gbm.get_split_value_histogram(gbm.feature_name()[0], xgboost_style=True).values,
3035
3036
3037
        )
        np.testing.assert_allclose(
            gbm.get_split_value_histogram(X.shape[-1] - 1, xgboost_style=True).values,
3038
            gbm.get_split_value_histogram(gbm.feature_name()[X.shape[-1] - 1], xgboost_style=True).values,
3039
3040
3041
3042
        )
    else:
        np.testing.assert_allclose(
            gbm.get_split_value_histogram(0, xgboost_style=True),
3043
            gbm.get_split_value_histogram(gbm.feature_name()[0], xgboost_style=True),
3044
3045
3046
        )
        np.testing.assert_allclose(
            gbm.get_split_value_histogram(X.shape[-1] - 1, xgboost_style=True),
3047
            gbm.get_split_value_histogram(gbm.feature_name()[X.shape[-1] - 1], xgboost_style=True),
3048
3049
3050
        )
    # test numpy-style return value
    hist, bins = gbm.get_split_value_histogram(0)
3051
3052
    assert len(hist) == 20
    assert len(bins) == 21
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
    hist, bins = gbm.get_split_value_histogram(0, bins=999)
    assert len(hist) == 999
    assert len(bins) == 1000
    with pytest.raises(ValueError):
        gbm.get_split_value_histogram(0, bins=-1)
    with pytest.raises(ValueError):
        gbm.get_split_value_histogram(0, bins=0)
    hist, bins = gbm.get_split_value_histogram(0, bins=1)
    assert len(hist) == 1
    assert len(bins) == 2
    hist, bins = gbm.get_split_value_histogram(0, bins=2)
    assert len(hist) == 2
    assert len(bins) == 3
    hist, bins = gbm.get_split_value_histogram(0, bins=6)
    assert len(hist) == 6
    assert len(bins) == 7
    hist, bins = gbm.get_split_value_histogram(0, bins=7)
    assert len(hist) == 7
    assert len(bins) == 8
    hist_idx, bins_idx = gbm.get_split_value_histogram(0)
    hist_name, bins_name = gbm.get_split_value_histogram(gbm.feature_name()[0])
    np.testing.assert_array_equal(hist_idx, hist_name)
    np.testing.assert_allclose(bins_idx, bins_name)
    hist_idx, bins_idx = gbm.get_split_value_histogram(X.shape[-1] - 1)
    hist_name, bins_name = gbm.get_split_value_histogram(gbm.feature_name()[X.shape[-1] - 1])
    np.testing.assert_array_equal(hist_idx, hist_name)
    np.testing.assert_allclose(bins_idx, bins_name)
    # test bins string type
3081
3082
    hist_vals, bin_edges = gbm.get_split_value_histogram(0, bins="auto")
    hist = gbm.get_split_value_histogram(0, bins="auto", xgboost_style=True)
3083
3084
    if lgb.compat.PANDAS_INSTALLED:
        mask = hist_vals > 0
3085
3086
        np.testing.assert_array_equal(hist_vals[mask], hist["Count"].values)
        np.testing.assert_allclose(bin_edges[1:][mask], hist["SplitValue"].values)
3087
3088
3089
3090
    else:
        mask = hist_vals > 0
        np.testing.assert_array_equal(hist_vals[mask], hist[:, 1])
        np.testing.assert_allclose(bin_edges[1:][mask], hist[:, 0])
3091
3092
3093
    # test histogram is disabled for categorical features
    with pytest.raises(lgb.basic.LightGBMError):
        gbm.get_split_value_histogram(2)
3094
3095


3096
3097
3098
@pytest.mark.skipif(
    getenv("TASK", "") == "cuda", reason="Skip due to differences in implementation details of CUDA version"
)
3099
def test_early_stopping_for_only_first_metric():
3100
    def metrics_combination_train_regression(valid_sets, metric_list, assumed_iteration, first_metric_only, feval=None):
3101
        params = {
3102
3103
3104
3105
3106
3107
            "objective": "regression",
            "learning_rate": 1.1,
            "num_leaves": 10,
            "metric": metric_list,
            "verbose": -1,
            "seed": 123,
3108
        }
3109
3110
3111
3112
3113
3114
        gbm = lgb.train(
            params,
            lgb_train,
            num_boost_round=25,
            valid_sets=valid_sets,
            feval=feval,
3115
            callbacks=[lgb.early_stopping(stopping_rounds=5, first_metric_only=first_metric_only)],
3116
        )
3117
        assert assumed_iteration == gbm.best_iteration
3118

3119
3120
3121
    def metrics_combination_cv_regression(
        metric_list, assumed_iteration, first_metric_only, eval_train_metric, feval=None
    ):
3122
        params = {
3123
3124
3125
3126
3127
3128
3129
            "objective": "regression",
            "learning_rate": 0.9,
            "num_leaves": 10,
            "metric": metric_list,
            "verbose": -1,
            "seed": 123,
            "gpu_use_dp": True,
3130
        }
3131
3132
3133
3134
3135
3136
3137
        ret = lgb.cv(
            params,
            train_set=lgb_train,
            num_boost_round=25,
            stratified=False,
            feval=feval,
            callbacks=[lgb.early_stopping(stopping_rounds=5, first_metric_only=first_metric_only)],
3138
            eval_train_metric=eval_train_metric,
3139
        )
3140
3141
        assert assumed_iteration == len(ret[list(ret.keys())[0]])

3142
    X, y = make_synthetic_regression()
3143
3144
3145
3146
3147
3148
3149
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
    X_test1, X_test2, y_test1, y_test2 = train_test_split(X_test, y_test, test_size=0.5, random_state=73)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_valid1 = lgb.Dataset(X_test1, y_test1, reference=lgb_train)
    lgb_valid2 = lgb.Dataset(X_test2, y_test2, reference=lgb_train)

    iter_valid1_l1 = 3
3150
3151
    iter_valid1_l2 = 3
    iter_valid2_l1 = 3
3152
    iter_valid2_l2 = 15
3153
    assert len({iter_valid1_l1, iter_valid1_l2, iter_valid2_l1, iter_valid2_l2}) == 2
3154
3155
3156
3157
    iter_min_l1 = min([iter_valid1_l1, iter_valid2_l1])
    iter_min_l2 = min([iter_valid1_l2, iter_valid2_l2])
    iter_min_valid1 = min([iter_valid1_l1, iter_valid1_l2])

3158
3159
    iter_cv_l1 = 15
    iter_cv_l2 = 13
3160
    assert len({iter_cv_l1, iter_cv_l2}) == 2
3161
3162
3163
3164
3165
3166
3167
    iter_cv_min = min([iter_cv_l1, iter_cv_l2])

    # test for lgb.train
    metrics_combination_train_regression(lgb_valid1, [], iter_valid1_l2, False)
    metrics_combination_train_regression(lgb_valid1, [], iter_valid1_l2, True)
    metrics_combination_train_regression(lgb_valid1, None, iter_valid1_l2, False)
    metrics_combination_train_regression(lgb_valid1, None, iter_valid1_l2, True)
3168
3169
3170
3171
3172
3173
    metrics_combination_train_regression(lgb_valid1, "l2", iter_valid1_l2, True)
    metrics_combination_train_regression(lgb_valid1, "l1", iter_valid1_l1, True)
    metrics_combination_train_regression(lgb_valid1, ["l2", "l1"], iter_valid1_l2, True)
    metrics_combination_train_regression(lgb_valid1, ["l1", "l2"], iter_valid1_l1, True)
    metrics_combination_train_regression(lgb_valid1, ["l2", "l1"], iter_min_valid1, False)
    metrics_combination_train_regression(lgb_valid1, ["l1", "l2"], iter_min_valid1, False)
3174
3175

    # test feval for lgb.train
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
    metrics_combination_train_regression(
        lgb_valid1,
        "None",
        1,
        False,
        feval=lambda preds, train_data: [decreasing_metric(preds, train_data), constant_metric(preds, train_data)],
    )
    metrics_combination_train_regression(
        lgb_valid1,
        "None",
        25,
        True,
        feval=lambda preds, train_data: [decreasing_metric(preds, train_data), constant_metric(preds, train_data)],
    )
    metrics_combination_train_regression(
        lgb_valid1,
        "None",
        1,
        True,
        feval=lambda preds, train_data: [constant_metric(preds, train_data), decreasing_metric(preds, train_data)],
    )
3197
3198

    # test with two valid data for lgb.train
3199
3200
3201
3202
    metrics_combination_train_regression([lgb_valid1, lgb_valid2], ["l2", "l1"], iter_min_l2, True)
    metrics_combination_train_regression([lgb_valid2, lgb_valid1], ["l2", "l1"], iter_min_l2, True)
    metrics_combination_train_regression([lgb_valid1, lgb_valid2], ["l1", "l2"], iter_min_l1, True)
    metrics_combination_train_regression([lgb_valid2, lgb_valid1], ["l1", "l2"], iter_min_l1, True)
3203
3204
3205

    # test for lgb.cv
    metrics_combination_cv_regression(None, iter_cv_l2, True, False)
3206
3207
3208
3209
3210
3211
    metrics_combination_cv_regression("l2", iter_cv_l2, True, False)
    metrics_combination_cv_regression("l1", iter_cv_l1, True, False)
    metrics_combination_cv_regression(["l2", "l1"], iter_cv_l2, True, False)
    metrics_combination_cv_regression(["l1", "l2"], iter_cv_l1, True, False)
    metrics_combination_cv_regression(["l2", "l1"], iter_cv_min, False, False)
    metrics_combination_cv_regression(["l1", "l2"], iter_cv_min, False, False)
3212
    metrics_combination_cv_regression(None, iter_cv_l2, True, True)
3213
3214
3215
3216
3217
3218
    metrics_combination_cv_regression("l2", iter_cv_l2, True, True)
    metrics_combination_cv_regression("l1", iter_cv_l1, True, True)
    metrics_combination_cv_regression(["l2", "l1"], iter_cv_l2, True, True)
    metrics_combination_cv_regression(["l1", "l2"], iter_cv_l1, True, True)
    metrics_combination_cv_regression(["l2", "l1"], iter_cv_min, False, True)
    metrics_combination_cv_regression(["l1", "l2"], iter_cv_min, False, True)
3219
3220

    # test feval for lgb.cv
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
    metrics_combination_cv_regression(
        "None",
        1,
        False,
        False,
        feval=lambda preds, train_data: [decreasing_metric(preds, train_data), constant_metric(preds, train_data)],
    )
    metrics_combination_cv_regression(
        "None",
        25,
        True,
        False,
        feval=lambda preds, train_data: [decreasing_metric(preds, train_data), constant_metric(preds, train_data)],
    )
    metrics_combination_cv_regression(
        "None",
        1,
        True,
        False,
        feval=lambda preds, train_data: [constant_metric(preds, train_data), decreasing_metric(preds, train_data)],
    )
3242
3243
3244
3245
3246
3247


def test_node_level_subcol():
    X, y = load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    params = {
3248
3249
3250
3251
3252
        "objective": "binary",
        "metric": "binary_logloss",
        "feature_fraction_bynode": 0.8,
        "feature_fraction": 1.0,
        "verbose": -1,
3253
3254
3255
3256
    }
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    evals_result = {}
3257
    gbm = lgb.train(
3258
        params, lgb_train, num_boost_round=25, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
3259
    )
3260
3261
    ret = log_loss(y_test, gbm.predict(X_test))
    assert ret < 0.14
3262
3263
    assert evals_result["valid_0"]["binary_logloss"][-1] == pytest.approx(ret)
    params["feature_fraction"] = 0.5
3264
3265
3266
3267
3268
    gbm2 = lgb.train(params, lgb_train, num_boost_round=25)
    ret2 = log_loss(y_test, gbm2.predict(X_test))
    assert ret != ret2


3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
def test_forced_split_feature_indices(tmp_path):
    X, y = make_synthetic_regression()
    forced_split = {
        "feature": 0,
        "threshold": 0.5,
        "left": {"feature": X.shape[1], "threshold": 0.5},
    }
    tmp_split_file = tmp_path / "forced_split.json"
    with open(tmp_split_file, "w") as f:
        f.write(json.dumps(forced_split))
    lgb_train = lgb.Dataset(X, y)
3280
    params = {"objective": "regression", "forcedsplits_filename": tmp_split_file}
3281
    with pytest.raises(lgb.basic.LightGBMError, match="Forced splits file includes feature index"):
3282
        lgb.train(params, lgb_train)
3283
3284


3285
def test_forced_bins():
3286
    x = np.empty((100, 2))
3287
3288
3289
    x[:, 0] = np.arange(0, 1, 0.01)
    x[:, 1] = -np.arange(0, 1, 0.01)
    y = np.arange(0, 1, 0.01)
3290
3291
3292
3293
3294
3295
3296
3297
3298
    forcedbins_filename = Path(__file__).absolute().parents[2] / "examples" / "regression" / "forced_bins.json"
    params = {
        "objective": "regression_l1",
        "max_bin": 5,
        "forcedbins_filename": forcedbins_filename,
        "num_leaves": 2,
        "min_data_in_leaf": 1,
        "verbose": -1,
    }
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
    lgb_x = lgb.Dataset(x, label=y)
    est = lgb.train(params, lgb_x, num_boost_round=20)
    new_x = np.zeros((3, x.shape[1]))
    new_x[:, 0] = [0.31, 0.37, 0.41]
    predicted = est.predict(new_x)
    assert len(np.unique(predicted)) == 3
    new_x[:, 0] = [0, 0, 0]
    new_x[:, 1] = [-0.9, -0.6, -0.3]
    predicted = est.predict(new_x)
    assert len(np.unique(predicted)) == 1
3309
    params["forcedbins_filename"] = ""
3310
3311
3312
3313
    lgb_x = lgb.Dataset(x, label=y)
    est = lgb.train(params, lgb_x, num_boost_round=20)
    predicted = est.predict(new_x)
    assert len(np.unique(predicted)) == 3
3314
3315
    params["forcedbins_filename"] = (
        Path(__file__).absolute().parents[2] / "examples" / "regression" / "forced_bins2.json"
3316
    )
3317
    params["max_bin"] = 11
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
    lgb_x = lgb.Dataset(x[:, :1], label=y)
    est = lgb.train(params, lgb_x, num_boost_round=50)
    predicted = est.predict(x[1:, :1])
    _, counts = np.unique(predicted, return_counts=True)
    assert min(counts) >= 9
    assert max(counts) <= 11


def test_binning_same_sign():
    # test that binning works properly for features with only positive or only negative values
3328
    x = np.empty((99, 2))
3329
3330
3331
    x[:, 0] = np.arange(0.01, 1, 0.01)
    x[:, 1] = -np.arange(0.01, 1, 0.01)
    y = np.arange(0.01, 1, 0.01)
3332
3333
3334
3335
3336
3337
3338
3339
    params = {
        "objective": "regression_l1",
        "max_bin": 5,
        "num_leaves": 2,
        "min_data_in_leaf": 1,
        "verbose": -1,
        "seed": 0,
    }
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
    lgb_x = lgb.Dataset(x, label=y)
    est = lgb.train(params, lgb_x, num_boost_round=20)
    new_x = np.zeros((3, 2))
    new_x[:, 0] = [-1, 0, 1]
    predicted = est.predict(new_x)
    assert predicted[0] == pytest.approx(predicted[1])
    assert predicted[1] != pytest.approx(predicted[2])
    new_x = np.zeros((3, 2))
    new_x[:, 1] = [-1, 0, 1]
    predicted = est.predict(new_x)
    assert predicted[0] != pytest.approx(predicted[1])
    assert predicted[1] == pytest.approx(predicted[2])


def test_dataset_update_params():
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
    default_params = {
        "max_bin": 100,
        "max_bin_by_feature": [20, 10],
        "bin_construct_sample_cnt": 10000,
        "min_data_in_bin": 1,
        "use_missing": False,
        "zero_as_missing": False,
        "categorical_feature": [0],
        "feature_pre_filter": True,
        "pre_partition": False,
        "enable_bundle": True,
        "data_random_seed": 0,
        "is_enable_sparse": True,
        "header": True,
        "two_round": True,
        "label_column": 0,
        "weight_column": 0,
        "group_column": 0,
        "ignore_column": 0,
        "min_data_in_leaf": 10,
        "linear_tree": False,
        "precise_float_parser": True,
        "verbose": -1,
    }
    unchangeable_params = {
        "max_bin": 150,
        "max_bin_by_feature": [30, 5],
        "bin_construct_sample_cnt": 5000,
        "min_data_in_bin": 2,
        "use_missing": True,
        "zero_as_missing": True,
        "categorical_feature": [0, 1],
        "feature_pre_filter": False,
        "pre_partition": True,
        "enable_bundle": False,
        "data_random_seed": 1,
        "is_enable_sparse": False,
        "header": False,
        "two_round": False,
        "label_column": 1,
        "weight_column": 1,
        "group_column": 1,
        "ignore_column": 1,
        "forcedbins_filename": "/some/path/forcedbins.json",
        "min_data_in_leaf": 2,
        "linear_tree": True,
        "precise_float_parser": False,
    }
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
    X = np.random.random((100, 2))
    y = np.random.random(100)

    # decreasing without freeing raw data is allowed
    lgb_data = lgb.Dataset(X, y, params=default_params, free_raw_data=False).construct()
    default_params["min_data_in_leaf"] -= 1
    lgb.train(default_params, lgb_data, num_boost_round=3)

    # decreasing before lazy init is allowed
    lgb_data = lgb.Dataset(X, y, params=default_params)
    default_params["min_data_in_leaf"] -= 1
    lgb.train(default_params, lgb_data, num_boost_round=3)

    # increasing is allowed
    default_params["min_data_in_leaf"] += 2
    lgb.train(default_params, lgb_data, num_boost_round=3)

    # decreasing with disabled filter is allowed
    default_params["feature_pre_filter"] = False
    lgb_data = lgb.Dataset(X, y, params=default_params).construct()
    default_params["min_data_in_leaf"] -= 4
    lgb.train(default_params, lgb_data, num_boost_round=3)

    # decreasing with enabled filter is disallowed;
    # also changes of other params are disallowed
    default_params["feature_pre_filter"] = True
    lgb_data = lgb.Dataset(X, y, params=default_params).construct()
    for key, value in unchangeable_params.items():
        new_params = default_params.copy()
        new_params[key] = value
3433
3434
3435
3436
        if key != "forcedbins_filename":
            param_name = key
        else:
            param_name = "forced bins"
3437
3438
3439
3440
3441
        err_msg = (
            "Reducing `min_data_in_leaf` with `feature_pre_filter=true` may cause *"
            if key == "min_data_in_leaf"
            else f"Cannot change {param_name} *"
        )
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
        with np.testing.assert_raises_regex(lgb.basic.LightGBMError, err_msg):
            lgb.train(new_params, lgb_data, num_boost_round=3)


def test_dataset_params_with_reference():
    default_params = {"max_bin": 100}
    X = np.random.random((100, 2))
    y = np.random.random(100)
    X_val = np.random.random((100, 2))
    y_val = np.random.random(100)
    lgb_train = lgb.Dataset(X, y, params=default_params, free_raw_data=False).construct()
    lgb_val = lgb.Dataset(X_val, y_val, reference=lgb_train, free_raw_data=False).construct()
    assert lgb_train.get_params() == default_params
    assert lgb_val.get_params() == default_params
    lgb.train(default_params, lgb_train, valid_sets=[lgb_val])


def test_extra_trees():
    # check extra trees increases regularization
3461
    X, y = make_synthetic_regression()
3462
    lgb_x = lgb.Dataset(X, label=y)
3463
    params = {"objective": "regression", "num_leaves": 32, "verbose": -1, "extra_trees": False, "seed": 0}
3464
3465
3466
    est = lgb.train(params, lgb_x, num_boost_round=10)
    predicted = est.predict(X)
    err = mean_squared_error(y, predicted)
3467
    params["extra_trees"] = True
3468
3469
3470
3471
3472
3473
3474
3475
    est = lgb.train(params, lgb_x, num_boost_round=10)
    predicted_new = est.predict(X)
    err_new = mean_squared_error(y, predicted_new)
    assert err < err_new


def test_path_smoothing():
    # check path smoothing increases regularization
3476
    X, y = make_synthetic_regression()
3477
    lgb_x = lgb.Dataset(X, label=y)
3478
    params = {"objective": "regression", "num_leaves": 32, "verbose": -1, "seed": 0}
3479
3480
3481
    est = lgb.train(params, lgb_x, num_boost_round=10)
    predicted = est.predict(X)
    err = mean_squared_error(y, predicted)
3482
    params["path_smooth"] = 1
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
    est = lgb.train(params, lgb_x, num_boost_round=10)
    predicted_new = est.predict(X)
    err_new = mean_squared_error(y, predicted_new)
    assert err < err_new


def test_trees_to_dataframe():
    pytest.importorskip("pandas")

    def _imptcs_to_numpy(X, impcts_dict):
3493
3494
        cols = [f"Column_{i}" for i in range(X.shape[1])]
        return [impcts_dict.get(col, 0.0) for col in cols]
3495
3496
3497
3498
3499
3500

    X, y = load_breast_cancer(return_X_y=True)
    data = lgb.Dataset(X, label=y)
    num_trees = 10
    bst = lgb.train({"objective": "binary", "verbose": -1}, data, num_trees)
    tree_df = bst.trees_to_dataframe()
3501
    split_dict = tree_df[~tree_df["split_gain"].isnull()].groupby("split_feature").size().to_dict()
3502

3503
    gains_dict = tree_df.groupby("split_feature")["split_gain"].sum().to_dict()
3504
3505
3506

    tree_split = _imptcs_to_numpy(X, split_dict)
    tree_gains = _imptcs_to_numpy(X, gains_dict)
3507
3508
3509
3510
    mod_split = bst.feature_importance("split")
    mod_gains = bst.feature_importance("gain")
    num_trees_from_df = tree_df["tree_index"].nunique()
    obs_counts_from_df = tree_df.loc[tree_df["node_depth"] == 1, "count"].values
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524

    np.testing.assert_equal(tree_split, mod_split)
    np.testing.assert_allclose(tree_gains, mod_gains)
    assert num_trees_from_df == num_trees
    np.testing.assert_equal(obs_counts_from_df, len(y))

    # test edge case with one leaf
    X = np.ones((10, 2))
    y = np.random.rand(10)
    data = lgb.Dataset(X, label=y)
    bst = lgb.train({"objective": "binary", "verbose": -1}, data, num_trees)
    tree_df = bst.trees_to_dataframe()

    assert len(tree_df) == 1
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
    assert tree_df.loc[0, "tree_index"] == 0
    assert tree_df.loc[0, "node_depth"] == 1
    assert tree_df.loc[0, "node_index"] == "0-L0"
    assert tree_df.loc[0, "value"] is not None
    for col in (
        "left_child",
        "right_child",
        "parent_index",
        "split_feature",
        "split_gain",
        "threshold",
        "decision_type",
        "missing_direction",
        "missing_type",
        "weight",
        "count",
    ):
3542
3543
3544
3545
        assert tree_df.loc[0, col] is None


def test_interaction_constraints():
3546
    X, y = make_synthetic_regression(n_samples=200)
3547
3548
3549
    num_features = X.shape[1]
    train_data = lgb.Dataset(X, label=y)
    # check that constraint containing all features is equivalent to no constraint
3550
    params = {"verbose": -1, "seed": 0}
3551
3552
    est = lgb.train(params, train_data, num_boost_round=10)
    pred1 = est.predict(X)
3553
    est = lgb.train(dict(params, interaction_constraints=[list(range(num_features))]), train_data, num_boost_round=10)
3554
3555
3556
    pred2 = est.predict(X)
    np.testing.assert_allclose(pred1, pred2)
    # check that constraint partitioning the features reduces train accuracy
3557
    est = lgb.train(dict(params, interaction_constraints=[[0, 2], [1, 3]]), train_data, num_boost_round=10)
3558
3559
3560
    pred3 = est.predict(X)
    assert mean_squared_error(y, pred1) < mean_squared_error(y, pred3)
    # check that constraints consisting of single features reduce accuracy further
3561
3562
3563
    est = lgb.train(
        dict(params, interaction_constraints=[[i] for i in range(num_features)]), train_data, num_boost_round=10
    )
3564
3565
3566
3567
3568
3569
    pred4 = est.predict(X)
    assert mean_squared_error(y, pred3) < mean_squared_error(y, pred4)
    # test that interaction constraints work when not all features are used
    X = np.concatenate([np.zeros((X.shape[0], 1)), X], axis=1)
    num_features = X.shape[1]
    train_data = lgb.Dataset(X, label=y)
3570
3571
3572
3573
3574
    est = lgb.train(
        dict(params, interaction_constraints=[[0] + list(range(2, num_features)), [1] + list(range(2, num_features))]),
        train_data,
        num_boost_round=10,
    )
3575
3576


3577
3578
3579
3580
3581
3582
3583
def test_linear_trees_num_threads():
    # check that number of threads does not affect result
    np.random.seed(0)
    x = np.arange(0, 1000, 0.1)
    y = 2 * x + np.random.normal(0, 0.1, len(x))
    x = x[:, np.newaxis]
    lgb_train = lgb.Dataset(x, label=y)
3584
    params = {"verbose": -1, "objective": "regression", "seed": 0, "linear_tree": True, "num_threads": 2}
3585
3586
3587
3588
3589
3590
3591
3592
    est = lgb.train(params, lgb_train, num_boost_round=100)
    pred1 = est.predict(x)
    params["num_threads"] = 4
    est = lgb.train(params, lgb_train, num_boost_round=100)
    pred2 = est.predict(x)
    np.testing.assert_allclose(pred1, pred2)


3593
3594
3595
3596
3597
3598
3599
def test_linear_trees(tmp_path):
    # check that setting linear_tree=True fits better than ordinary trees when data has linear relationship
    np.random.seed(0)
    x = np.arange(0, 100, 0.1)
    y = 2 * x + np.random.normal(0, 0.1, len(x))
    x = x[:, np.newaxis]
    lgb_train = lgb.Dataset(x, label=y)
3600
    params = {"verbose": -1, "metric": "mse", "seed": 0, "num_leaves": 2}
3601
3602
3603
3604
    est = lgb.train(params, lgb_train, num_boost_round=10)
    pred1 = est.predict(x)
    lgb_train = lgb.Dataset(x, label=y)
    res = {}
3605
    est = lgb.train(
3606
        dict(params, linear_tree=True),
3607
3608
3609
        lgb_train,
        num_boost_round=10,
        valid_sets=[lgb_train],
3610
3611
        valid_names=["train"],
        callbacks=[lgb.record_evaluation(res)],
3612
    )
3613
    pred2 = est.predict(x)
3614
    assert res["train"]["l2"][-1] == pytest.approx(mean_squared_error(y, pred2), abs=1e-1)
3615
3616
3617
3618
3619
3620
3621
3622
    assert mean_squared_error(y, pred2) < mean_squared_error(y, pred1)
    # test again with nans in data
    x[:10] = np.nan
    lgb_train = lgb.Dataset(x, label=y)
    est = lgb.train(params, lgb_train, num_boost_round=10)
    pred1 = est.predict(x)
    lgb_train = lgb.Dataset(x, label=y)
    res = {}
3623
    est = lgb.train(
3624
        dict(params, linear_tree=True),
3625
3626
3627
        lgb_train,
        num_boost_round=10,
        valid_sets=[lgb_train],
3628
3629
        valid_names=["train"],
        callbacks=[lgb.record_evaluation(res)],
3630
    )
3631
    pred2 = est.predict(x)
3632
    assert res["train"]["l2"][-1] == pytest.approx(mean_squared_error(y, pred2), abs=1e-1)
3633
3634
3635
    assert mean_squared_error(y, pred2) < mean_squared_error(y, pred1)
    # test again with bagging
    res = {}
3636
    est = lgb.train(
3637
        dict(params, linear_tree=True, subsample=0.8, bagging_freq=1),
3638
3639
3640
        lgb_train,
        num_boost_round=10,
        valid_sets=[lgb_train],
3641
3642
        valid_names=["train"],
        callbacks=[lgb.record_evaluation(res)],
3643
    )
3644
    pred = est.predict(x)
3645
    assert res["train"]["l2"][-1] == pytest.approx(mean_squared_error(y, pred), abs=1e-1)
3646
3647
3648
3649
3650
3651
    # test with a feature that has only one non-nan value
    x = np.concatenate([np.ones([x.shape[0], 1]), x], 1)
    x[500:, 1] = np.nan
    y[500:] += 10
    lgb_train = lgb.Dataset(x, label=y)
    res = {}
3652
    est = lgb.train(
3653
        dict(params, linear_tree=True, subsample=0.8, bagging_freq=1),
3654
3655
3656
        lgb_train,
        num_boost_round=10,
        valid_sets=[lgb_train],
3657
3658
        valid_names=["train"],
        callbacks=[lgb.record_evaluation(res)],
3659
    )
3660
    pred = est.predict(x)
3661
    assert res["train"]["l2"][-1] == pytest.approx(mean_squared_error(y, pred), abs=1e-1)
3662
3663
3664
3665
    # test with a categorical feature
    x[:250, 0] = 0
    y[:250] += 10
    lgb_train = lgb.Dataset(x, label=y)
3666
3667
3668
3669
3670
3671
    est = lgb.train(
        dict(params, linear_tree=True, subsample=0.8, bagging_freq=1),
        lgb_train,
        num_boost_round=10,
        categorical_feature=[0],
    )
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
    # test refit: same results on same data
    est2 = est.refit(x, label=y)
    p1 = est.predict(x)
    p2 = est2.predict(x)
    assert np.mean(np.abs(p1 - p2)) < 2

    # test refit with save and load
    temp_model = str(tmp_path / "temp_model.txt")
    est.save_model(temp_model)
    est2 = lgb.Booster(model_file=temp_model)
    est2 = est2.refit(x, label=y)
    p1 = est.predict(x)
    p2 = est2.predict(x)
    assert np.mean(np.abs(p1 - p2)) < 2
    # test refit: different results training on different data
    est3 = est.refit(x[:100, :], label=y[:100])
    p3 = est3.predict(x)
    assert np.mean(np.abs(p2 - p1)) > np.abs(np.max(p3 - p1))
    # test when num_leaves - 1 < num_features and when num_leaves - 1 > num_features
    X_train, _, y_train, _ = train_test_split(*load_breast_cancer(return_X_y=True), test_size=0.1, random_state=2)
3692
    params = {"linear_tree": True, "verbose": -1, "metric": "mse", "seed": 0}
3693
3694
3695
3696
3697
3698
    train_data = lgb.Dataset(X_train, label=y_train, params=dict(params, num_leaves=2))
    est = lgb.train(params, train_data, num_boost_round=10, categorical_feature=[0])
    train_data = lgb.Dataset(X_train, label=y_train, params=dict(params, num_leaves=60))
    est = lgb.train(params, train_data, num_boost_round=10, categorical_feature=[0])


3699
def test_save_and_load_linear(tmp_path):
3700
3701
3702
    X_train, X_test, y_train, y_test = train_test_split(
        *load_breast_cancer(return_X_y=True), test_size=0.1, random_state=2
    )
3703
    X_train = np.concatenate([np.ones((X_train.shape[0], 1)), X_train], 1)
3704
3705
3706
    X_train[: X_train.shape[0] // 2, 0] = 0
    y_train[: X_train.shape[0] // 2] = 1
    params = {"linear_tree": True}
3707
3708
3709
3710
    train_data_1 = lgb.Dataset(X_train, label=y_train, params=params)
    est_1 = lgb.train(params, train_data_1, num_boost_round=10, categorical_feature=[0])
    pred_1 = est_1.predict(X_train)

3711
    tmp_dataset = str(tmp_path / "temp_dataset.bin")
3712
3713
3714
3715
3716
3717
    train_data_1.save_binary(tmp_dataset)
    train_data_2 = lgb.Dataset(tmp_dataset)
    est_2 = lgb.train(params, train_data_2, num_boost_round=10)
    pred_2 = est_2.predict(X_train)
    np.testing.assert_allclose(pred_1, pred_2)

3718
    model_file = str(tmp_path / "model.txt")
3719
3720
3721
3722
3723
3724
    est_2.save_model(model_file)
    est_3 = lgb.Booster(model_file=model_file)
    pred_3 = est_3.predict(X_train)
    np.testing.assert_allclose(pred_2, pred_3)


3725
3726
3727
def test_linear_single_leaf():
    X_train, y_train = load_breast_cancer(return_X_y=True)
    train_data = lgb.Dataset(X_train, label=y_train)
3728
    params = {"objective": "binary", "linear_tree": True, "min_sum_hessian": 5000}
3729
3730
3731
3732
3733
    bst = lgb.train(params, train_data, num_boost_round=5)
    y_pred = bst.predict(X_train)
    assert log_loss(y_train, y_pred) < 0.661


3734
3735
3736
3737
3738
def test_predict_with_start_iteration():
    def inner_test(X, y, params, early_stopping_rounds):
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
        train_data = lgb.Dataset(X_train, label=y_train)
        valid_data = lgb.Dataset(X_test, label=y_test)
3739
        callbacks = [lgb.early_stopping(early_stopping_rounds)] if early_stopping_rounds is not None else []
3740
        booster = lgb.train(params, train_data, num_boost_round=50, valid_sets=[valid_data], callbacks=callbacks)
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757

        # test that the predict once with all iterations equals summed results with start_iteration and num_iteration
        all_pred = booster.predict(X, raw_score=True)
        all_pred_contrib = booster.predict(X, pred_contrib=True)
        steps = [10, 12]
        for step in steps:
            pred = np.zeros_like(all_pred)
            pred_contrib = np.zeros_like(all_pred_contrib)
            for start_iter in range(0, 50, step):
                pred += booster.predict(X, start_iteration=start_iter, num_iteration=step, raw_score=True)
                pred_contrib += booster.predict(X, start_iteration=start_iter, num_iteration=step, pred_contrib=True)
            np.testing.assert_allclose(all_pred, pred)
            np.testing.assert_allclose(all_pred_contrib, pred_contrib)
        # test the case where start_iteration <= 0, and num_iteration is None
        pred1 = booster.predict(X, start_iteration=-1)
        pred2 = booster.predict(X, num_iteration=booster.best_iteration)
        np.testing.assert_allclose(pred1, pred2)
3758

3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
        # test the case where start_iteration > 0, and num_iteration <= 0
        pred4 = booster.predict(X, start_iteration=10, num_iteration=-1)
        pred5 = booster.predict(X, start_iteration=10, num_iteration=90)
        pred6 = booster.predict(X, start_iteration=10, num_iteration=0)
        np.testing.assert_allclose(pred4, pred5)
        np.testing.assert_allclose(pred4, pred6)

        # test the case where start_iteration > 0, and num_iteration <= 0, with pred_leaf=True
        pred4 = booster.predict(X, start_iteration=10, num_iteration=-1, pred_leaf=True)
        pred5 = booster.predict(X, start_iteration=10, num_iteration=40, pred_leaf=True)
        pred6 = booster.predict(X, start_iteration=10, num_iteration=0, pred_leaf=True)
        np.testing.assert_allclose(pred4, pred5)
        np.testing.assert_allclose(pred4, pred6)

        # test the case where start_iteration > 0, and num_iteration <= 0, with pred_contrib=True
        pred4 = booster.predict(X, start_iteration=10, num_iteration=-1, pred_contrib=True)
        pred5 = booster.predict(X, start_iteration=10, num_iteration=40, pred_contrib=True)
        pred6 = booster.predict(X, start_iteration=10, num_iteration=0, pred_contrib=True)
        np.testing.assert_allclose(pred4, pred5)
        np.testing.assert_allclose(pred4, pred6)

    # test for regression
3781
    X, y = make_synthetic_regression()
3782
    params = {"objective": "regression", "verbose": -1, "metric": "l2", "learning_rate": 0.5}
3783
3784
3785
3786
3787
3788
3789
    # test both with and without early stopping
    inner_test(X, y, params, early_stopping_rounds=1)
    inner_test(X, y, params, early_stopping_rounds=5)
    inner_test(X, y, params, early_stopping_rounds=None)

    # test for multi-class
    X, y = load_iris(return_X_y=True)
3790
    params = {"objective": "multiclass", "num_class": 3, "verbose": -1, "metric": "multi_error"}
3791
3792
3793
3794
3795
3796
3797
    # test both with and without early stopping
    inner_test(X, y, params, early_stopping_rounds=1)
    inner_test(X, y, params, early_stopping_rounds=5)
    inner_test(X, y, params, early_stopping_rounds=None)

    # test for binary
    X, y = load_breast_cancer(return_X_y=True)
3798
    params = {"objective": "binary", "verbose": -1, "metric": "auc"}
3799
3800
3801
3802
3803
3804
3805
3806
3807
    # test both with and without early stopping
    inner_test(X, y, params, early_stopping_rounds=1)
    inner_test(X, y, params, early_stopping_rounds=5)
    inner_test(X, y, params, early_stopping_rounds=None)


def test_average_precision_metric():
    # test against sklearn average precision metric
    X, y = load_breast_cancer(return_X_y=True)
3808
    params = {"objective": "binary", "metric": "average_precision", "verbose": -1}
3809
3810
    res = {}
    lgb_X = lgb.Dataset(X, label=y)
3811
3812
    est = lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(res)])
    ap = res["training"]["average_precision"][-1]
3813
3814
3815
3816
3817
3818
3819
    pred = est.predict(X)
    sklearn_ap = average_precision_score(y, pred)
    assert ap == pytest.approx(sklearn_ap)
    # test that average precision is 1 where model predicts perfectly
    y = y.copy()
    y[:] = 1
    lgb_X = lgb.Dataset(X, label=y)
3820
3821
    lgb.train(params, lgb_X, num_boost_round=1, valid_sets=[lgb_X], callbacks=[lgb.record_evaluation(res)])
    assert res["training"]["average_precision"][-1] == pytest.approx(1)
3822
3823
3824
3825
3826
3827


def test_reset_params_works_with_metric_num_class_and_boosting():
    X, y = load_breast_cancer(return_X_y=True)
    dataset_params = {"max_bin": 150}
    booster_params = {
3828
3829
3830
3831
3832
3833
        "objective": "multiclass",
        "max_depth": 4,
        "bagging_fraction": 0.8,
        "metric": ["multi_logloss", "multi_error"],
        "boosting": "gbdt",
        "num_class": 5,
3834
3835
    }
    dtrain = lgb.Dataset(X, y, params=dataset_params)
3836
    bst = lgb.Booster(params=booster_params, train_set=dtrain)
3837
3838
3839
3840

    expected_params = dict(dataset_params, **booster_params)
    assert bst.params == expected_params

3841
    booster_params["bagging_fraction"] += 0.1
3842
3843
3844
3845
3846
    new_bst = bst.reset_parameter(booster_params)

    expected_params = dict(dataset_params, **booster_params)
    assert bst.params == expected_params
    assert new_bst.params == expected_params
3847
3848
3849
3850
3851


def test_dump_model():
    X, y = load_breast_cancer(return_X_y=True)
    train_data = lgb.Dataset(X, label=y)
3852
    params = {"objective": "binary", "verbose": -1}
3853
3854
3855
3856
3857
3858
3859
    bst = lgb.train(params, train_data, num_boost_round=5)
    dumped_model_str = str(bst.dump_model(5, 0))
    assert "leaf_features" not in dumped_model_str
    assert "leaf_coeff" not in dumped_model_str
    assert "leaf_const" not in dumped_model_str
    assert "leaf_value" in dumped_model_str
    assert "leaf_count" in dumped_model_str
3860
    params["linear_tree"] = True
3861
3862
3863
3864
3865
3866
3867
3868
    train_data = lgb.Dataset(X, label=y)
    bst = lgb.train(params, train_data, num_boost_round=5)
    dumped_model_str = str(bst.dump_model(5, 0))
    assert "leaf_features" in dumped_model_str
    assert "leaf_coeff" in dumped_model_str
    assert "leaf_const" in dumped_model_str
    assert "leaf_value" in dumped_model_str
    assert "leaf_count" in dumped_model_str
3869
3870
3871
3872


def test_dump_model_hook():
    def hook(obj):
3873
3874
3875
        if "leaf_value" in obj:
            obj["LV"] = obj["leaf_value"]
            del obj["leaf_value"]
3876
3877
3878
3879
        return obj

    X, y = load_breast_cancer(return_X_y=True)
    train_data = lgb.Dataset(X, label=y)
3880
    params = {"objective": "binary", "verbose": -1}
3881
3882
3883
3884
    bst = lgb.train(params, train_data, num_boost_round=5)
    dumped_model_str = str(bst.dump_model(5, 0, object_hook=hook))
    assert "leaf_value" not in dumped_model_str
    assert "LV" in dumped_model_str
3885
3886


3887
@pytest.mark.skipif(getenv("TASK", "") == "cuda", reason="Forced splits are not yet supported by CUDA version")
3888
def test_force_split_with_feature_fraction(tmp_path):
3889
    X, y = make_synthetic_regression()
3890
3891
3892
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)

3893
    forced_split = {"feature": 0, "threshold": 0.5, "right": {"feature": 2, "threshold": 10.0}}
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903

    tmp_split_file = tmp_path / "forced_split.json"
    with open(tmp_split_file, "w") as f:
        f.write(json.dumps(forced_split))

    params = {
        "objective": "regression",
        "feature_fraction": 0.6,
        "force_col_wise": True,
        "feature_fraction_seed": 1,
3904
        "forcedsplits_filename": tmp_split_file,
3905
3906
3907
3908
    }

    gbm = lgb.train(params, lgb_train)
    ret = mean_absolute_error(y_test, gbm.predict(X_test))
3909
    assert ret < 15.7
3910
3911
3912
3913
3914

    tree_info = gbm.dump_model()["tree_info"]
    assert len(tree_info) > 1
    for tree in tree_info:
        tree_structure = tree["tree_structure"]
3915
        assert tree_structure["split_feature"] == 0
3916
3917


3918
3919
3920
3921
3922
3923
def test_goss_boosting_and_strategy_equivalent():
    X, y = make_synthetic_regression(n_samples=10_000, n_features=10, n_informative=5, random_state=42)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
    base_params = {
3924
3925
3926
3927
3928
3929
3930
        "metric": "l2",
        "verbose": -1,
        "bagging_seed": 0,
        "learning_rate": 0.05,
        "num_threads": 1,
        "force_row_wise": True,
        "gpu_use_dp": True,
3931
    }
3932
    params1 = {**base_params, "boosting": "goss"}
3933
    evals_result1 = {}
3934
3935
3936
3937
    lgb.train(
        params1, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result1)]
    )
    params2 = {**base_params, "data_sample_strategy": "goss"}
3938
    evals_result2 = {}
3939
3940
3941
3942
    lgb.train(
        params2, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result2)]
    )
    assert evals_result1["valid_0"]["l2"] == evals_result2["valid_0"]["l2"]
3943
3944
3945
3946
3947
3948
3949
3950
3951


def test_sample_strategy_with_boosting():
    X, y = make_synthetic_regression(n_samples=10_000, n_features=10, n_informative=5, random_state=42)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)

    base_params = {
3952
3953
3954
3955
3956
        "metric": "l2",
        "verbose": -1,
        "num_threads": 1,
        "force_row_wise": True,
        "gpu_use_dp": True,
3957
3958
    }

3959
    params1 = {**base_params, "boosting": "dart", "data_sample_strategy": "goss"}
3960
    evals_result = {}
3961
3962
3963
3964
    gbm = lgb.train(
        params1, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res1 = evals_result["valid_0"]["l2"][-1]
3965
3966
3967
3968
    test_res1 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res1 == pytest.approx(3149.393862, abs=1.0)
    assert eval_res1 == pytest.approx(test_res1)

3969
    params2 = {**base_params, "boosting": "gbdt", "data_sample_strategy": "goss"}
3970
    evals_result = {}
3971
3972
3973
3974
    gbm = lgb.train(
        params2, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res2 = evals_result["valid_0"]["l2"][-1]
3975
3976
3977
3978
    test_res2 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res2 == pytest.approx(2547.715968, abs=1.0)
    assert eval_res2 == pytest.approx(test_res2)

3979
    params3 = {**base_params, "boosting": "goss", "data_sample_strategy": "goss"}
3980
    evals_result = {}
3981
3982
3983
3984
    gbm = lgb.train(
        params3, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res3 = evals_result["valid_0"]["l2"][-1]
3985
3986
3987
3988
    test_res3 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res3 == pytest.approx(2547.715968, abs=1.0)
    assert eval_res3 == pytest.approx(test_res3)

3989
    params4 = {**base_params, "boosting": "rf", "data_sample_strategy": "goss"}
3990
    evals_result = {}
3991
3992
3993
3994
    gbm = lgb.train(
        params4, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res4 = evals_result["valid_0"]["l2"][-1]
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
    test_res4 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res4 == pytest.approx(2095.538735, abs=1.0)
    assert eval_res4 == pytest.approx(test_res4)

    assert test_res1 != test_res2
    assert eval_res1 != eval_res2
    assert test_res2 == test_res3
    assert eval_res2 == eval_res3
    assert eval_res1 != eval_res4
    assert test_res1 != test_res4
    assert eval_res2 != eval_res4
    assert test_res2 != test_res4

4008
4009
4010
4011
4012
4013
4014
    params5 = {
        **base_params,
        "boosting": "dart",
        "data_sample_strategy": "bagging",
        "bagging_freq": 1,
        "bagging_fraction": 0.5,
    }
4015
    evals_result = {}
4016
4017
4018
4019
    gbm = lgb.train(
        params5, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res5 = evals_result["valid_0"]["l2"][-1]
4020
4021
4022
4023
    test_res5 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res5 == pytest.approx(3134.866931, abs=1.0)
    assert eval_res5 == pytest.approx(test_res5)

4024
4025
4026
4027
4028
4029
4030
    params6 = {
        **base_params,
        "boosting": "gbdt",
        "data_sample_strategy": "bagging",
        "bagging_freq": 1,
        "bagging_fraction": 0.5,
    }
4031
    evals_result = {}
4032
4033
4034
4035
    gbm = lgb.train(
        params6, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res6 = evals_result["valid_0"]["l2"][-1]
4036
4037
4038
4039
4040
4041
    test_res6 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res6 == pytest.approx(2539.792378, abs=1.0)
    assert eval_res6 == pytest.approx(test_res6)
    assert test_res5 != test_res6
    assert eval_res5 != eval_res6

4042
4043
4044
4045
4046
4047
4048
    params7 = {
        **base_params,
        "boosting": "rf",
        "data_sample_strategy": "bagging",
        "bagging_freq": 1,
        "bagging_fraction": 0.5,
    }
4049
    evals_result = {}
4050
4051
4052
4053
    gbm = lgb.train(
        params7, lgb_train, num_boost_round=10, valid_sets=lgb_eval, callbacks=[lgb.record_evaluation(evals_result)]
    )
    eval_res7 = evals_result["valid_0"]["l2"][-1]
4054
4055
4056
4057
4058
4059
4060
4061
4062
    test_res7 = mean_squared_error(y_test, gbm.predict(X_test))
    assert test_res7 == pytest.approx(1518.704481, abs=1.0)
    assert eval_res7 == pytest.approx(test_res7)
    assert test_res5 != test_res7
    assert eval_res5 != eval_res7
    assert test_res6 != test_res7
    assert eval_res6 != eval_res7


4063
4064
4065
4066
4067
def test_record_evaluation_with_train():
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, y)
    eval_result = {}
    callbacks = [lgb.record_evaluation(eval_result)]
4068
    params = {"objective": "l2", "num_leaves": 3}
4069
4070
    num_boost_round = 5
    bst = lgb.train(params, ds, num_boost_round=num_boost_round, valid_sets=[ds], callbacks=callbacks)
4071
    assert list(eval_result.keys()) == ["training"]
4072
4073
4074
4075
4076
    train_mses = []
    for i in range(num_boost_round):
        pred = bst.predict(X, num_iteration=i + 1)
        mse = mean_squared_error(y, pred)
        train_mses.append(mse)
4077
    np.testing.assert_allclose(eval_result["training"]["l2"], train_mses)
4078
4079


4080
@pytest.mark.parametrize("train_metric", [False, True])
4081
4082
4083
4084
4085
def test_record_evaluation_with_cv(train_metric):
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, y)
    eval_result = {}
    callbacks = [lgb.record_evaluation(eval_result)]
4086
4087
4088
4089
4090
4091
    metrics = ["l2", "rmse"]
    params = {"objective": "l2", "num_leaves": 3, "metric": metrics}
    cv_hist = lgb.cv(
        params, ds, num_boost_round=5, stratified=False, callbacks=callbacks, eval_train_metric=train_metric
    )
    expected_datasets = {"valid"}
4092
    if train_metric:
4093
        expected_datasets.add("train")
4094
4095
4096
    assert set(eval_result.keys()) == expected_datasets
    for dataset in expected_datasets:
        for metric in metrics:
4097
4098
4099
            for agg in ("mean", "stdv"):
                key = f"{dataset} {metric}-{agg}"
                np.testing.assert_allclose(cv_hist[key], eval_result[dataset][f"{metric}-{agg}"])
4100
4101
4102


def test_pandas_with_numpy_regular_dtypes():
4103
4104
4105
4106
    pd = pytest.importorskip("pandas")
    uints = ["uint8", "uint16", "uint32", "uint64"]
    ints = ["int8", "int16", "int32", "int64"]
    bool_and_floats = ["bool", "float16", "float32", "float64"]
4107
4108
4109
4110
    rng = np.random.RandomState(42)

    n_samples = 100
    # data as float64
4111
4112
4113
4114
4115
4116
4117
4118
    df = pd.DataFrame(
        {
            "x1": rng.randint(0, 2, n_samples),
            "x2": rng.randint(1, 3, n_samples),
            "x3": 10 * rng.randint(1, 3, n_samples),
            "x4": 100 * rng.randint(1, 3, n_samples),
        }
    )
4119
    df = df.astype(np.float64)
4120
    y = df["x1"] * (df["x2"] + df["x3"] + df["x4"])
4121
    ds = lgb.Dataset(df, y)
4122
    params = {"objective": "l2", "num_leaves": 31, "min_child_samples": 1}
4123
4124
4125
4126
    bst = lgb.train(params, ds, num_boost_round=5)
    preds = bst.predict(df)

    # test all features were used
4127
    assert bst.trees_to_dataframe()["split_feature"].nunique() == df.shape[1]
4128
4129
4130
4131
4132
4133
    # test the score is better than predicting the mean
    baseline = np.full_like(y, y.mean())
    assert mean_squared_error(y, preds) < mean_squared_error(y, baseline)

    # test all predictions are equal using different input dtypes
    for target_dtypes in [uints, ints, bool_and_floats]:
4134
        df2 = df.astype({f"x{i}": dtype for i, dtype in enumerate(target_dtypes, start=1)})
4135
4136
4137
4138
4139
4140
4141
4142
        assert df2.dtypes.tolist() == target_dtypes
        ds2 = lgb.Dataset(df2, y)
        bst2 = lgb.train(params, ds2, num_boost_round=5)
        preds2 = bst2.predict(df2)
        np.testing.assert_allclose(preds, preds2)


def test_pandas_nullable_dtypes():
4143
    pd = pytest.importorskip("pandas")
4144
    rng = np.random.RandomState(0)
4145
4146
4147
4148
4149
4150
4151
4152
    df = pd.DataFrame(
        {
            "x1": rng.randint(1, 3, size=100),
            "x2": np.linspace(-1, 1, 100),
            "x3": pd.arrays.SparseArray(rng.randint(0, 11, size=100)),
            "x4": rng.rand(100) < 0.5,
        }
    )
4153
    # introduce some missing values
4154
4155
4156
    df.loc[1, "x1"] = np.nan
    df.loc[2, "x2"] = np.nan
    df.loc[3, "x4"] = np.nan
4157
    # the previous line turns x3 into object dtype in recent versions of pandas
4158
4159
    df["x4"] = df["x4"].astype(np.float64)
    y = df["x1"] * df["x2"] + df["x3"] * (1 + df["x4"])
4160
4161
4162
    y = y.fillna(0)

    # train with regular dtypes
4163
    params = {"objective": "l2", "num_leaves": 31, "min_child_samples": 1}
4164
4165
4166
4167
4168
4169
    ds = lgb.Dataset(df, y)
    bst = lgb.train(params, ds, num_boost_round=5)
    preds = bst.predict(df)

    # convert to nullable dtypes
    df2 = df.copy()
4170
4171
4172
    df2["x1"] = df2["x1"].astype("Int32")
    df2["x2"] = df2["x2"].astype("Float64")
    df2["x4"] = df2["x4"].astype("boolean")
4173
4174
4175
4176
4177
4178
4179
4180

    # test training succeeds
    ds_nullable_dtypes = lgb.Dataset(df2, y)
    bst_nullable_dtypes = lgb.train(params, ds_nullable_dtypes, num_boost_round=5)
    preds_nullable_dtypes = bst_nullable_dtypes.predict(df2)

    trees_df = bst_nullable_dtypes.trees_to_dataframe()
    # test all features were used
4181
    assert trees_df["split_feature"].nunique() == df.shape[1]
4182
4183
4184
4185
4186
4187
    # test the score is better than predicting the mean
    baseline = np.full_like(y, y.mean())
    assert mean_squared_error(y, preds) < mean_squared_error(y, baseline)

    # test equal predictions
    np.testing.assert_allclose(preds, preds_nullable_dtypes)
4188
4189
4190
4191
4192


def test_boost_from_average_with_single_leaf_trees():
    # test data are taken from bug report
    # https://github.com/microsoft/LightGBM/issues/4708
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
    X = np.array(
        [
            [1021.0589, 1018.9578],
            [1023.85754, 1018.7854],
            [1024.5468, 1018.88513],
            [1019.02954, 1018.88513],
            [1016.79926, 1018.88513],
            [1007.6, 1018.88513],
        ],
        dtype=np.float32,
    )
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
    y = np.array([1023.8, 1024.6, 1024.4, 1023.8, 1022.0, 1014.4], dtype=np.float32)
    params = {
        "extra_trees": True,
        "min_data_in_bin": 1,
        "extra_seed": 7,
        "objective": "regression",
        "verbose": -1,
        "boost_from_average": True,
        "min_data_in_leaf": 1,
    }
    train_set = lgb.Dataset(X, y)
    model = lgb.train(params=params, train_set=train_set, num_boost_round=10)

    preds = model.predict(X)
    mean_preds = np.mean(preds)
    assert y.min() <= mean_preds <= y.max()
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245


def test_cegb_split_buffer_clean():
    # modified from https://github.com/microsoft/LightGBM/issues/3679#issuecomment-938652811
    # and https://github.com/microsoft/LightGBM/pull/5087
    # test that the ``splits_per_leaf_`` of CEGB is cleaned before training a new tree
    # which is done in the fix #5164
    # without the fix:
    #    Check failed: (best_split_info.left_count) > (0)

    R, C = 1000, 100
    seed = 29
    np.random.seed(seed)
    data = np.random.randn(R, C)
    for i in range(1, C):
        data[i] += data[0] * np.random.randn()

    N = int(0.8 * len(data))
    train_data = data[:N]
    test_data = data[N:]
    train_y = np.sum(train_data, axis=1)
    test_y = np.sum(test_data, axis=1)

    train = lgb.Dataset(train_data, train_y, free_raw_data=True)

    params = {
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
        "boosting_type": "gbdt",
        "objective": "regression",
        "max_bin": 255,
        "num_leaves": 31,
        "seed": 0,
        "learning_rate": 0.1,
        "min_data_in_leaf": 0,
        "verbose": -1,
        "min_split_gain": 1000.0,
        "cegb_penalty_feature_coupled": 5 * np.arange(C),
        "cegb_penalty_split": 0.0002,
        "cegb_tradeoff": 10.0,
        "force_col_wise": True,
4259
4260
4261
4262
4263
4264
    }

    model = lgb.train(params, train, num_boost_round=10)
    predicts = model.predict(test_data)
    rmse = np.sqrt(mean_squared_error(test_y, predicts))
    assert rmse < 10.0
4265
4266


4267
4268
4269
4270
def test_verbosity_and_verbose(capsys):
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, y)
    params = {
4271
4272
4273
        "num_leaves": 3,
        "verbose": 1,
        "verbosity": 0,
4274
4275
    }
    lgb.train(params, ds, num_boost_round=1)
4276
    expected_msg = "[LightGBM] [Warning] verbosity is set=0, verbose=1 will be ignored. " "Current value: verbosity=0"
4277
4278
4279
4280
    stdout = capsys.readouterr().out
    assert expected_msg in stdout


4281
4282
@pytest.mark.parametrize("verbosity_param", lgb.basic._ConfigAliases.get("verbosity"))
@pytest.mark.parametrize("verbosity", [-1, 0])
4283
4284
4285
4286
def test_verbosity_can_suppress_alias_warnings(capsys, verbosity_param, verbosity):
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, y)
    params = {
4287
4288
4289
4290
        "num_leaves": 3,
        "subsample": 0.75,
        "bagging_fraction": 0.8,
        "force_col_wise": True,
4291
4292
4293
4294
        verbosity_param: verbosity,
    }
    lgb.train(params, ds, num_boost_round=1)
    expected_msg = (
4295
4296
        "[LightGBM] [Warning] bagging_fraction is set=0.8, subsample=0.75 will be ignored. "
        "Current value: bagging_fraction=0.8"
4297
4298
4299
4300
4301
    )
    stdout = capsys.readouterr().out
    if verbosity >= 0:
        assert expected_msg in stdout
    else:
4302
        assert re.search(r"\[LightGBM\]", stdout) is None
4303
4304


4305
@pytest.mark.skipif(not PANDAS_INSTALLED, reason="pandas is not installed")
4306
4307
def test_validate_features():
    X, y = make_synthetic_regression()
4308
    features = ["x1", "x2", "x3", "x4"]
4309
4310
    df = pd_DataFrame(X, columns=features)
    ds = lgb.Dataset(df, y)
4311
    bst = lgb.train({"num_leaves": 15, "verbose": -1}, ds, num_boost_round=10)
4312
4313
4314
    assert bst.feature_name() == features

    # try to predict with a different feature
4315
    df2 = df.rename(columns={"x3": "z"})
4316
4317
4318
4319
4320
    with pytest.raises(lgb.basic.LightGBMError, match="Expected 'x3' at position 2 but found 'z'"):
        bst.predict(df2, validate_features=True)

    # check that disabling the check doesn't raise the error
    bst.predict(df2, validate_features=False)
4321
4322
4323
4324
4325
4326
4327

    # try to refit with a different feature
    with pytest.raises(lgb.basic.LightGBMError, match="Expected 'x3' at position 2 but found 'z'"):
        bst.refit(df2, y, validate_features=True)

    # check that disabling the check doesn't raise the error
    bst.refit(df2, y, validate_features=False)
4328
4329


4330
4331
4332
4333
4334
4335
4336
def test_train_and_cv_raise_informative_error_for_train_set_of_wrong_type():
    with pytest.raises(TypeError, match=r"train\(\) only accepts Dataset object, train_set has type 'list'\."):
        lgb.train({}, train_set=[])
    with pytest.raises(TypeError, match=r"cv\(\) only accepts Dataset object, train_set has type 'list'\."):
        lgb.cv({}, train_set=[])


4337
@pytest.mark.parametrize("num_boost_round", [-7, -1, 0])
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
def test_train_and_cv_raise_informative_error_for_impossible_num_boost_round(num_boost_round):
    X, y = make_synthetic_regression(n_samples=100)
    error_msg = rf"num_boost_round must be greater than 0\. Got {num_boost_round}\."
    with pytest.raises(ValueError, match=error_msg):
        lgb.train({}, train_set=lgb.Dataset(X, y), num_boost_round=num_boost_round)
    with pytest.raises(ValueError, match=error_msg):
        lgb.cv({}, train_set=lgb.Dataset(X, y), num_boost_round=num_boost_round)


def test_train_raises_informative_error_if_any_valid_sets_are_not_dataset_objects():
    X, y = make_synthetic_regression(n_samples=100)
    X_valid = X * 2.0
4350
4351
4352
    with pytest.raises(
        TypeError, match=r"Every item in valid_sets must be a Dataset object\. Item 1 has type 'tuple'\."
    ):
4353
4354
4355
        lgb.train(
            params={},
            train_set=lgb.Dataset(X, y),
4356
            valid_sets=[lgb.Dataset(X_valid, y), ([1.0], [2.0]), [5.6, 5.7, 5.8]],
4357
4358
4359
        )


4360
4361
def test_train_raises_informative_error_for_params_of_wrong_type():
    X, y = make_synthetic_regression()
4362
    params = {"num_leaves": "too-many"}
4363
    dtrain = lgb.Dataset(X, label=y)
4364
    with pytest.raises(lgb.basic.LightGBMError, match='Parameter num_leaves should be of type int, got "too-many"'):
4365
        lgb.train(params, dtrain)
4366
4367
4368
4369
4370


def test_quantized_training():
    X, y = make_synthetic_regression()
    ds = lgb.Dataset(X, label=y)
4371
    bst_params = {"num_leaves": 15, "verbose": -1, "seed": 0}
4372
4373
    bst = lgb.train(bst_params, ds, num_boost_round=10)
    rmse = np.sqrt(np.mean((bst.predict(X) - y) ** 2))
4374
4375
4376
4377
4378
4379
4380
    bst_params.update(
        {
            "use_quantized_grad": True,
            "num_grad_quant_bins": 30,
            "quant_train_renew_leaf": True,
        }
    )
4381
4382
4383
    quant_bst = lgb.train(bst_params, ds, num_boost_round=10)
    quant_rmse = np.sqrt(np.mean((quant_bst.predict(X) - y) ** 2))
    assert quant_rmse < rmse + 6.0