[tests][python] added test for huge string model (#1964)

* added test for huge string model

* fixed tree sizes field

* simplified model structure

* fixed test and added try/except

.appveyor.yml

@@ -30,7 +30,7 @@ install:
   - ps: $env:LGB_VER = (Get-Content VERSION.txt).trim()
   - conda config --set always_yes yes --set changeps1 no
   - conda update -q conda
-  - conda create -q -n test-env python=%PYTHON_VERSION% matplotlib nose numpy pandas pytest python-graphviz scikit-learn scipy
+  - conda create -q -n test-env python=%PYTHON_VERSION% matplotlib nose numpy pandas psutil pytest python-graphviz scikit-learn scipy
   - activate test-env
 
 build_script:

.ci/test.sh

@@ -58,7 +58,7 @@ if [[ $TASK == "if-else" ]]; then
     exit 0
 fi
 
-conda install -q -y -n $CONDA_ENV matplotlib nose numpy pandas pytest python-graphviz scikit-learn scipy
+conda install -q -y -n $CONDA_ENV matplotlib nose numpy pandas psutil pytest python-graphviz scikit-learn scipy
 
 if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
     sudo ln -sf `ls -d "$(brew --cellar libomp)"/*/lib`/* $CONDA_PREFIX/lib || exit -1  # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)

.vsts-ci.yml

@@ -146,7 +146,7 @@ jobs:
       createCustomEnvironment: true
       updateConda: true
       environmentName: $(CONDA_ENV)
-      packageSpecs: 'python=$(PYTHON_VERSION) matplotlib nose numpy pandas pytest python-graphviz scikit-learn scipy'
+      packageSpecs: 'python=$(PYTHON_VERSION) matplotlib nose numpy pandas psutil pytest python-graphviz scikit-learn scipy'
       createOptions: '-q'
   - powershell: $(Build.SourcesDirectory)/.ci/test_windows.ps1
     displayName: Test

tests/python_package_test/test_engine.py

@@ -3,6 +3,7 @@
 import copy
 import math
 import os
+import psutil
 import random
 import unittest
 
@@ -1208,3 +1209,30 @@ class TestEngine(unittest.TestCase):
         # binary metric with non-default num_class for custom objective
         self.assertRaises(lgb.basic.LightGBMError, get_cv_result,
                           params_class_3_verbose, metrics='binary_error', fobj=custom_obj)
+
+    @unittest.skipIf(psutil.virtual_memory().available / 1024 / 1024 / 1024 < 3, 'not enough RAM')
+    def test_model_size(self):
+        X, y = load_boston(True)
+        data = lgb.Dataset(X, y)
+        bst = lgb.train({'verbose': -1}, data, num_boost_round=2)
+        y_pred = bst.predict(X)
+        model_str = bst.model_to_string()
+        one_tree = model_str[model_str.find('Tree=1'):model_str.find('end of trees')]
+        one_tree_size = len(one_tree)
+        one_tree = one_tree.replace('Tree=1', 'Tree={}')
+        multiplier = 100
+        total_trees = multiplier + 2
+        try:
+            new_model_str = (model_str[:model_str.find('tree_sizes')]
+                             + '\n\n'
+                             + model_str[model_str.find('Tree=0'):model_str.find('end of trees')]
+                             + (one_tree * multiplier).format(*range(2, total_trees))
+                             + model_str[model_str.find('end of trees'):]
+                             + ' ' * (2**31 - one_tree_size * total_trees))
+            self.assertGreater(len(new_model_str), 2**31)
+            bst.model_from_string(new_model_str, verbose=False)
+            self.assertEqual(bst.num_trees(), total_trees)
+            y_pred_new = bst.predict(X, num_iteration=2)
+            np.testing.assert_allclose(y_pred, y_pred_new)
+        except MemoryError:
+            self.skipTest('not enough RAM')