[ci] [python-package] temporarily stop testing against scikit-learn nightlies,...

[ci] [python-package] temporarily stop testing against scikit-learn nightlies, load lib_lightgbm earlier (#6654)

.ci/conda-envs/ci-core.txt

@@ -23,7 +23,7 @@ joblib>=1.3.2
 matplotlib-base>=3.7.3
 numpy>=1.24.4
 pandas>2.0
-pyarrow>=6.0
+pyarrow-core>=6.0
 python-graphviz>=0.20.3
 scikit-learn>=1.3.2
 scipy>=1.1

.ci/test-python-latest.sh

@@ -22,7 +22,7 @@ python -m pip install \
         'numpy>=2.0.0.dev0' \
         'matplotlib>=3.10.0.dev0' \
         'pandas>=3.0.0.dev0' \
-        'scikit-learn>=1.6.dev0' \
+        'scikit-learn==1.5.*' \
         'scipy>=1.15.0.dev0'
 
 python -m pip install \

docs/FAQ.rst

@@ -206,6 +206,49 @@ Detailed description of conflicts between multiple OpenMP instances is provided
 
 If this is not your case, then you should find conflicting OpenMP library installations on your own and leave only one of them.
 
+17. Loading LightGBM fails like: ``cannot allocate memory in static TLS block``
+-------------------------------------------------------------------------------
+
+When loading LightGBM, you may encounter errors like the following.
+
+.. code-block:: console
+
+   lib/libgomp.so.1: cannot allocate memory in static TLS block
+
+This most commonly happens on aarch64 Linux systems.
+
+``gcc``'s OpenMP library (``libgomp.so``) tries to allocate a small amount of static thread-local storage ("TLS")
+when it's dynamically loaded.
+
+That error can happen when the loader isn't able to find a large enough block of memory.
+
+On aarch64 Linux, processes and loaded libraries share the same pool of static TLS,
+which makes such failures more likely. See these discussions:
+
+* https://bugzilla.redhat.com/show_bug.cgi?id=1722181#c6
+* https://gcc.gcc.gnu.narkive.com/vOXMQqLA/failure-to-dlopen-libgomp-due-to-static-tls-data
+
+If you are experiencing this issue when using the ``lightgbm`` Python package, try upgrading
+to at least ``v4.6.0``.
+
+For older versions of the Python package, or for other LightGBM APIs, this issue can
+often be avoided by loading ``libgomp.so.1``. That can be done directly by setting environment
+variable ``LD_PRELOAD``, like this:
+
+.. code-block:: console
+
+    export LD_PRELOAD=/root/miniconda3/envs/test-env/lib/libgomp.so.1
+
+It can also be done indirectly by changing the order that other libraries are loaded
+into processes, which varies by programming language and application type.
+
+For more details, see these discussions:
+
+* https://github.com/microsoft/LightGBM/pull/6654#issuecomment-2352014275
+* https://github.com/microsoft/LightGBM/issues/6509
+* https://maskray.me/blog/2021-02-14-all-about-thread-local-storage
+* https://bugzilla.redhat.com/show_bug.cgi?id=1722181#c6
+
 ------
 
 R-package

python-package/lightgbm/__init__.py

@@ -6,6 +6,8 @@ Contributors: https://github.com/microsoft/LightGBM/graphs/contributors.
 
 from pathlib import Path
 
+# .basic is intentionally loaded as early as possible, to dlopen() lib_lightgbm.{dll,dylib,so}
+# and its dependencies as early as possible
 from .basic import Booster, Dataset, Sequence, register_logger
 from .callback import EarlyStopException, early_stopping, log_evaluation, record_evaluation, reset_parameter
 from .engine import CVBooster, cv, train

python-package/lightgbm/basic.py

 # coding: utf-8
 """Wrapper for C API of LightGBM."""
 
+# This import causes lib_lightgbm.{dll,dylib,so} to be loaded.
+# It's intentionally done here, as early as possible, to avoid issues like
+# "libgomp.so.1: cannot allocate memory in static TLS block" on aarch64 Linux.
+#
+# For details, see the "cannot allocate memory in static TLS block" entry in docs/FAQ.rst.
+from .libpath import _LIB  # isort: skip
+
 import abc
 import ctypes
 import inspect
@@ -37,7 +44,6 @@ from .compat import (
     pd_DataFrame,
     pd_Series,
 )
-from .libpath import find_lib_path
 
 if TYPE_CHECKING:
     from typing import Literal
@@ -160,6 +166,12 @@ ZERO_THRESHOLD = 1e-35
 _MULTICLASS_OBJECTIVES = {"multiclass", "multiclassova", "multiclass_ova", "ova", "ovr", "softmax"}
 
 
+class LightGBMError(Exception):
+    """Error thrown by LightGBM."""
+
+    pass
+
+
 def _is_zero(x: float) -> bool:
     return -ZERO_THRESHOLD <= x <= ZERO_THRESHOLD
 
@@ -259,26 +271,13 @@ def _log_callback(msg: bytes) -> None:
     _log_native(str(msg.decode("utf-8")))
 
 
-def _load_lib() -> ctypes.CDLL:
-    """Load LightGBM library."""
-    lib_path = find_lib_path()
-    lib = ctypes.cdll.LoadLibrary(lib_path[0])
-    lib.LGBM_GetLastError.restype = ctypes.c_char_p
+# connect the Python logger to logging in lib_lightgbm
+if not environ.get("LIGHTGBM_BUILD_DOC", False):
+    _LIB.LGBM_GetLastError.restype = ctypes.c_char_p
     callback = ctypes.CFUNCTYPE(None, ctypes.c_char_p)
-    lib.callback = callback(_log_callback)  # type: ignore[attr-defined]
-    if lib.LGBM_RegisterLogCallback(lib.callback) != 0:
-        raise LightGBMError(lib.LGBM_GetLastError().decode("utf-8"))
-    return lib
-
-
-# we don't need lib_lightgbm while building docs
-_LIB: ctypes.CDLL
-if environ.get("LIGHTGBM_BUILD_DOC", False):
-    from unittest.mock import Mock  # isort: skip
-
-    _LIB = Mock(ctypes.CDLL)  # type: ignore
-else:
-    _LIB = _load_lib()
+    _LIB.callback = callback(_log_callback)  # type: ignore[attr-defined]
+    if _LIB.LGBM_RegisterLogCallback(_LIB.callback) != 0:
+        raise LightGBMError(_LIB.LGBM_GetLastError().decode("utf-8"))
 
 
 _NUMERIC_TYPES = (int, float, bool)
@@ -552,12 +551,6 @@ class _TempFile:
             self.path.unlink()
 
 
-class LightGBMError(Exception):
-    """Error thrown by LightGBM."""
-
-    pass
-
-
 # DeprecationWarning is not shown by default, so let's create our own with higher level
 # ref: https://peps.python.org/pep-0565/#additional-use-case-for-futurewarning
 class LGBMDeprecationWarning(FutureWarning):

python-package/lightgbm/libpath.py

 # coding: utf-8
 """Find the path to LightGBM dynamic library files."""
 
+import ctypes
+from os import environ
 from pathlib import Path
 from platform import system
 from typing import List
@@ -8,7 +10,7 @@ from typing import List
 __all__: List[str] = []
 
 
-def find_lib_path() -> List[str]:
+def _find_lib_path() -> List[str]:
     """Find the path to LightGBM library files.
 
     Returns
@@ -35,3 +37,13 @@ def find_lib_path() -> List[str]:
         dll_path_joined = "\n".join(map(str, dll_path))
         raise Exception(f"Cannot find lightgbm library file in following paths:\n{dll_path_joined}")
     return lib_path
+
+
+# we don't need lib_lightgbm while building docs
+_LIB: ctypes.CDLL
+if environ.get("LIGHTGBM_BUILD_DOC", False):
+    from unittest.mock import Mock  # isort: skip
+
+    _LIB = Mock(ctypes.CDLL)  # type: ignore
+else:
+    _LIB = ctypes.cdll.LoadLibrary(_find_lib_path()[0])