[ci] Build integrated OpenCL Linux wheels (#5252)

* Add integrated OpenCL build on Linux

* Build integrated OpenCL Linux wheel in CI

* Fix test_dual.py on Linux arm64

* Enable integrated OpenCL Linux wheel arm64 testing in CI

* Update documentation

* Add comment about gpu_use_dp

* add missing fi dropped in merge conflict resolution

* install opencl-headers on bdist task

* use new CI image for x86_64

* update check_dynamic_dependencies script

* use main CI image
Co-authored-by: James Lamb <jaylamb20@gmail.com>

.ci/setup.sh

@@ -79,8 +79,7 @@ else  # Linux
             sudo apt-get update
             sudo apt-get install --no-install-recommends -y \
                 libboost1.74-dev \
-                ocl-icd-opencl-dev \
-                pocl-opencl-icd
+                ocl-icd-opencl-dev
         else  # in manylinux image
             sudo yum update -y
             sudo yum install -y \
@@ -90,6 +89,47 @@ else  # Linux
             || exit -1
         fi
     fi
+    if [[ $TASK == "gpu" || $TASK == "bdist" ]]; then
+        if [[ $IN_UBUNTU_LATEST_CONTAINER == "true" ]]; then
+            sudo apt-get update
+            sudo apt-get install --no-install-recommends -y \
+                pocl-opencl-icd
+        elif [[ $(uname -m) == "aarch64" ]]; then
+            yum install -y \
+                epel-release \
+                gcc-c++ \
+                hwloc-devel \
+                sudo
+            yum install -y \
+                llvm-toolset-7.0-clang-devel \
+                llvm-toolset-7.0-llvm-devel \
+                ocl-icd-devel
+            git clone --depth 1 --branch v1.8 https://github.com/pocl/pocl.git
+            cmake \
+              -B pocl/build \
+              -S pocl \
+              -DCMAKE_BUILD_TYPE=release \
+              -DCMAKE_C_COMPILER=/usr/bin/gcc \
+              -DCMAKE_CXX_COMPILER=/usr/bin/g++ \
+              -DCMAKE_C_FLAGS=-std=gnu99 \
+              -DPOCL_INSTALL_ICD_VENDORDIR=/etc/OpenCL/vendors \
+              -DPOCL_DEBUG_MESSAGES=OFF \
+              -DINSTALL_OPENCL_HEADERS=OFF \
+              -DENABLE_SPIR=OFF \
+              -DENABLE_POCLCC=OFF \
+              -DENABLE_TESTS=OFF \
+              -DENABLE_EXAMPLES=OFF \
+              -DLLC_HOST_CPU=generic
+            cmake --build pocl/build -j4
+            sudo cmake --install pocl/build
+        elif [[ $(uname -m) == "x86_64" ]]; then
+            sudo yum update -y
+            sudo yum install -y \
+                ocl-icd-devel \
+                opencl-headers \
+            || exit -1
+        fi
+    fi
     if [[ $TASK == "cuda" || $TASK == "cuda_exp" ]]; then
         echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
         apt-get update

.ci/test.sh

@@ -161,10 +161,12 @@ elif [[ $TASK == "bdist" ]]; then
         else
             PLATFORM="manylinux2014_$ARCH"
         fi
-        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --plat-name=$PLATFORM --python-tag py3 || exit -1
+        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --integrated-opencl --plat-name=$PLATFORM --python-tag py3 || exit -1
         if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
             cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
         fi
+        # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
+        export LIGHTGBM_TEST_DUAL_CPU_GPU=1
     fi
     pip install --user $BUILD_DIRECTORY/python-package/dist/*.whl || exit -1
     pytest $BUILD_DIRECTORY/tests || exit -1

.vsts-ci.yml

@@ -187,7 +187,8 @@ jobs:
       EOF
       cat > docker-script.sh <<EOF
       export CONDA=\$HOME/miniforge
-      export PATH=\$CONDA/bin:\$PATH
+      export PATH=\$CONDA/bin:/opt/rh/llvm-toolset-7.0/root/usr/bin:\$PATH
+      export LD_LIBRARY_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64:\$LD_LIBRARY_PATH
       $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
       $ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
       EOF

CMakeLists.txt

@@ -184,11 +184,11 @@ if(USE_GPU)
 endif()
 
 if(__INTEGRATE_OPENCL)
-    if(WIN32)
+    if(APPLE)
+        message(FATAL_ERROR "Integrated OpenCL build is not available on macOS")
+    else()
         include(cmake/IntegratedOpenCL.cmake)
         add_definitions(-DUSE_GPU)
-    else()
-        message(FATAL_ERROR "Integrated OpenCL build is available only for Windows")
     endif()
 endif()
 
@@ -544,7 +544,7 @@ if(__INTEGRATE_OPENCL)
   # variables INTEGRATED_OPENCL_* are set in IntegratedOpenCL.cmake
   target_include_directories(lightgbm_objs PRIVATE ${INTEGRATED_OPENCL_INCLUDES})
   target_compile_definitions(lightgbm_objs PRIVATE ${INTEGRATED_OPENCL_DEFINITIONS})
-  target_link_libraries(lightgbm_objs PUBLIC ${INTEGRATED_OPENCL_LIBRARIES})
+  target_link_libraries(lightgbm_objs PUBLIC ${INTEGRATED_OPENCL_LIBRARIES} ${CMAKE_DL_LIBS})
 endif()
 
 if(USE_CUDA OR USE_CUDA_EXP)

cmake/IntegratedOpenCL.cmake

@@ -26,28 +26,45 @@ FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY
 FetchContent_GetProperties(OpenCL-ICD-Loader)
 if(NOT OpenCL-ICD-Loader_POPULATED)
   FetchContent_Populate(OpenCL-ICD-Loader)
-  set(USE_DYNAMIC_VCXX_RUNTIME ON)
+  if(WIN32)
+    set(USE_DYNAMIC_VCXX_RUNTIME ON)
+  endif()
   add_subdirectory(${opencl-icd-loader_SOURCE_DIR} ${opencl-icd-loader_BINARY_DIR} EXCLUDE_FROM_ALL)
   message(STATUS "Populated OpenCL ICD Loader")
 endif()
 list(APPEND INTEGRATED_OPENCL_INCLUDES ${OPENCL_ICD_LOADER_HEADERS_DIR})
-list(
-  APPEND
-  INTEGRATED_OPENCL_LIBRARIES
-    ${opencl-icd-loader_BINARY_DIR}/Release/OpenCL.lib
-    cfgmgr32.lib
-    runtimeobject.lib
-)
 list(APPEND INTEGRATED_OPENCL_DEFINITIONS CL_TARGET_OPENCL_VERSION=120)
+if(WIN32)
+  list(
+    APPEND
+    INTEGRATED_OPENCL_LIBRARIES
+      ${opencl-icd-loader_BINARY_DIR}/Release/OpenCL.lib
+      cfgmgr32.lib
+      runtimeobject.lib
+  )
+else()
+  list(
+    APPEND
+    INTEGRATED_OPENCL_LIBRARIES
+      ${opencl-icd-loader_BINARY_DIR}/libOpenCL.a
+  )
+  set_property(TARGET OpenCL PROPERTY POSITION_INDEPENDENT_CODE ON)
+endif()
 
 # Build Independent Boost libraries
 include(ExternalProject)
 include(ProcessorCount)
 ProcessorCount(J)
 set(BOOST_BASE "${PROJECT_BINARY_DIR}/Boost")
-set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.bat")
-set(BOOST_BUILD "${BOOST_BASE}/source/b2.exe")
-set(BOOST_FLAGS "")
+if(WIN32)
+  set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.bat")
+  set(BOOST_BUILD "${BOOST_BASE}/source/b2.exe")
+  set(BOOST_FLAGS "")
+else()
+  set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.sh")
+  set(BOOST_BUILD "${BOOST_BASE}/source/b2")
+  set(BOOST_FLAGS "-fPIC")
+endif()
 list(
   APPEND
   BOOST_SUBMODULES
@@ -137,35 +154,37 @@ ExternalProject_Add(
 set(BOOST_INCLUDE "${BOOST_BASE}/source" CACHE PATH "")
 set(BOOST_LIBRARY "${BOOST_BASE}/source/stage/lib" CACHE PATH "")
 list(APPEND INTEGRATED_OPENCL_INCLUDES ${BOOST_INCLUDE})
-if(MSVC)
-  if(${MSVC_VERSION} GREATER 1929)
-    message(FATAL_ERROR "Unrecognized MSVC version number: ${MSVC_VERSION}")
-  elseif(${MSVC_VERSION} GREATER 1919)
-    set(MSVC_TOOLCHAIN_ID "142")
-  elseif(${MSVC_VERSION} GREATER 1909)
-    set(MSVC_TOOLCHAIN_ID "141")
-  elseif(${MSVC_VERSION} GREATER 1899)
-    set(MSVC_TOOLCHAIN_ID "140")
+if(WIN32)
+  if(MSVC)
+    if(${MSVC_VERSION} GREATER 1929)
+      message(FATAL_ERROR "Unrecognized MSVC version number: ${MSVC_VERSION}")
+    elseif(${MSVC_VERSION} GREATER 1919)
+      set(MSVC_TOOLCHAIN_ID "142")
+    elseif(${MSVC_VERSION} GREATER 1909)
+      set(MSVC_TOOLCHAIN_ID "141")
+    elseif(${MSVC_VERSION} GREATER 1899)
+      set(MSVC_TOOLCHAIN_ID "140")
+    else()
+      message(FATAL_ERROR "Unrecognized MSVC version number: ${MSVC_VERSION}")
+    endif()
+    list(
+      APPEND
+      INTEGRATED_OPENCL_LIBRARIES
+      ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib
+      ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib
+      ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib
+    )
   else()
-    message(FATAL_ERROR "Unrecognized MSVC version number: ${MSVC_VERSION}")
+    message(FATAL_ERROR "Integrated OpenCL build is not yet available for MinGW")
   endif()
+else()
   list(
     APPEND
     INTEGRATED_OPENCL_LIBRARIES
-    ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib
-  )
-  list(
-    APPEND
-    INTEGRATED_OPENCL_LIBRARIES
-    ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib
-  )
-  list(
-    APPEND
-    INTEGRATED_OPENCL_LIBRARIES
-    ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib
+    ${BOOST_LIBRARY}/libboost_filesystem.a
+    ${BOOST_LIBRARY}/libboost_system.a
+    ${BOOST_LIBRARY}/libboost_chrono.a
   )
-else()
-  message(FATAL_ERROR "Integrated OpenCL build is not yet available for MinGW")
 endif()
 
 set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE)

python-package/README.rst

@@ -22,7 +22,7 @@ Install from `PyPI <https://pypi.org/project/lightgbm>`_
 
 You may need to install `wheel <https://pythonwheels.com>`_ via ``pip install wheel`` first.
 
-Compiled library that is included in the wheel file supports both **GPU** and **CPU** versions out of the box. This feature is experimental and available only for **Windows** currently. To use **GPU** version you only need to install OpenCL Runtime libraries. For NVIDIA and AMD GPU they are included in the ordinary drivers for your graphics card, so no action is required. If you would like your AMD or Intel CPU to act like a GPU (for testing and debugging) you can install `AMD APP SDK <https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe>`_.
+Compiled library that is included in the wheel file supports both **GPU** and **CPU** versions out of the box. This feature is experimental and available only for **Windows** and **Linux** currently. To use **GPU** version you only need to install OpenCL Runtime libraries. For NVIDIA and AMD GPU they are included in the ordinary drivers for your graphics card, so no action is required. If you would like your AMD or Intel CPU to act like a GPU (for testing and debugging) you can install `AMD APP SDK <https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe>`_ on **Windows** and `PoCL <http://portablecl.org>`_ on **Linux**. Many modern Linux distributions provide packages for PoCL, look for ``pocl-opencl-icd`` on Debian-based distributions and ``pocl`` on RedHat-based distributions.
 
 For **Windows** users, `VC runtime <https://support.microsoft.com/en-us/help/2977003/the-latest-supported-visual-c-downloads>`_ is needed if **Visual Studio** (2015 or newer) is not installed.
 

tests/python_package_test/test_dual.py

@@ -2,6 +2,7 @@
 """Tests for dual GPU+CPU support."""
 
 import os
+import platform
 
 import pytest
 from sklearn.metrics import log_loss
@@ -26,9 +27,11 @@ def test_cpu_and_gpu_work():
 
     params_gpu = params_cpu.copy()
     params_gpu["device"] = "gpu"
-    params_gpu["gpu_use_dp"] = True
+    # Double-precision floats are only supported on x86_64 with PoCL
+    params_gpu["gpu_use_dp"] = (platform.machine() == "x86_64")
     gpu_bst = lgb.train(params_gpu, data, num_boost_round=10)
     gpu_score = log_loss(y, gpu_bst.predict(X))
 
-    assert cpu_score == pytest.approx(gpu_score)
+    rel = 1e-6 if params_gpu["gpu_use_dp"] else 1e-4
+    assert cpu_score == pytest.approx(gpu_score, rel=rel)
     assert gpu_score < 0.242