[python] Windows wheels that support GPUs via OpenCL (#3403)

* Add specialized OpenCL/Python package build path

* Refer to upstream OpenCL repository

* Reset build job count in setup.py

* Run test for dual CPU/GPU code in CI

* TEMPORARY: refer to OpenCL fork to ensure Linux CI builds succeed

* Try switching to OpenCL 2.0

* Add --opencl-python-pacakge option to wheel

* Fix some typos.

* Remove intermediate cmake target

* Restrict OpenCL headers to documented API version

* Refresh the environment after installation

* Try testing with two processes

* Use command line definition to activate integrated build

* Try newer OpenCL runtime

* If we hit OpenCL code path that's good enough

* Make tests run correctly

* Correct directory

* Flag reference to unofficial repo with FIXME

* TEMPORARY: update private repo tag for dependency

* Further post-merge fixes.

* Fix attribute name.

* Reformat with black.

* Rewrite config.h before library compilation for Windows wheel

* Run dual cpu/gpu test once only for Windows bdist CI

* Avoid C API tests when building gpu library for wheel

* TEMPORARY -- add clinfo tool to vm for debugging

* Increase compilation logging

* Rename OpenCL registry parameter after driver installation

* Install legacy OpenCL runtime

* ...

* Remove config.h change, clinfo installation, latest opencl runtime reference

* Add explicit script exit

* revert to 18.1 intel driver

* Run test_dual.py in two different modes

* Revert "Run test_dual.py in two different modes"

This reverts commit cbfec02659816f0cb321e1184994a9c2f37d2a77.

* Remove unused expected failure test

* Install OpenCL in separate powershell script

* fix setup.py imports

* activate conda env after session update

* separate update functions; exit opencl installation on error

* nothing; start pipeline

* download from original source

* specify tls version for installer download

* eliminate variable from command

* download opencl installer in parts

* remove duplicate setup.py command class entry

* download smaller parts

* remove redundant bdist wheel class

* Import chocolatey modules and use RefreshEnv

* insert pipeline cache for opencl installer

* rerun to ensure pipeline cache hit

* add comments, test another pipeline cache hit

* remove cache task; download full installer file from new location

* try another installer download timing

* reinsert pipeline cache

* retain cache task; download full file on cache miss

* test cache hit again

* remove todo
Co-authored-by: TP Boudreau <tpboudreau@gmail.com>

.ci/install_opencl.ps1

+
+Write-Output "Installing OpenCL CPU platform"
+
+$cache = "$env:PIPELINE_WORKSPACE\opencl_windows-amd_cpu-v3_0_130_135"
+$installer = "AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe"
+
+if ($env:OPENCL_INSTALLER_FOUND -ne 'true') {
+  # Pipeline cache miss; download OpenCL platform installer executable into workspace cache
+
+  Write-Output "Downloading OpenCL platform installer"
+  Invoke-WebRequest -OutFile "$installer" -Uri "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$installer"
+
+  Write-Output "Caching OpenCL platform installer"
+  New-Item $cache -ItemType Directory | Out-Null
+  Move-Item -Path "$installer" -Destination "$cache\$installer" | Out-Null
+
+  if (Test-Path "$cache\$installer") {
+    Write-Output "Successfully downloaded OpenCL platform installer"
+  } else {
+    Write-Output "Unable to download OpenCL platform installer"
+    Write-Output "Setting EXIT"
+    $host.SetShouldExit(-1)
+    Exit -1
+  }
+}
+
+# Install OpenCL platform from installer executable expected in workspace cache
+
+Write-Output "Running OpenCL installer"
+Invoke-Command -ScriptBlock {Start-Process "$cache\$installer" -ArgumentList '/S /V"/quiet /norestart /passive /log opencl.log"' -Wait}
+
+$property = Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors
+if ($property -eq $null) {
+  Write-Output "Unable to install OpenCL CPU platform"
+  Write-Output "OpenCL installation log:"
+  Get-Content "opencl.log"
+  Write-Output "Setting EXIT"
+  $host.SetShouldExit(-1)
+  Exit -1
+} else {
+  Write-Output "Successfully installed OpenCL CPU platform"
+  Write-Output "Current OpenCL drivers:"
+  Write-Output $property
+}
+

.ci/test_windows.ps1

@@ -6,6 +6,11 @@ function Check-Output {
   }
 }
 
+# Import the Chocolatey profile module so that the RefreshEnv command
+# invoked below properly updates the current PowerShell session enviroment.
+$module = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
+Import-Module "$module" ; Check-Output $?
+
 # unify environment variables for Azure devops and AppVeyor
 if (Test-Path env:APPVEYOR) {
   $env:APPVEYOR = "true"
@@ -23,9 +28,9 @@ conda activate
 conda config --set always_yes yes --set changeps1 no
 conda update -q -y conda
 conda create -q -y -n $env:CONDA_ENV python=$env:PYTHON_VERSION joblib matplotlib numpy pandas psutil pytest python-graphviz scikit-learn scipy ; Check-Output $?
-conda activate $env:CONDA_ENV
 
 if ($env:TASK -eq "regular") {
+  conda activate $env:CONDA_ENV
   mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build
   cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $?
   cd $env:BUILD_SOURCESDIRECTORY/python-package
@@ -34,6 +39,7 @@ if ($env:TASK -eq "regular") {
   cp $env:BUILD_SOURCESDIRECTORY/Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY
 }
 elseif ($env:TASK -eq "sdist") {
+  conda activate $env:CONDA_ENV
   cd $env:BUILD_SOURCESDIRECTORY/python-package
   python setup.py sdist --formats gztar ; Check-Output $?
   cd dist; pip install @(Get-ChildItem *.gz) -v ; Check-Output $?
@@ -48,11 +54,17 @@ elseif ($env:TASK -eq "sdist") {
   cp $env:BUILD_SOURCESDIRECTORY/build/lightgbmlib.jar $env:BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_win.jar
 }
 elseif ($env:TASK -eq "bdist") {
+  RefreshEnv
+  Write-Output "Current OpenCL drivers:"
+  Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors
+
+  conda activate $env:CONDA_ENV
   cd $env:BUILD_SOURCESDIRECTORY/python-package
-  python setup.py bdist_wheel --plat-name=win-amd64 --python-tag py3 ; Check-Output $?
-  cd dist; pip install @(Get-ChildItem *.whl) ; Check-Output $?
+  python setup.py bdist_wheel --integrated-opencl --plat-name=win-amd64 --python-tag py3 ; Check-Output $?
+  cd dist; pip install --user @(Get-ChildItem *.whl) ; Check-Output $?
   cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
 } elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
+  conda activate $env:CONDA_ENV
   cd $env:BUILD_SOURCESDIRECTORY\python-package
   if ($env:COMPILER -eq "MINGW") {
     python setup.py install --mingw ; Check-Output $?
@@ -62,9 +74,13 @@ elseif ($env:TASK -eq "bdist") {
 }
 
 if (($env:TASK -eq "sdist") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
+  # cannot test C API with "sdist" task
   $tests = $env:BUILD_SOURCESDIRECTORY + "/tests/python_package_test"
+} elseif ($env:TASK -eq "bdist") {
+  $tests = $env:BUILD_SOURCESDIRECTORY + "/tests"
+  # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
+  $env:LIGHTGBM_TEST_DUAL_CPU_GPU = "1"
 } else {
-  # cannot test C API with "sdist" task
   $tests = $env:BUILD_SOURCESDIRECTORY + "/tests"
 }
 pytest $tests ; Check-Output $?

.vsts-ci.yml

@@ -124,6 +124,17 @@ jobs:
       Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
       Write-Host "##vso[task.setvariable variable=AZURE]true"
     displayName: 'Set Variables'
+  - task: Cache@2
+    inputs:
+      key: '"opencl.windows" | "amd.cpu" | "v3.0.130.135" | "1"'
+      path: $(Pipeline.Workspace)/opencl_windows-amd_cpu-v3_0_130_135
+      cacheHitVar: OPENCL_INSTALLER_FOUND
+    condition: eq(variables.TASK, 'bdist')
+    displayName: 'Cache OpenCL'
+  - script: |
+      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
+    condition: eq(variables.TASK, 'bdist')
+    displayName: 'Install OpenCL'
   - script: |
       cmd /c "conda init powershell"
       cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"

python-package/setup.py

@@ -101,14 +101,10 @@ def clear_path(path):
 
 def silent_call(cmd, raise_error=False, error_msg=''):
     try:
-        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         with open(LOG_PATH, "ab") as log:
-            log.write(output)
+            subprocess.check_call(cmd, stderr=log, stdout=log)
         return 0
     except Exception as err:
-        if isinstance(err, subprocess.CalledProcessError):
-            with open(LOG_PATH, "ab") as log:
-                log.write(err.output)
         if raise_error:
             raise Exception("\n".join((error_msg, LOG_NOTICE)))
         return 1
@@ -185,7 +181,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_cuda=False, use_mpi=False,
                 arch = "Win32" if bit32 else "x64"
                 vs_versions = ("Visual Studio 16 2019", "Visual Studio 15 2017", "Visual Studio 14 2015")
                 for vs in vs_versions:
-                    logger.info("Starting to compile with %s." % vs)
+                    logger.info("Starting to compile with %s (%s).", vs, arch)
                     status = silent_call(cmake_cmd + ["-G", vs, "-A", arch])
                     if status == 0:
                         break

tests/python_package_test/test_dual.py

+"""Tests for dual GPU+CPU support."""
+
+import os
+import pytest
+
+import lightgbm as lgb
+import numpy as np
+from lightgbm.basic import LightGBMError
+
+
+@pytest.mark.skipif(
+    os.environ.get("LIGHTGBM_TEST_DUAL_CPU_GPU", None) is None,
+    reason="Only run if appropriate env variable is set",
+)
+def test_cpu_works():
+    """If compiled appropriately, the same installation will support both GPU and CPU."""
+    data = np.random.rand(500, 10)
+    label = np.random.randint(2, size=500)
+    validation_data = train_data = lgb.Dataset(data, label=label)
+
+    param = {"verbosity": 2, "num_leaves": 31, "objective": "binary", "device": "cpu"}
+    gbm = lgb.train(param, train_data, 10, valid_sets=[validation_data])
+
+
+@pytest.mark.skipif(
+    os.environ.get("LIGHTGBM_TEST_DUAL_CPU_GPU", None) is None,
+    reason="Only run if appropriate env variable is set",
+)
+def test_gpu_works():
+    """If compiled appropriately, the same installation will support both GPU and CPU."""
+    data = np.random.rand(500, 10)
+    label = np.random.randint(2, size=500)
+    validation_data = train_data = lgb.Dataset(data, label=label)
+
+    param = {"verbosity": 2, "num_leaves": 31, "objective": "binary", "device": "gpu"}
+    gbm = lgb.train(param, train_data, 10, valid_sets=[validation_data])