[tests][ci] run cpp tests with sanitizers on Linux and macOS (#4330)

* run cpp tests with sanitizers

* re-trigger CI

* continue

* small cleanup

* restore cpp test

.ci/test.sh

@@ -15,7 +15,15 @@ fi
 
 if [[ "$TASK" == "cpp-tests" ]]; then
     mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
-    cmake -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF ..
+    if [[ $METHOD == "with-sanitizers" ]]; then
+        extra_cmake_opts="-DUSE_SANITIZER=ON"
+        if [[ -n $SANITIZERS ]]; then
+            extra_cmake_opts="$extra_cmake_opts -DENABLED_SANITIZERS=$SANITIZERS"
+        fi
+    else
+        extra_cmake_opts=""
+    fi
+    cmake -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON $extra_cmake_opts ..
     make testlightgbm -j4 || exit -1
     ./../testlightgbm || exit -1
     exit 0

.ci/test_windows.ps1

@@ -18,9 +18,10 @@ if ($env:TASK -eq "r-package") {
 
 if ($env:TASK -eq "cpp-tests") {
   mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build
-  cmake -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -A x64 ..
+  cmake -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON -A x64 ..
   cmake --build . --target testlightgbm --config Debug ; Check-Output $?
-  Start-Process -FilePath "./../Debug/testlightgbm.exe" -NoNewWindow -Wait ; Check-Output $?
+  cd ../Debug
+  .\testlightgbm.exe ; Check-Output $?
   Exit 0
 }
 

.vsts-ci.yml

@@ -51,8 +51,6 @@ jobs:
         METHOD: source
       swig:
         TASK: swig
-      cpp_tests:
-        TASK: cpp-tests
   steps:
   - script: |
       echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
@@ -119,6 +117,9 @@ jobs:
         TASK: gpu
         METHOD: wheel
         PYTHON_VERSION: 3.7
+      cpp_tests:
+        TASK: cpp-tests
+        METHOD: with-sanitizers
   steps:
   - script: |
       echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
@@ -226,6 +227,8 @@ jobs:
         TASK: swig
       cpp_tests:
         TASK: cpp-tests
+        METHOD: with-sanitizers
+        SANITIZERS: "address;undefined"
   steps:
   - script: |
       echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"

CMakeLists.txt

@@ -7,7 +7,6 @@ OPTION(USE_TIMETAG "Set to ON to output time costs" OFF)
 OPTION(USE_CUDA "Enable CUDA-accelerated training (EXPERIMENTAL)" OFF)
 OPTION(USE_DEBUG "Set to ON for Debug mode" OFF)
 OPTION(USE_SANITIZER "Use santizer flags" OFF)
-SET(SANITIZER_PATH "" CACHE STRING "Path to sanitizer libs")
 SET(ENABLED_SANITIZERS "address" "leak" "undefined" CACHE STRING
   "Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are
 address, leak, undefined and thread.")
@@ -36,9 +35,12 @@ list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/modules")
 
 #-- Sanitizer
 if (USE_SANITIZER)
+  if(MSVC)
+    message(FATAL_ERROR "Sanitizers are not supported with MSVC.")
+  endif(MSVC)
   include(cmake/Sanitizer.cmake)
   enable_sanitizers("${ENABLED_SANITIZERS}")
-endif (USE_SANITIZER)
+endif(USE_SANITIZER)
 
 if(__INTEGRATE_OPENCL)
   set(__INTEGRATE_OPENCL ON CACHE BOOL "" FORCE)

cmake/Sanitizer.cmake

@@ -24,16 +24,16 @@ endmacro()
 
 macro(enable_sanitizers SANITIZERS)
   # Check sanitizers compatibility.
-  foreach ( _san ${SANITIZERS} )
+  foreach(_san ${SANITIZERS})
     string(TOLOWER ${_san} _san)
-    if (_san MATCHES "thread")
-      if (${_use_other_sanitizers})
+    if(_san MATCHES "thread")
+      if(${_use_other_sanitizers})
         message(FATAL_ERROR
           "thread sanitizer is not compatible with ${_san} sanitizer.")
       endif()
       set(_use_thread_sanitizer 1)
-    else ()
-      if (${_use_thread_sanitizer})
+    else()
+      if(${_use_thread_sanitizer})
         message(FATAL_ERROR
           "${_san} sanitizer is not compatible with thread sanitizer.")
       endif()
@@ -43,7 +43,7 @@ macro(enable_sanitizers SANITIZERS)
 
   message(STATUS "Sanitizers: ${SANITIZERS}")
 
-  foreach( _san ${SANITIZERS} )
+  foreach(_san ${SANITIZERS})
     string(TOLOWER ${_san} _san)
     enable_sanitizer(${_san})
   endforeach()