Add CMake option to enable sanitizers and build gtest (#3555)

* Add CMake option to enable sanitizer

* Set up gtest

* Address reviewer's feedback

* Address reviewer's feedback

* Update CMakeLists.txt
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

.gitignore

@@ -269,6 +269,7 @@ _Pvt_Extensions
 *.app
 /windows/LightGBM.VC.db
 lightgbm
+/testlightgbm
 
 # Created by https://www.gitignore.io/api/python
 

CMakeLists.txt

@@ -6,6 +6,12 @@ OPTION(USE_HDFS "Enable HDFS support (EXPERIMENTAL)" OFF)
 OPTION(USE_TIMETAG "Set to ON to output time costs" OFF)
 OPTION(USE_CUDA "Enable CUDA-accelerated training (EXPERIMENTAL)" OFF)
 OPTION(USE_DEBUG "Set to ON for Debug mode" OFF)
+OPTION(USE_SANITIZER "Use santizer flags" OFF)
+SET(SANITIZER_PATH "" CACHE STRING "Path to sanitizer libs")
+SET(ENABLED_SANITIZERS "address" "leak" "undefined" CACHE STRING
+  "Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are
+address, leak, undefined and thread.")
+OPTION(BUILD_CPP_TEST "Build C++ tests with Google Test" OFF)
 OPTION(BUILD_STATIC_LIB "Build static library" OFF)
 OPTION(__BUILD_FOR_R "Set to ON if building lib_lightgbm for use with the R package" OFF)
 OPTION(__INTEGRATE_OPENCL "Set to ON if building LightGBM with the OpenCL ICD Loader and its dependencies included" OFF)
@@ -26,6 +32,14 @@ endif()
 
 PROJECT(lightgbm LANGUAGES C CXX)
 
+list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/modules")
+
+#-- Sanitizer
+if (USE_SANITIZER)
+  include(cmake/Sanitizer.cmake)
+  enable_sanitizers("${ENABLED_SANITIZERS}")
+endif (USE_SANITIZER)
+
 if(__INTEGRATE_OPENCL)
   set(__INTEGRATE_OPENCL ON CACHE BOOL "" FORCE)
   set(USE_GPU OFF CACHE BOOL "" FORCE)
@@ -451,6 +465,25 @@ if(__BUILD_FOR_R)
   endif(MSVC)
 endif(__BUILD_FOR_R)
 
+#-- Google C++ tests
+if(BUILD_CPP_TEST)
+  find_package(GTest CONFIG)
+  if(NOT GTEST_FOUND)
+    message(STATUS "Did not find Google Test in the system root. Fetching Google Test now...")
+    include(FetchContent)
+    FetchContent_Declare(
+      googletest
+      GIT_REPOSITORY https://github.com/google/googletest.git
+      GIT_TAG        release-1.10.0
+    )
+    FetchContent_MakeAvailable(googletest)
+    add_library(GTest::GTest ALIAS gtest)
+  endif()
+  file(GLOB CPP_TEST_SOURCES tests/cpp_test/*.cpp)
+  add_executable(testlightgbm ${CPP_TEST_SOURCES} ${SOURCES})
+  target_link_libraries(testlightgbm PRIVATE GTest::GTest)
+endif()
+
 install(TARGETS lightgbm _lightgbm
         RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
         LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib

cmake/Sanitizer.cmake

+# Set appropriate compiler and linker flags for sanitizers.
+#
+# Usage of this module:
+#  enable_sanitizers("address;leak")
+
+# Add flags
+macro(enable_sanitizer sanitizer)
+  if(${sanitizer} MATCHES "address")
+    find_package(ASan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=address")
+    link_libraries(${ASan_LIBRARY})
+
+  elseif(${sanitizer} MATCHES "thread")
+    find_package(TSan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=thread")
+    link_libraries(${TSan_LIBRARY})
+
+  elseif(${sanitizer} MATCHES "leak")
+    find_package(LSan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=leak")
+    link_libraries(${LSan_LIBRARY})
+
+  elseif(${sanitizer} MATCHES "undefined")
+    find_package(UBSan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=undefined -fno-sanitize-recover=undefined")
+    link_libraries(${UBSan_LIBRARY})
+
+  else()
+    message(FATAL_ERROR "Santizer ${sanitizer} not supported.")
+  endif()
+endmacro()
+
+macro(enable_sanitizers SANITIZERS)
+  # Check sanitizers compatibility.
+  foreach ( _san ${SANITIZERS} )
+    string(TOLOWER ${_san} _san)
+    if (_san MATCHES "thread")
+      if (${_use_other_sanitizers})
+        message(FATAL_ERROR
+          "thread sanitizer is not compatible with ${_san} sanitizer.")
+      endif()
+      set(_use_thread_sanitizer 1)
+    else ()
+      if (${_use_thread_sanitizer})
+        message(FATAL_ERROR
+          "${_san} sanitizer is not compatible with thread sanitizer.")
+      endif()
+      set(_use_other_sanitizers 1)
+    endif()
+  endforeach()
+
+  message(STATUS "Sanitizers: ${SANITIZERS}")
+
+  foreach( _san ${SANITIZERS} )
+    string(TOLOWER ${_san} _san)
+    enable_sanitizer(${_san})
+  endforeach()
+  message(STATUS "Sanitizers compile flags: ${SAN_COMPILE_FLAGS}")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_COMPILE_FLAGS}")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_COMPILE_FLAGS}")
+endmacro()

cmake/modules/FindASan.cmake

+set(ASan_LIB_NAME ASan)
+
+find_library(ASan_LIBRARY
+  NAMES libasan.so libasan.so.5 libasan.so.4 libasan.so.3 libasan.so.2 libasan.so.1 libasan.so.0 libasan.so.0.0.0
+  PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(ASan DEFAULT_MSG
+  ASan_LIBRARY)

cmake/modules/FindLSan.cmake

+set(LSan_LIB_NAME lsan)
+
+find_library(LSan_LIBRARY
+  NAMES liblsan.so liblsan.so.0 liblsan.so.0.0.0
+  PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LSan DEFAULT_MSG
+  LSan_LIBRARY)

cmake/modules/FindTSan.cmake

+set(TSan_LIB_NAME tsan)
+
+find_library(TSan_LIBRARY
+  NAMES libtsan.so libtsan.so.0 libtsan.so.0.0.0
+  PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(TSan DEFAULT_MSG
+  TSan_LIBRARY)

cmake/modules/FindUBSan.cmake

+set(UBSan_LIB_NAME UBSan)
+
+find_library(UBSan_LIBRARY
+  NAMES libubsan.so libubsan.so.1 libubsan.so.0 libubsan.so.0.0.0
+  PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(UBSan DEFAULT_MSG
+  UBSan_LIBRARY)

tests/cpp_test/test_main.cpp

+/*!
+ * Copyright (c) 2021 Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See LICENSE file in the project root for license information.
+ */
+#include <gtest/gtest.h>
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  testing::FLAGS_gtest_death_test_style = "threadsafe";
+  return RUN_ALL_TESTS();
+}