[R-package][ci] Add test on R package with sanitizers (#3439)

* [ci] add R CI job with UBSAN

* stuff

* fix command

* update template

* fail on errors

* spaces

* trigger by comment

* stuff

* add all CI back

* trying things

* run sanitizers as a regular job

* remove comments

* sanitizers

* try to trigger UBSAN

* try to trigger failure

* remove int overflow

* remove testing change

* Apply suggestions from code review
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

.github/workflows/r_package.yml

@@ -159,10 +159,35 @@ jobs:
           $env:TASK = "${{ matrix.task }}"
           conda init powershell
           & "$env:GITHUB_WORKSPACE/.ci/test_windows.ps1"
+  test-r-sanitizers:
+    name: r-package (ubuntu-latest, R-devel, GCC ASAN/UBSAN)
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    container:
+      image: rhub/rocker-gcc-san
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 5
+          submodules: true
+      - name: Install packages
+        shell: bash
+        run: |
+          Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org')"
+          sh build-cran-package.sh
+          Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1
+      - name: Run tests with sanitizers
+        shell: bash
+        run: |
+          cd R-package/tests
+          Rscriptdevel testthat.R 2>&1 > ubsan-tests.log
+          cat ubsan-tests.log
+          exit $(cat ubsan-tests.log | grep "runtime error" | wc -l)
   all-successful:
     # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert
     runs-on: ubuntu-latest
-    needs: [test]
+    needs: [test, test-r-sanitizers]
     steps:
     - name: Note that all tests succeeded
       run: echo "🎉"