[ci] update CI stuff (#2079)

* updated boost submodule

* updated docker with new stable Clang and CMake

* switch to dev docker

* updated setup script

* updated MinGW on Appveyor

* updated Azure config to use docker for GPU task

* do not upgrade gcc - takes too long

* test: switch compilers

* switch compilers back

* get back to main docker

.appveyor.yml

@@ -15,7 +15,7 @@ clone_depth: 50
 install:
   - git submodule update --init --recursive  # get `compute` folder
   - set PATH=%PATH:C:\Program Files\Git\usr\bin;=%  # delete sh.exe from PATH (mingw32-make fix)
-  - set PATH=C:\mingw-w64\x86_64-7.3.0-posix-seh-rt_v5-rev0\mingw64\bin;%PATH%
+  - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
   - set PYTHON_VERSION=%CONFIGURATION%
   - ps: >-
       switch ($env:PYTHON_VERSION) {

.ci/dockers/ubuntu-14.04/Dockerfile

@@ -24,18 +24,19 @@ RUN apt-get update \
     build-essential \
  && rm -rf /var/lib/apt/lists/*
 
-# Install clang 6.0
+# Install clang 7.0
 RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y \
  && cd /tmp \
  && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
- && add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-6.0 main" -y \
+ && add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-7 main" -y \
  && apt-get update \
  && apt-get install -y --no-install-recommends \
-    clang-6.0 \
+    clang-7 \
+    libomp-7-dev \
  && rm -rf /var/lib/apt/lists/*
 
 # Install CMake
-RUN curl -sL https://cmake.org/files/v3.13/cmake-3.13.2-Linux-x86_64.sh -o cmake.sh \
+RUN curl -sL https://cmake.org/files/v3.14/cmake-3.14.1-Linux-x86_64.sh -o cmake.sh \
  && chmod +x cmake.sh \
  && ./cmake.sh --prefix=/usr/local --exclude-subdir \
  && rm cmake.sh

.ci/setup.sh

@@ -3,11 +3,10 @@
 if [[ $OS_NAME == "macos" ]]; then
     if  [[ $COMPILER == "clang" ]]; then
         brew install libomp
-        brew upgrade cmake  # CMake >=3.12 is needed to find OpenMP at macOS
         if [[ $AZURE == "true" ]]; then
             sudo xcode-select -s /Applications/Xcode_8.3.3.app/Contents/Developer
         fi
-    else
+    else  # gcc
         if [[ $TRAVIS == "true" ]]; then
 #            rm '/usr/local/include/c++'  # previous variant to deal with conflict link
 #            brew cask uninstall oclint  #  reserve variant to deal with conflict link
@@ -23,22 +22,18 @@ if [[ $OS_NAME == "macos" ]]; then
     wget -q -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh
 else  # Linux
     if [[ $AZURE == "true" ]] && [[ $COMPILER == "clang" ]]; then
-        sudo apt-get update
-        sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
-        sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
-        sudo apt-get install --no-install-recommends -y libomp-dev
+        sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-7 100
+        sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-7 100
     fi
     if [[ $TASK == "mpi" ]]; then
         sudo apt-get update
         sudo apt-get install --no-install-recommends -y libopenmpi-dev openmpi-bin
     fi
     if [[ $TASK == "gpu" ]]; then
-        if [[ $TRAVIS == "true" ]]; then
-            sudo add-apt-repository ppa:kzemek/boost -y
-        fi
+        sudo add-apt-repository ppa:mhier/libboost-latest -y
         sudo apt-get update
-        sudo apt-get install --no-install-recommends -y libboost1.58-dev libboost-system1.58-dev libboost-filesystem1.58-dev ocl-icd-opencl-dev
-        cd $HOME_DIRECTORY
+        sudo apt-get install --no-install-recommends -y libboost1.68-dev ocl-icd-opencl-dev
+        cd $BUILD_DIRECTORY  # to avoid permission errors
         wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
         tar -xjf AMD-APP-SDK*.tar.bz2
         mkdir -p $OPENCL_VENDOR_PATH
@@ -47,12 +42,12 @@ else  # Linux
         mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
         echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
     fi
-    if [[ $TRAVIS == "true" ]] || [[ $TASK == "gpu" ]]; then
+    if [[ $TRAVIS == "true" ]]; then
         wget -q -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh
     fi
 fi
 
-if [[ $TRAVIS == "true" ]] || [[ $OS_NAME == "macos" ]] || [[ $TASK == "gpu" ]]; then
+if [[ $TRAVIS == "true" ]] || [[ $OS_NAME == "macos" ]]; then
     sh conda.sh -b -p $CONDA
 fi
 conda config --set always_yes yes --set changeps1 no

.vsts-ci.yml

@@ -15,7 +15,7 @@ jobs:
     vmImage: 'ubuntu-16.04'
   container: ubuntu1404
   strategy:
-    maxParallel: 5
+    maxParallel: 6
     matrix:
       regular:
         TASK: regular
@@ -31,35 +31,6 @@ jobs:
         TASK: mpi
         METHOD: source
         PYTHON_VERSION: 2.7
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY"
-      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
-      echo "##vso[task.setvariable variable=OS_NAME]linux"
-      echo "##vso[task.setvariable variable=AZURE]true"
-      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
-      echo "##vso[task.prependpath]$CONDA/bin"
-    displayName: 'Set variables'
-  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
-    displayName: Setup
-  - bash: $(Build.SourcesDirectory)/.ci/test.sh
-    displayName: Test
-  - task: PublishBuildArtifacts@1
-    condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
-    inputs:
-      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-      artifactName: PackageAssets
-      artifactType: container
-###########################################
-- job: Linux_gpu
-###########################################
-  variables:
-    COMPILER: gcc
-  pool:
-    vmImage: 'ubuntu-16.04'
-  strategy:
-    maxParallel: 1
-    matrix:
       gpu_source:
         TASK: gpu
         METHOD: source
@@ -71,19 +42,23 @@ jobs:
       echo "##vso[task.setvariable variable=OS_NAME]linux"
       echo "##vso[task.setvariable variable=AZURE]true"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
-      AMDAPPSDK_PATH=$AGENT_HOMEDIRECTORY/AMDAPPSDK
+      echo "##vso[task.prependpath]$CONDA/bin"
+      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
       echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
       LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
       echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
       echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
-      CONDA=$AGENT_HOMEDIRECTORY/miniconda
-      echo "##vso[task.setvariable variable=CONDA]$CONDA"
-      echo "##vso[task.prependpath]$CONDA/bin"
     displayName: 'Set variables'
   - bash: $(Build.SourcesDirectory)/.ci/setup.sh
     displayName: Setup
   - bash: $(Build.SourcesDirectory)/.ci/test.sh
     displayName: Test
+  - task: PublishBuildArtifacts@1
+    condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
+    inputs:
+      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+      artifactName: PackageAssets
+      artifactType: container
 ###########################################
 - job: MacOS
 ###########################################

compute @ 36c89134

-Subproject commit 509ebe4a9282eec8a92c65ce3bbc1925f1fdbe07
+Subproject commit 36c89134d4013b2e5e45bc55656a18bd6141995a