[ci] speed up the linux builds (#1745)

* speed up the build

* remove gpu docker

* add boost ppa

* remove duplicate installs

* Update test.sh

* conda install mkl qt

* addd windows

* Update .vsts-ci.yml

* fix gpu boost

.ci/dockers/ubuntu-14.04/Dockerfile

+FROM microsoft/vsts-agent:ubuntu-14.04
+
+# Install basic command-line utilities
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+    curl \
+    locales \
+    sudo \
+    unzip \
+    wget \
+    zip \
+    graphviz \
+ && rm -rf /var/lib/apt/lists/*
+
+# Setup the locale
+ENV LANG en_US.UTF-8
+ENV LC_ALL $LANG
+RUN locale-gen $LANG \
+ && update-locale
+
+# Install essential build tools
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+    build-essential \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install CMake
+RUN curl -sL https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh -o cmake.sh \
+ && chmod +x cmake.sh \
+ && ./cmake.sh --prefix=/usr/local --exclude-subdir \
+ && rm cmake.sh
+
+# Install Miniconda
+RUN curl -sL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh \
+ && chmod +x miniconda.sh \
+ && ./miniconda.sh -b -p /usr/local/miniconda \
+ && rm miniconda.sh \
+ && chmod -R 777 /usr/local/miniconda \
+ && /usr/local/miniconda/bin/conda install python=3 -q -y \
+ && /usr/local/miniconda/bin/conda install mkl qt -q -y \
+ && /usr/local/miniconda/bin/conda clean -a -y
+ENV CONDA=/usr/local/miniconda
+
+# Clean system
+RUN apt-get clean \
+ && rm -rf /var/lib/apt/lists/* \
+ && rm -rf /etc/apt/sources.list.d/* \
+ && rm -rf /tmp/*

.ci/setup.sh

@@ -25,28 +25,22 @@ if [[ $OS_NAME == "macos" ]]; then
         wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh
     fi
 else  # Linux
-    sudo apt-get update
     if [[ $AZURE == "true" ]] && [[ $COMPILER == "clang" ]]; then
+        sudo apt-get update
         sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
         sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
         sudo apt-get install libomp-dev
-    elif [[ $AZURE == "true" ]] && [[ $COMPILER == "gcc" ]] && [[ $TASK != "gpu" ]]; then
-        # downgrade gcc version
-        sudo apt-get remove -y gcc || exit -1
-        sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
-        sudo apt-get update
-        sudo apt-get install --no-install-recommends -y g++-4.8 || exit -1
-        sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100
-        sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100
     fi
     if [[ $TASK == "mpi" ]]; then
+        sudo apt-get update
         sudo apt-get install --no-install-recommends -y libopenmpi-dev openmpi-bin
     fi
     if [[ $TASK == "gpu" ]]; then
-        if [[ $AZURE == "true" ]]; then
-            sudo apt-get install --no-install-recommends -y libboost-dev libboost-system-dev libboost-filesystem-dev
+        if [[ $TRAVIS == "true" ]]; then
+            sudo add-apt-repository ppa:kzemek/boost -y
         fi
-        sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev
+        sudo apt-get update
+        sudo apt-get install --no-install-recommends -y libboost1.58-dev libboost-system1.58-dev libboost-filesystem1.58-dev ocl-icd-opencl-dev
         cd $HOME_DIRECTORY
         wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
         tar -xjf AMD-APP-SDK*.tar.bz2

.ci/test.sh

@@ -32,6 +32,7 @@ if [[ $TRAVIS == "true" ]] && [[ $TASK == "check-docs" ]]; then
     sed -i'.bak' -e 's;\(\.\/[^.]*\.\)rst\([^[:space:]]*\);\1html\2;g' {} \;  # emulate js function
 #    html5validator --root ./_build/html/ || exit -1
     if [[ $OS_NAME == "linux" ]]; then
+        sudo apt-get update
         sudo apt-get install linkchecker
         linkchecker --config=.linkcheckerrc ./_build/html/*.html || exit -1
     fi
@@ -53,9 +54,9 @@ fi
 
 if [[ $TASK == "if-else" ]]; then
     conda install -y -n $CONDA_ENV numpy
-    mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build && cmake .. && make lightgbm || exit -1
+    mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build && cmake .. && make lightgbm -j4 || exit -1
     cd $BUILD_DIRECTORY/tests/cpp_test && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
-    cd $BUILD_DIRECTORY/build && make lightgbm || exit -1
+    cd $BUILD_DIRECTORY/build && make lightgbm -j4 || exit -1
     cd $BUILD_DIRECTORY/tests/cpp_test && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit -1
     exit 0
 fi
@@ -94,18 +95,11 @@ elif [[ $TASK == "bdist" ]]; then
 fi
 
 if [[ $TASK == "gpu" ]]; then
-    if [[ $TRAVIS == "true" ]]; then
-        conda install -y -n $CONDA_ENV -c conda-forge boost
-    fi
     sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h
     grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
     if [[ $METHOD == "pip" ]]; then
         cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
-        if [[ $AZURE == "true" ]]; then
-            pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
-        else
-            pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--boost-root=$CONDA_PREFIX" --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
-        fi
+        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
         exit 0
     fi
@@ -122,16 +116,12 @@ if [[ $TASK == "mpi" ]]; then
     fi
     cmake -DUSE_MPI=ON ..
 elif [[ $TASK == "gpu" ]]; then
-    if [[ $AZURE == "true" ]]; then
-        cmake -DUSE_GPU=ON -DOpenCL_INCLUDE_DIR=$AMDAPPSDK_PATH/include/ ..
-    else
-        cmake -DUSE_GPU=ON -DBOOST_ROOT=$CONDA_PREFIX -DOpenCL_INCLUDE_DIR=$AMDAPPSDK_PATH/include/ ..
-    fi
+    cmake -DUSE_GPU=ON -DOpenCL_INCLUDE_DIR=$AMDAPPSDK_PATH/include/ ..
 else
     cmake ..
 fi
 
-make _lightgbm || exit -1
+make _lightgbm -j4 || exit -1
 
 cd $BUILD_DIRECTORY/python-package && python setup.py install --precompile --user || exit -1
 pytest $BUILD_DIRECTORY/tests || exit -1

.ci/test_windows.ps1

 function Check-Output {
- param( [int]$ExitCode )
- if ($ExitCode -ne 0) {
+  param( [int]$ExitCode )
+  if ($ExitCode -ne 0) {
     $host.SetShouldExit($ExitCode)
     Exit -1
   }

.vsts-ci.yml

 variables:
   PYTHON_VERSION: 3.7
   CONDA_ENV: test-env
+resources:
+  containers:
+  - container: ubuntu1404
+    image: lightgbm/vsts-agent:ubuntu-14.04
 jobs:
 ###########################################
 - job: Linux
@@ -9,8 +13,9 @@ jobs:
     COMPILER: gcc
   pool:
     vmImage: 'ubuntu-16.04'
+  container: ubuntu1404
   strategy:
-    maxParallel: 6
+    maxParallel: 5
     matrix:
       regular:
         TASK: regular
@@ -26,6 +31,37 @@ jobs:
         TASK: mpi
         METHOD: source
         PYTHON_VERSION: 2.7
+  steps:
+  - task: CondaEnvironment@1
+    inputs:
+      updateConda: true
+  - script: |
+      echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY"
+      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
+      echo "##vso[task.setvariable variable=OS_NAME]linux"
+      echo "##vso[task.setvariable variable=AZURE]true"
+      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
+    displayName: 'Set variables'
+  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
+    displayName: Setup
+  - bash: $(Build.SourcesDirectory)/.ci/test.sh
+    displayName: Test
+  - task: PublishBuildArtifacts@1
+    condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
+    inputs:
+      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+      artifactName: PackageAssets
+      artifactType: container
+###########################################
+- job: Linux_gpu
+###########################################
+  variables:
+    COMPILER: gcc
+  pool:
+    vmImage: 'ubuntu-16.04'
+  strategy:
+    maxParallel: 1
+    matrix:
       gpu_source:
         TASK: gpu
         METHOD: source
@@ -50,12 +86,6 @@ jobs:
     displayName: Setup
   - bash: $(Build.SourcesDirectory)/.ci/test.sh
     displayName: Test
-  - task: PublishBuildArtifacts@1
-    condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
-    inputs:
-      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-      artifactName: PackageAssets
-      artifactType: container
 ###########################################
 - job: MacOS
 ###########################################
@@ -118,7 +148,7 @@ jobs:
       createCustomEnvironment: true
       updateConda: true
       environmentName: $(CONDA_ENV)
-      packageSpecs: 'python=$(PYTHON_VERSION) numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest'
+      packageSpecs: 'python=$(PYTHON_VERSION) numpy nose matplotlib python-graphviz pytest scipy scikit-learn pandas'
       createOptions: '-q'
   - powershell: $(Build.SourcesDirectory)/.ci/test_windows.ps1
     displayName: Test

python-package/setup.py

@@ -162,7 +162,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, use_hdfs=False,
     else:  # Linux, Darwin (macOS), etc.
         logger.info("Starting to compile with CMake.")
         silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first')
-        silent_call(["make", "_lightgbm"], raise_error=True,
+        silent_call(["make", "_lightgbm", "-j4"], raise_error=True,
                     error_msg='An error has occurred while building lightgbm library file')
     os.chdir(CURRENT_DIR)