[ci] merged CI setup scripts into one (#1669)

* reorganized CI files

* merged scripts into one

.vsts-ci/setup.sh → .ci/setup.sh

 #!/bin/bash
 
-if [[ $AGENT_OS == "Darwin" ]]; then
+if [[ $OS_NAME == "macos" ]]; then
     if  [[ $COMPILER == "clang" ]]; then
         brew install libomp
         brew reinstall cmake  # CMake >=3.12 is needed to find OpenMP at macOS
     else
+        if [[ $TRAVIS == "true" ]]; then
+            sudo softwareupdate -i "Command Line Tools (macOS High Sierra version 10.13) for Xcode-9.3"  # fix "fatal error: _stdio.h: No such file or directory"
+            rm '/usr/local/include/c++'
+#            brew cask uninstall oclint  #  reserve variant to deal with conflict link
+#            brew link --overwrite gcc  # previous variant to deal with conflict link
+        fi
         if [[ $TASK != "mpi" ]]; then
             brew install gcc
         fi
@@ -12,21 +18,29 @@ if [[ $AGENT_OS == "Darwin" ]]; then
     if [[ $TASK == "mpi" ]]; then
         brew install open-mpi
     fi
+    if [[ $TRAVIS == "true" ]]; then
+        wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh
+    fi
 else  # Linux
-    if [[ $COMPILER == "clang" ]]; then
+    if [[ $AZURE == "true" ]] && [[ $COMPILER == "clang" ]]; then
         update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
         update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
         sudo apt-get update
         sudo apt-get install libomp-dev
     fi
     if [[ $TASK == "mpi" ]]; then
-        sudo apt-get update
+        if [[ $AZURE == "true" ]]; then
+            sudo apt-get update
+        fi
         sudo apt-get install -y libopenmpi-dev openmpi-bin
     fi
     if [[ $TASK == "gpu" ]]; then
-        sudo apt-get update
-        sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev libboost-dev libboost-system-dev libboost-filesystem-dev
-        cd $AGENT_HOMEDIRECTORY
+        if [[ $AZURE == "true" ]]; then
+            sudo apt-get update
+            sudo apt-get install --no-install-recommends -y libboost-dev libboost-system-dev libboost-filesystem-dev
+        fi
+        sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev
+        cd $HOME_DIRECTORY
         wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
         tar -xjf AMD-APP-SDK*.tar.bz2
         mkdir -p $OPENCL_VENDOR_PATH
@@ -35,4 +49,13 @@ else  # Linux
         mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
         echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
     fi
+    if [[ $TRAVIS == "true" ]]; then
+        wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh
+    fi
+fi
+
+if [[ $TRAVIS == "true" ]]; then
+    sh conda.sh -b -p $HOME_DIRECTORY/miniconda
+    conda config --set always_yes yes --set changeps1 no
+    conda update -q conda
 fi

.travis.yml

@@ -38,6 +38,12 @@ matrix:
 before_install:
   - test -n $CC  && unset CC
   - test -n $CXX && unset CXX
+  - export HOME_DIRECTORY="$HOME"
+  - if [[ $TRAVIS_OS_NAME == "osx" ]]; then
+        export OS_NAME="macos";
+    else
+        export OS_NAME="linux";
+    fi
   - export PATH="$HOME/miniconda/bin:$PATH"
   - export LGB_VER=$(head -n 1 VERSION.txt)
   - export AMDAPPSDK_PATH=$HOME/AMDAPPSDK
@@ -46,7 +52,7 @@ before_install:
   - export OPENCL_VENDOR_PATH=$AMDAPPSDK_PATH/etc/OpenCL/vendors
 
 install:
-  - bash .travis/setup.sh
+  - bash .ci/setup.sh
 
 script:
   - bash .travis/test.sh

.travis/setup.sh

-#!/bin/bash
-
-if [[ $TRAVIS_OS_NAME == "osx" ]]; then
-    if  [[ $COMPILER == "clang" ]]; then
-        brew install libomp
-        brew reinstall cmake  # CMake >=3.12 is needed to find OpenMP at macOS
-    else
-        sudo softwareupdate -i "Command Line Tools (macOS High Sierra version 10.13) for Xcode-9.3"  # fix "fatal error: _stdio.h: No such file or directory"
-        rm '/usr/local/include/c++'
-#        brew cask uninstall oclint  #  reserve variant to deal with conflict link
-        if [[ $TASK != "mpi" ]]; then
-            brew install gcc
-        fi
-#        brew link --overwrite gcc  # previous variant to deal with conflict link
-    fi
-    if [[ $TASK == "mpi" ]]; then
-        brew install open-mpi
-    fi
-    wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh
-else  # Linux
-    if [[ $TASK == "mpi" ]]; then
-        sudo apt-get install -y libopenmpi-dev openmpi-bin
-    fi
-    if [[ $TASK == "gpu" ]]; then
-        sudo apt-get install -y ocl-icd-opencl-dev
-        wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
-        tar -xjf AMD-APP-SDK*.tar.bz2
-        mkdir -p $OPENCL_VENDOR_PATH
-        mkdir -p $AMDAPPSDK_PATH
-        sh AMD-APP-SDK*.sh --tar -xf -C $AMDAPPSDK_PATH
-        mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
-        echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
-    fi
-    wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh
-fi
-
-sh conda.sh -b -p $HOME/miniconda
-conda config --set always_yes yes --set changeps1 no
-conda update -q conda

.vsts-ci.yml

@@ -37,18 +37,21 @@ phases:
       packageSpecs: 'python=$(PYTHON_VERSION)'
       createOptions: '-q'
   - script: |
+      echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY"
+      echo "##vso[task.setvariable variable=OS_NAME]linux"
+      echo "##vso[task.setvariable variable=AZURE]true"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
       AMDAPPSDK_PATH=$AGENT_HOMEDIRECTORY/AMDAPPSDK
       echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
       LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
       echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
       echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
-      chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/setup.sh
-      chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/test.sh
+      chmod +x $BUILD_SOURCESDIRECTORY/.ci/setup.sh
+      chmod +x $BUILD_SOURCESDIRECTORY/.ci/test.sh
     displayName: 'Set variables'
-  - bash: $(Build.SourcesDirectory)/.vsts-ci/setup.sh
+  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
     displayName: Setup
-  - bash: $(Build.SourcesDirectory)/.vsts-ci/test.sh
+  - bash: $(Build.SourcesDirectory)/.ci/test.sh
     displayName: Test
   - task: PublishBuildArtifacts@1
     condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
@@ -84,13 +87,16 @@ phases:
       packageSpecs: 'python=$(PYTHON_VERSION)'
       createOptions: '-q'
   - script: |
+      echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY"
+      echo "##vso[task.setvariable variable=OS_NAME]macos"
+      echo "##vso[task.setvariable variable=AZURE]true"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
-      chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/setup.sh
-      chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/test.sh
+      chmod +x $BUILD_SOURCESDIRECTORY/.ci/setup.sh
+      chmod +x $BUILD_SOURCESDIRECTORY/.ci/test.sh
     displayName: 'Set variables'
-  - bash: $(Build.SourcesDirectory)/.vsts-ci/setup.sh
+  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
     displayName: Setup
-  - bash: $(Build.SourcesDirectory)/.vsts-ci/test.sh
+  - bash: $(Build.SourcesDirectory)/.ci/test.sh
     displayName: Test
   - task: PublishBuildArtifacts@1
     condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
@@ -121,7 +127,7 @@ phases:
       environmentName: $(CONDA_ENV)
       packageSpecs: 'python=$(PYTHON_VERSION) numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest'
       createOptions: '-q'
-  - powershell: $(Build.SourcesDirectory)/.vsts-ci/test_windows.ps1
+  - powershell: $(Build.SourcesDirectory)/.ci/test_windows.ps1
     displayName: Test
   - task: PublishBuildArtifacts@1
     condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))