[ci] rebalanced macOS tests (#1665)

* refined macos tests

* hotfix

* use the same naming for env variables

.vsts-ci.yml

@@ -59,31 +59,39 @@ phases:
 ###########################################
 - phase: MacOS
 ###########################################
+  variables:
+    COMPILER: gcc
   queue:
     name: 'Hosted macOS Preview'
     parallel: 3
     matrix:
       regular:
         TASK: regular
-        PYTHON_VERSION: 2.7
+        PYTHON_VERSION: 3.6
       sdist:
         TASK: sdist
-        PYTHON_VERSION: 3.4
+        PYTHON_VERSION: 3.5
       bdist:
         TASK: bdist
-        PYTHON_VERSION: 3.5
   steps:
   - script: |
-      brew install libomp
-      brew reinstall cmake
-      wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh
-      bash conda.sh -b -p $HOME/miniconda
-      export PATH=$HOME/miniconda/bin:$PATH
-      conda config --set always_yes yes --set changeps1 no
-      conda create -q -n $CONDA_ENV python=$PYTHON_VERSION
-      source activate $CONDA_ENV
-      export LGB_VER=$(head -n 1 VERSION.txt)
-      bash .vsts-ci/test.sh
+      sudo chmod -R 777 /usr/local/miniconda/envs
+    displayName: 'Fix for conda error on macOS'
+  - task: CondaEnvironment@0
+    inputs:
+      updateConda: true
+      environmentName: $(CONDA_ENV)
+      packageSpecs: 'python=$(PYTHON_VERSION)'
+      createOptions: '-q'
+  - script: |
+      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
+      chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/setup.sh
+      chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/test.sh
+    displayName: 'Set variables'
+  - bash: $(Build.SourcesDirectory)/.vsts-ci/setup.sh
+    displayName: Setup
+  - bash: $(Build.SourcesDirectory)/.vsts-ci/test.sh
+    displayName: Test
   - task: PublishBuildArtifacts@1
     condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
     inputs:

.vsts-ci/setup.sh

 #!/bin/bash
 
-if [[ $COMPILER == "clang" ]]; then
+if [[ $AGENT_OS == "Darwin" ]]; then
+    if  [[ $COMPILER == "clang" ]]; then
+        brew install libomp
+        brew reinstall cmake  # CMake >=3.12 is needed to find OpenMP at macOS
+    else
+        if [[ $TASK != "mpi" ]]; then
+            brew install gcc
+        fi
+    fi
+    if [[ $TASK == "mpi" ]]; then
+        brew install open-mpi
+    fi
+else  # Linux
+    if [[ $COMPILER == "clang" ]]; then
         update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
         update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
         sudo apt-get update
         sudo apt-get install libomp-dev
-fi
-if [[ $TASK == "mpi" ]]; then
+    fi
+    if [[ $TASK == "mpi" ]]; then
         sudo apt-get update
         sudo apt-get install -y libopenmpi-dev openmpi-bin
-fi
-if [[ $TASK == "gpu" ]]; then
+    fi
+    if [[ $TASK == "gpu" ]]; then
         sudo apt-get update
         sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev libboost-dev libboost-system-dev libboost-filesystem-dev
         cd $AGENT_HOMEDIRECTORY
@@ -21,4 +34,5 @@ if [[ $TASK == "gpu" ]]; then
         sh AMD-APP-SDK*.sh --tar -xf -C $AMDAPPSDK_PATH
         mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
         echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
+    fi
 fi

.vsts-ci/test.sh

 #!/bin/bash
 
-if [[ $AGENT_OS == "Linux" ]] && [[ $COMPILER == "clang" ]]; then
+if [[ $AGENT_OS == "Darwin" ]] && [[ $COMPILER == "gcc" ]]; then
+    export CXX=g++-8
+    export CC=gcc-8
+elif [[ $AGENT_OS == "Linux" ]] && [[ $COMPILER == "clang" ]]; then
     export CXX=clang++
     export CC=clang
 fi
@@ -24,7 +27,7 @@ fi
 
 conda install -q -y -n $CONDA_ENV numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest
 
-if [[ $AGENT_OS == "Darwin" ]] ; then
+if [[ $AGENT_OS == "Darwin" ]] && [[ $COMPILER == "clang" ]]; then
     ln -sf `ls -d "$(brew --cellar libomp)"/*/lib`/* $CONDA_PREFIX/lib || exit -1  # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
 fi
 
@@ -36,15 +39,15 @@ if [[ $TASK == "sdist" ]]; then
     exit 0
 elif [[ $TASK == "bdist" ]]; then
     if [[ $AGENT_OS == "Darwin" ]]; then
-        cd ${BUILD_REPOSITORY_LOCALPATH}/python-package && python setup.py bdist_wheel --plat-name=macdarwin --universal || exit -1
-        cp dist/lightgbm-$LGB_VER-py2.py3-none-macdarwin.whl ${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl
+        cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py bdist_wheel --plat-name=macdarwin --universal || exit -1
         mv dist/lightgbm-$LGB_VER-py2.py3-none-macdarwin.whl dist/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl
+        cp dist/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
     else
         cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py bdist_wheel --plat-name=manylinux1_x86_64 --universal || exit -1
         cp dist/lightgbm-$LGB_VER-py2.py3-none-manylinux1_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
     fi
     pip install $BUILD_SOURCESDIRECTORY/python-package/dist/*.whl || exit -1
-    pytest $BUILD_SOURCESDIRECTORY/tests/python_package_test || exit -1
+    pytest $BUILD_SOURCESDIRECTORY/tests || exit -1
     exit 0
 fi
 
@@ -82,7 +85,7 @@ pytest $BUILD_SOURCESDIRECTORY/tests || exit -1
 
 if [[ $TASK == "regular" ]]; then
     if [[ $AGENT_OS == "Darwin" ]]; then
-        cp ${BUILD_REPOSITORY_LOCALPATH}/lib_lightgbm.so ${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.dylib
+        cp $BUILD_SOURCESDIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
     else
         cp $BUILD_SOURCESDIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
     fi