[ci] rebalanced macOS tests (#1665)

* refined macos tests * hotfix * use the same naming for env variables

[ci] rebalanced macOS tests (#1665)
* refined macos tests * hotfix * use the same naming for env variables
e1b5901f · Nikita Titov · Guolin Ke · e17a28ec · e1b5901f · e1b5901f
Commit e1b5901f authored Sep 14, 2018 by Nikita Titov Committed by Guolin Ke Sep 14, 2018
Hide whitespace changes
Inline Side-by-side

Showing with 65 additions and 40 deletions

.vsts-ci.yml .vsts-ci.yml +21 -13

.vsts-ci/setup.sh .vsts-ci/setup.sh +35 -21

.vsts-ci/test.sh .vsts-ci/test.sh +9 -6

No files found.
--- a/.vsts-ci.yml
+++ b/.vsts-ci.yml
@@ -59,31 +59,39 @@ phases:
 ###########################################
 - phase: MacOS
 ###########################################
+  variables:
+    COMPILER: gcc
  queue:
    name: 'Hosted macOS Preview'
    parallel: 3
    matrix:
      regular:
        TASK: regular
-        PYTHON_VERSION: 2.7
+        PYTHON_VERSION: 3.6
      sdist:
        TASK: sdist
-        PYTHON_VERSION: 3.4
+        PYTHON_VERSION: 3.5
      bdist:
        TASK: bdist
-        PYTHON_VERSION: 3.5
  steps:
  - script: |
-      brew install libomp
-      brew reinstall cmake
-      wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh
-      bash conda.sh -b -p $HOME/miniconda
-      export PATH=$HOME/miniconda/bin:$PATH
-      conda config --set always_yes yes --set changeps1 no
-      conda create -q -n $CONDA_ENV python=$PYTHON_VERSION
-      source activate $CONDA_ENV
-      export LGB_VER=$(head -n 1 VERSION.txt)
-      bash .vsts-ci/test.sh
+      sudo chmod -R 777 /usr/local/miniconda/envs
+    displayName: 'Fix for conda error on macOS'
+  - task: CondaEnvironment@0
+    inputs:
+      updateConda: true
+      environmentName: $(CONDA_ENV)
+      packageSpecs: 'python=$(PYTHON_VERSION)'
+      createOptions: '-q'
+  - script: |
+      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
+      chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/setup.sh
+      chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/test.sh
+    displayName: 'Set variables'
+  - bash: $(Build.SourcesDirectory)/.vsts-ci/setup.sh
+    displayName: Setup
+  - bash: $(Build.SourcesDirectory)/.vsts-ci/test.sh
+    displayName: Test
  - task: PublishBuildArtifacts@1
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
    inputs:

--- a/.vsts-ci/setup.sh
+++ b/.vsts-ci/setup.sh
 #!/bin/bash

-if [[ $COMPILER == "clang" ]]; then
-    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
-    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
-    sudo apt-get update
-    sudo apt-get install libomp-dev
-fi
-if [[ $TASK == "mpi" ]]; then
-    sudo apt-get update
-    sudo apt-get install -y libopenmpi-dev openmpi-bin
-fi
-if [[ $TASK == "gpu" ]]; then
-    sudo apt-get update
-    sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev libboost-dev libboost-system-dev libboost-filesystem-dev
-    cd $AGENT_HOMEDIRECTORY
-    wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
-    tar -xjf AMD-APP-SDK*.tar.bz2
-    mkdir -p $OPENCL_VENDOR_PATH
-    mkdir -p $AMDAPPSDK_PATH
-    sh AMD-APP-SDK*.sh --tar -xf -C $AMDAPPSDK_PATH
-    mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
-    echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
+if [[ $AGENT_OS == "Darwin" ]]; then
+    if  [[ $COMPILER == "clang" ]]; then
+        brew install libomp
+        brew reinstall cmake  # CMake >=3.12 is needed to find OpenMP at macOS
+    else
+        if [[ $TASK != "mpi" ]]; then
+            brew install gcc
+        fi
+    fi
+    if [[ $TASK == "mpi" ]]; then
+        brew install open-mpi
+    fi
+else  # Linux
+    if [[ $COMPILER == "clang" ]]; then
+        update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
+        update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
+        sudo apt-get update
+        sudo apt-get install libomp-dev
+    fi
+    if [[ $TASK == "mpi" ]]; then
+        sudo apt-get update
+        sudo apt-get install -y libopenmpi-dev openmpi-bin
+    fi
+    if [[ $TASK == "gpu" ]]; then
+        sudo apt-get update
+        sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev libboost-dev libboost-system-dev libboost-filesystem-dev
+        cd $AGENT_HOMEDIRECTORY
+        wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
+        tar -xjf AMD-APP-SDK*.tar.bz2
+        mkdir -p $OPENCL_VENDOR_PATH
+        mkdir -p $AMDAPPSDK_PATH
+        sh AMD-APP-SDK*.sh --tar -xf -C $AMDAPPSDK_PATH
+        mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
+        echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
+    fi
 fi
--- a/.vsts-ci/test.sh
+++ b/.vsts-ci/test.sh
 #!/bin/bash

-if [[ $AGENT_OS == "Linux" ]] && [[ $COMPILER == "clang" ]]; then
+if [[ $AGENT_OS == "Darwin" ]] && [[ $COMPILER == "gcc" ]]; then
+    export CXX=g++-8
+    export CC=gcc-8
+elif [[ $AGENT_OS == "Linux" ]] && [[ $COMPILER == "clang" ]]; then
    export CXX=clang++
    export CC=clang
 fi
@@ -24,7 +27,7 @@ fi

 conda install -q -y -n $CONDA_ENV numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest

-if [[ $AGENT_OS == "Darwin" ]] ; then
+if [[ $AGENT_OS == "Darwin" ]] && [[ $COMPILER == "clang" ]]; then
    ln -sf `ls -d "$(brew --cellar libomp)"/*/lib`/* $CONDA_PREFIX/lib || exit -1  # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
 fi

@@ -36,15 +39,15 @@ if [[ $TASK == "sdist" ]]; then
    exit 0
 elif [[ $TASK == "bdist" ]]; then
    if [[ $AGENT_OS == "Darwin" ]]; then
-        cd ${BUILD_REPOSITORY_LOCALPATH}/python-package && python setup.py bdist_wheel --plat-name=macdarwin --universal || exit -1
-        cp dist/lightgbm-$LGB_VER-py2.py3-none-macdarwin.whl ${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl
+        cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py bdist_wheel --plat-name=macdarwin --universal || exit -1
        mv dist/lightgbm-$LGB_VER-py2.py3-none-macdarwin.whl dist/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl
+        cp dist/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
    else
        cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py bdist_wheel --plat-name=manylinux1_x86_64 --universal || exit -1
        cp dist/lightgbm-$LGB_VER-py2.py3-none-manylinux1_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
    fi
    pip install $BUILD_SOURCESDIRECTORY/python-package/dist/*.whl || exit -1
-    pytest $BUILD_SOURCESDIRECTORY/tests/python_package_test || exit -1
+    pytest $BUILD_SOURCESDIRECTORY/tests || exit -1
    exit 0
 fi

@@ -82,7 +85,7 @@ pytest $BUILD_SOURCESDIRECTORY/tests || exit -1

 if [[ $TASK == "regular" ]]; then
    if [[ $AGENT_OS == "Darwin" ]]; then
-        cp ${BUILD_REPOSITORY_LOCALPATH}/lib_lightgbm.so ${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.dylib
+        cp $BUILD_SOURCESDIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
    else
        cp $BUILD_SOURCESDIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
    fi