Skip to content

GitLab

"...flagembedding_pytorch.git" did not exist on "f75058c7f49c1e567703363c18de9bc67bdbc1a4"
Commit e1b5901f authored by Nikita Titov's avatar Nikita Titov Committed by Guolin Ke
Browse files

[ci] rebalanced macOS tests (#1665) 

* refined macos tests

* hotfix

* use the same naming for env variables
parent e17a28ec
Hide whitespace changes
Inline Side-by-side
Showing with 65 additions and 40 deletions
.vsts-ci.yml
View file @ e1b5901f
...@@ -59,31 +59,39 @@ phases: ...@@ -59,31 +59,39 @@ phases:
########################################### ###########################################
- phase: MacOS - phase: MacOS
########################################### ###########################################
variables:
COMPILER: gcc
queue: queue:
name: 'Hosted macOS Preview' name: 'Hosted macOS Preview'
parallel: 3 parallel: 3
matrix: matrix:
regular: regular:
TASK: regular TASK: regular
PYTHON_VERSION: 2.7 PYTHON_VERSION: 3.6
sdist: sdist:
TASK: sdist TASK: sdist
PYTHON_VERSION: 3.4 PYTHON_VERSION: 3.5
bdist: bdist:
TASK: bdist TASK: bdist
PYTHON_VERSION: 3.5
steps: steps:
- script: | - script: |
brew install libomp sudo chmod -R 777 /usr/local/miniconda/envs
brew reinstall cmake displayName: 'Fix for conda error on macOS'
wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh - task: CondaEnvironment@0
bash conda.sh -b -p $HOME/miniconda inputs:
export PATH=$HOME/miniconda/bin:$PATH updateConda: true
conda config --set always_yes yes --set changeps1 no environmentName: $(CONDA_ENV)
conda create -q -n $CONDA_ENV python=$PYTHON_VERSION packageSpecs: 'python=$(PYTHON_VERSION)'
source activate $CONDA_ENV createOptions: '-q'
export LGB_VER=$(head -n 1 VERSION.txt) - script: |
bash .vsts-ci/test.sh echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/setup.sh
chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/test.sh
displayName: 'Set variables'
- bash: $(Build.SourcesDirectory)/.vsts-ci/setup.sh
displayName: Setup
- bash: $(Build.SourcesDirectory)/.vsts-ci/test.sh
displayName: Test
- task: PublishBuildArtifacts@1 - task: PublishBuildArtifacts@1
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
inputs: inputs:
......
.vsts-ci/setup.sh
View file @ e1b5901f
#!/bin/bash #!/bin/bash
if [[ $COMPILER == "clang" ]]; then if [[ $AGENT_OS == "Darwin" ]]; then
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100 if [[ $COMPILER == "clang" ]]; then
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100 brew install libomp
sudo apt-get update brew reinstall cmake # CMake >=3.12 is needed to find OpenMP at macOS
sudo apt-get install libomp-dev else
fi if [[ $TASK != "mpi" ]]; then
if [[ $TASK == "mpi" ]]; then brew install gcc
sudo apt-get update fi
sudo apt-get install -y libopenmpi-dev openmpi-bin fi
fi if [[ $TASK == "mpi" ]]; then
if [[ $TASK == "gpu" ]]; then brew install open-mpi
sudo apt-get update fi
sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev libboost-dev libboost-system-dev libboost-filesystem-dev else # Linux
cd $AGENT_HOMEDIRECTORY if [[ $COMPILER == "clang" ]]; then
wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2 update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
tar -xjf AMD-APP-SDK*.tar.bz2 update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
mkdir -p $OPENCL_VENDOR_PATH sudo apt-get update
mkdir -p $AMDAPPSDK_PATH sudo apt-get install libomp-dev
sh AMD-APP-SDK*.sh --tar -xf -C $AMDAPPSDK_PATH fi
mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/ if [[ $TASK == "mpi" ]]; then
echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd sudo apt-get update
sudo apt-get install -y libopenmpi-dev openmpi-bin
fi
if [[ $TASK == "gpu" ]]; then
sudo apt-get update
sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev libboost-dev libboost-system-dev libboost-filesystem-dev
cd $AGENT_HOMEDIRECTORY
wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
tar -xjf AMD-APP-SDK*.tar.bz2
mkdir -p $OPENCL_VENDOR_PATH
mkdir -p $AMDAPPSDK_PATH
sh AMD-APP-SDK*.sh --tar -xf -C $AMDAPPSDK_PATH
mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
fi
fi fi
.vsts-ci/test.sh
View file @ e1b5901f
#!/bin/bash #!/bin/bash
if [[ $AGENT_OS == "Linux" ]] && [[ $COMPILER == "clang" ]]; then if [[ $AGENT_OS == "Darwin" ]] && [[ $COMPILER == "gcc" ]]; then
export CXX=g++-8
export CC=gcc-8
elif [[ $AGENT_OS == "Linux" ]] && [[ $COMPILER == "clang" ]]; then
export CXX=clang++ export CXX=clang++
export CC=clang export CC=clang
fi fi
...@@ -24,7 +27,7 @@ fi ...@@ -24,7 +27,7 @@ fi
conda install -q -y -n $CONDA_ENV numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest conda install -q -y -n $CONDA_ENV numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest
if [[ $AGENT_OS == "Darwin" ]] ; then if [[ $AGENT_OS == "Darwin" ]] && [[ $COMPILER == "clang" ]]; then
ln -sf `ls -d "$(brew --cellar libomp)"/*/lib`/* $CONDA_PREFIX/lib || exit -1 # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL) ln -sf `ls -d "$(brew --cellar libomp)"/*/lib`/* $CONDA_PREFIX/lib || exit -1 # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
fi fi
...@@ -36,15 +39,15 @@ if [[ $TASK == "sdist" ]]; then ...@@ -36,15 +39,15 @@ if [[ $TASK == "sdist" ]]; then
exit 0 exit 0
elif [[ $TASK == "bdist" ]]; then elif [[ $TASK == "bdist" ]]; then
if [[ $AGENT_OS == "Darwin" ]]; then if [[ $AGENT_OS == "Darwin" ]]; then
cd ${BUILD_REPOSITORY_LOCALPATH}/python-package && python setup.py bdist_wheel --plat-name=macdarwin --universal || exit -1 cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py bdist_wheel --plat-name=macdarwin --universal || exit -1
cp dist/lightgbm-$LGB_VER-py2.py3-none-macdarwin.whl ${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl
mv dist/lightgbm-$LGB_VER-py2.py3-none-macdarwin.whl dist/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl mv dist/lightgbm-$LGB_VER-py2.py3-none-macdarwin.whl dist/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl
cp dist/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
else else
cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py bdist_wheel --plat-name=manylinux1_x86_64 --universal || exit -1 cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py bdist_wheel --plat-name=manylinux1_x86_64 --universal || exit -1
cp dist/lightgbm-$LGB_VER-py2.py3-none-manylinux1_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY cp dist/lightgbm-$LGB_VER-py2.py3-none-manylinux1_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
fi fi
pip install $BUILD_SOURCESDIRECTORY/python-package/dist/*.whl || exit -1 pip install $BUILD_SOURCESDIRECTORY/python-package/dist/*.whl || exit -1
pytest $BUILD_SOURCESDIRECTORY/tests/python_package_test || exit -1 pytest $BUILD_SOURCESDIRECTORY/tests || exit -1
exit 0 exit 0
fi fi
...@@ -82,7 +85,7 @@ pytest $BUILD_SOURCESDIRECTORY/tests || exit -1 ...@@ -82,7 +85,7 @@ pytest $BUILD_SOURCESDIRECTORY/tests || exit -1
if [[ $TASK == "regular" ]]; then if [[ $TASK == "regular" ]]; then
if [[ $AGENT_OS == "Darwin" ]]; then if [[ $AGENT_OS == "Darwin" ]]; then
cp ${BUILD_REPOSITORY_LOCALPATH}/lib_lightgbm.so ${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.dylib cp $BUILD_SOURCESDIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
else else
cp $BUILD_SOURCESDIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so cp $BUILD_SOURCESDIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
fi fi
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment