Unverified Commit e17a28ec authored by Nikita Titov's avatar Nikita Titov Committed by GitHub
Browse files

[ci] rebalanced Linux tests (#1663)

* refined Travis tests

* refined VSTS Linux part

* cleanup

* added possibility to test Clang at VSTS

* simplified cp command
parent 4a1e5fa5
......@@ -11,16 +11,14 @@ os:
env:
global: # default values
- COMPILER=gcc
- COMPILER=clang
- PYTHON_VERSION=3.7
matrix:
- TASK=regular PYTHON_VERSION=3.6
- TASK=regular COMPILER=clang
- TASK=pylint
- TASK=check-docs
- TASK=if-else
- TASK=sdist PYTHON_VERSION=2.7
- TASK=bdist
- TASK=pylint
- TASK=check-docs
- TASK=mpi METHOD=source
- TASK=mpi METHOD=pip
- TASK=gpu METHOD=source PYTHON_VERSION=3.5
......@@ -32,8 +30,6 @@ matrix:
env: TASK=gpu METHOD=source PYTHON_VERSION=3.5
- os: osx
env: TASK=gpu METHOD=pip PYTHON_VERSION=3.6
- os: osx
env: TASK=if-else
- os: osx
env: TASK=pylint
- os: osx
......@@ -44,10 +40,10 @@ before_install:
- test -n $CXX && unset CXX
- export PATH="$HOME/miniconda/bin:$PATH"
- export LGB_VER=$(head -n 1 VERSION.txt)
- export AMDAPPSDK=$HOME/AMDAPPSDK
- export LD_LIBRARY_PATH="$AMDAPPSDK/lib/x86_64:$LD_LIBRARY_PATH"
- export AMDAPPSDK_PATH=$HOME/AMDAPPSDK
- export LD_LIBRARY_PATH="$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH"
- export LD_LIBRARY_PATH="/usr/local/clang/lib:$LD_LIBRARY_PATH" # fix error "libomp.so: cannot open shared object file: No such file or directory" on Linux with Clang
- export OPENCL_VENDOR_PATH=$AMDAPPSDK/etc/OpenCL/vendors
- export OPENCL_VENDOR_PATH=$AMDAPPSDK_PATH/etc/OpenCL/vendors
install:
- bash .travis/setup.sh
......
......@@ -23,6 +23,13 @@ else # Linux
fi
if [[ $TASK == "gpu" ]]; then
sudo apt-get install -y ocl-icd-opencl-dev
wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
tar -xjf AMD-APP-SDK*.tar.bz2
mkdir -p $OPENCL_VENDOR_PATH
mkdir -p $AMDAPPSDK_PATH
sh AMD-APP-SDK*.sh --tar -xf -C $AMDAPPSDK_PATH
mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
fi
wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh
fi
......@@ -30,12 +37,3 @@ fi
sh conda.sh -b -p $HOME/miniconda
conda config --set always_yes yes --set changeps1 no
conda update -q conda
if [[ $TASK == "gpu" ]] && [[ $TRAVIS_OS_NAME == "linux" ]]; then
wget https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
tar -xjf AMD-APP-SDK*.tar.bz2
mkdir -p $OPENCL_VENDOR_PATH
sh AMD-APP-SDK*.sh --tar -xf -C $AMDAPPSDK
mv $AMDAPPSDK/lib/x86_64/sdk/* $AMDAPPSDK/lib/x86_64/
echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
fi
......@@ -76,7 +76,7 @@ elif [[ $TASK == "bdist" ]]; then
cd $TRAVIS_BUILD_DIR/python-package && python setup.py bdist_wheel --plat-name=manylinux1_x86_64 --universal || exit -1
fi
pip install $TRAVIS_BUILD_DIR/python-package/dist/*.whl || exit -1
pytest $TRAVIS_BUILD_DIR/tests/python_package_test || exit -1
pytest $TRAVIS_BUILD_DIR/tests || exit -1
exit 0
fi
......@@ -86,7 +86,7 @@ if [[ $TASK == "gpu" ]]; then
grep -q 'std::string device_type = "gpu"' $TRAVIS_BUILD_DIR/include/LightGBM/config.h || exit -1 # make sure that changes were really done
if [[ $METHOD == "pip" ]]; then
cd $TRAVIS_BUILD_DIR/python-package && python setup.py sdist || exit -1
pip install $TRAVIS_BUILD_DIR/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--boost-root=$HOME/miniconda/envs/test-env/" --install-option="--opencl-include-dir=$AMDAPPSDK/include/" || exit -1
pip install $TRAVIS_BUILD_DIR/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--boost-root=$HOME/miniconda/envs/test-env/" --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
pytest $TRAVIS_BUILD_DIR/tests/python_package_test || exit -1
exit 0
fi
......@@ -103,7 +103,7 @@ if [[ $TASK == "mpi" ]]; then
fi
cmake -DUSE_MPI=ON ..
elif [[ $TASK == "gpu" ]]; then
cmake -DUSE_GPU=ON -DBOOST_ROOT=$HOME/miniconda/envs/test-env/ -DOpenCL_INCLUDE_DIR=$AMDAPPSDK/include/ ..
cmake -DUSE_GPU=ON -DBOOST_ROOT=$HOME/miniconda/envs/test-env/ -DOpenCL_INCLUDE_DIR=$AMDAPPSDK_PATH/include/ ..
else
cmake ..
fi
......@@ -111,7 +111,7 @@ fi
make _lightgbm || exit -1
cd $TRAVIS_BUILD_DIR/python-package && python setup.py install --precompile || exit -1
pytest $TRAVIS_BUILD_DIR || exit -1
pytest $TRAVIS_BUILD_DIR/tests || exit -1
if [[ $TASK == "regular" ]]; then
cd $TRAVIS_BUILD_DIR/examples/python-guide
......@@ -119,5 +119,6 @@ if [[ $TASK == "regular" ]]; then
import matplotlib\
matplotlib.use\(\"Agg\"\)\
' plot_example.py # prevent interactive window mode
sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
for f in *.py; do python $f || exit -1; done # run all examples
fi
variables:
PYTHON_VERSION: 3.6
PYTHON_VERSION: 3.7
CONDA_ENV: test-env
phases:
###########################################
- phase: Linux
###########################################
variables:
COMPILER: gcc
queue:
name: 'Hosted Linux Preview'
parallel: 9
parallel: 6
matrix:
regular:
TASK: regular
mpi:
TASK: mpi
PYTHON_VERSION: 2.7
pylint:
TASK: pylint
inference:
TASK: if-else
sdist:
TASK: sdist
PYTHON_VERSION: 3.4
PYTHON_VERSION: 3.5
bdist:
TASK: bdist
PYTHON_VERSION: 3.5
gpu_1:
TASK: gpu
PYTHON_VERSION: 3.6
inference:
TASK: if-else
mpi_source:
TASK: mpi
METHOD: source
gpu_2:
TASK: gpu
METHOD: pip
PYTHON_VERSION: 2.7
gpu_source:
TASK: gpu
METHOD: source
PYTHON_VERSION: 3.6
steps:
- task: CondaEnvironment@0
inputs:
updateConda: true
environmentName: $(CONDA_ENV)
packageSpecs: 'python=$(PYTHON_VERSION)'
createOptions: '-q'
- script: |
sudo apt-get update
export LGB_VER=$(head -n 1 VERSION.txt)
export AMDAPPSDK=$HOME/AMDAPPSDK
export LD_LIBRARY_PATH="$AMDAPPSDK/lib/x86_64:$LD_LIBRARY_PATH"
export OPENCL_VENDOR_PATH=$AMDAPPSDK/etc/OpenCL/vendors
if [[ $TASK == "mpi" ]]; then
sudo apt-get install -y libopenmpi-dev openmpi-bin
fi
if [[ $TASK == "gpu" ]]; then
sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev libboost-dev libboost-system-dev libboost-filesystem-dev
wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
tar -xjf AMD-APP-SDK*.tar.bz2
mkdir -p $OPENCL_VENDOR_PATH
sh AMD-APP-SDK*.sh --tar -xf -C $AMDAPPSDK
mv $AMDAPPSDK/lib/x86_64/sdk/* $AMDAPPSDK/lib/x86_64/
echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
fi
bash .vsts-ci/test.sh
- script: |
echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
AMDAPPSDK_PATH=$AGENT_HOMEDIRECTORY/AMDAPPSDK
echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/setup.sh
chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/test.sh
displayName: 'Set variables'
- bash: $(Build.SourcesDirectory)/.vsts-ci/setup.sh
displayName: Setup
- bash: $(Build.SourcesDirectory)/.vsts-ci/test.sh
displayName: Test
- task: PublishBuildArtifacts@1
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
###########################################
......
#!/bin/bash
if [[ $COMPILER == "clang" ]]; then
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
sudo apt-get update
sudo apt-get install libomp-dev
fi
if [[ $TASK == "mpi" ]]; then
sudo apt-get update
sudo apt-get install -y libopenmpi-dev openmpi-bin
fi
if [[ $TASK == "gpu" ]]; then
sudo apt-get update
sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev libboost-dev libboost-system-dev libboost-filesystem-dev
cd $AGENT_HOMEDIRECTORY
wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
tar -xjf AMD-APP-SDK*.tar.bz2
mkdir -p $OPENCL_VENDOR_PATH
mkdir -p $AMDAPPSDK_PATH
sh AMD-APP-SDK*.sh --tar -xf -C $AMDAPPSDK_PATH
mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
fi
#!/bin/bash
cd ${BUILD_REPOSITORY_LOCALPATH}
if [[ $TASK == "check-docs" ]]; then
if [[ $AGENT_OS == "Linux" ]]; then
sudo apt-get install linkchecker -y
fi
if [[ ${PYTHON_VERSION} == "2.7" ]]; then
conda install -y -n $CONDA_ENV mock
fi
conda install -y -n $CONDA_ENV sphinx "sphinx_rtd_theme>=0.3" # html5validator
pip install rstcheck
cd ${BUILD_REPOSITORY_LOCALPATH}/python-package
rstcheck --report warning `find . -type f -name "*.rst"` || exit -1
cd ${BUILD_REPOSITORY_LOCALPATH}/docs
rstcheck --report warning --ignore-directives=autoclass,autofunction `find . -type f -name "*.rst"` || exit -1
make html || exit -1
find ./_build/html/ -type f -name '*.html' -exec \
sed -i -e 's;\(\.\/[^.]*\.\)rst\([^[:space:]]*\);\1html\2;g' {} \; # emulate js function
# html5validator --root ./_build/html/ || exit -1
if [[ $AGENT_OS == "Linux" ]]; then
linkchecker --config=.linkcheckerrc ./_build/html/*.html || exit -1
fi
exit 0
if [[ $AGENT_OS == "Linux" ]] && [[ $COMPILER == "clang" ]]; then
export CXX=clang++
export CC=clang
fi
cd $BUILD_SOURCESDIRECTORY
if [[ $TASK == "pylint" ]]; then
conda install -y -n $CONDA_ENV pycodestyle
pycodestyle --ignore=E501,W503 --exclude=./compute,./docs,./.nuget . || exit -1
......@@ -33,10 +15,10 @@ fi
if [[ $TASK == "if-else" ]]; then
conda install -y -n $CONDA_ENV numpy
mkdir build && cd build && cmake .. && make lightgbm || exit -1
cd ${BUILD_REPOSITORY_LOCALPATH}/tests/cpp_test && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
cd ${BUILD_REPOSITORY_LOCALPATH}/build && make lightgbm || exit -1
cd ${BUILD_REPOSITORY_LOCALPATH}/tests/cpp_test && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit -1
mkdir $BUILD_SOURCESDIRECTORY/build && cd $BUILD_SOURCESDIRECTORY/build && cmake .. && make lightgbm || exit -1
cd $BUILD_SOURCESDIRECTORY/tests/cpp_test && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
cd $BUILD_SOURCESDIRECTORY/build && make lightgbm || exit -1
cd $BUILD_SOURCESDIRECTORY/tests/cpp_test && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit -1
exit 0
fi
......@@ -47,10 +29,10 @@ if [[ $AGENT_OS == "Darwin" ]] ; then
fi
if [[ $TASK == "sdist" ]]; then
cd ${BUILD_REPOSITORY_LOCALPATH}/python-package && python setup.py sdist || exit -1
pip install ${BUILD_REPOSITORY_LOCALPATH}/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
cp ${BUILD_REPOSITORY_LOCALPATH}/python-package/dist/lightgbm-$LGB_VER.tar.gz ${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbm-$LGB_VER.tar.gz
pytest ${BUILD_REPOSITORY_LOCALPATH}/tests/python_package_test || exit -1
cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py sdist || exit -1
pip install $BUILD_SOURCESDIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
cp $BUILD_SOURCESDIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY
pytest $BUILD_SOURCESDIRECTORY/tests/python_package_test || exit -1
exit 0
elif [[ $TASK == "bdist" ]]; then
if [[ $AGENT_OS == "Darwin" ]]; then
......@@ -58,55 +40,57 @@ elif [[ $TASK == "bdist" ]]; then
cp dist/lightgbm-$LGB_VER-py2.py3-none-macdarwin.whl ${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl
mv dist/lightgbm-$LGB_VER-py2.py3-none-macdarwin.whl dist/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl
else
cd ${BUILD_REPOSITORY_LOCALPATH}/python-package && python setup.py bdist_wheel --plat-name=manylinux1_x86_64 --universal || exit -1
cp dist/lightgbm-$LGB_VER-py2.py3-none-manylinux1_x86_64.whl ${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbm-$LGB_VER-py2.py3-none-manylinux1_x86_64.whl
cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py bdist_wheel --plat-name=manylinux1_x86_64 --universal || exit -1
cp dist/lightgbm-$LGB_VER-py2.py3-none-manylinux1_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
fi
pip install ${BUILD_REPOSITORY_LOCALPATH}/python-package/dist/*.whl || exit -1
pytest ${BUILD_REPOSITORY_LOCALPATH}/tests/python_package_test || exit -1
pip install $BUILD_SOURCESDIRECTORY/python-package/dist/*.whl || exit -1
pytest $BUILD_SOURCESDIRECTORY/tests/python_package_test || exit -1
exit 0
fi
if [[ $TASK == "gpu" ]]; then
sed -i 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' ${BUILD_REPOSITORY_LOCALPATH}/include/LightGBM/config.h
grep -q 'std::string device_type = "gpu"' ${BUILD_REPOSITORY_LOCALPATH}/include/LightGBM/config.h || exit -1 # make sure that changes were really done
sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_SOURCESDIRECTORY/include/LightGBM/config.h
grep -q 'std::string device_type = "gpu"' $BUILD_SOURCESDIRECTORY/include/LightGBM/config.h || exit -1 # make sure that changes were really done
if [[ $METHOD == "pip" ]]; then
cd ${BUILD_REPOSITORY_LOCALPATH}/python-package && python setup.py sdist || exit -1
pip install ${BUILD_REPOSITORY_LOCALPATH}/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--opencl-include-dir=$AMDAPPSDK/include/" || exit -1
pytest ${BUILD_REPOSITORY_LOCALPATH}/tests/python_package_test || exit -1
cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py sdist || exit -1
pip install $BUILD_SOURCESDIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
pytest $BUILD_SOURCESDIRECTORY/tests/python_package_test || exit -1
exit 0
fi
fi
mkdir build && cd build
mkdir $BUILD_SOURCESDIRECTORY/build && cd $BUILD_SOURCESDIRECTORY/build
if [[ $TASK == "mpi" ]]; then
cd ${BUILD_REPOSITORY_LOCALPATH}/python-package && python setup.py sdist || exit -1
pip install ${BUILD_REPOSITORY_LOCALPATH}/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
cd ${BUILD_REPOSITORY_LOCALPATH}/build
if [[ $METHOD == "pip" ]]; then
cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py sdist || exit -1
pip install $BUILD_SOURCESDIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
pytest $BUILD_SOURCESDIRECTORY/tests/python_package_test || exit -1
exit 0
fi
cmake -DUSE_MPI=ON ..
elif [[ $TASK == "gpu" ]]; then
cmake -DUSE_GPU=ON -DOpenCL_INCLUDE_DIR=$AMDAPPSDK/include/ ..
cmake -DUSE_GPU=ON -DOpenCL_INCLUDE_DIR=$AMDAPPSDK_PATH/include/ ..
else
cmake ..
fi
make _lightgbm || exit -1
cd ${BUILD_REPOSITORY_LOCALPATH}/python-package && python setup.py install --precompile || exit -1
pytest ${BUILD_REPOSITORY_LOCALPATH} || exit -1
cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py install --precompile || exit -1
pytest $BUILD_SOURCESDIRECTORY/tests || exit -1
if [[ $TASK == "regular" ]]; then
if [[ $AGENT_OS == "Darwin" ]]; then
cp ${BUILD_REPOSITORY_LOCALPATH}/lib_lightgbm.so ${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.dylib
else
cp ${BUILD_REPOSITORY_LOCALPATH}/lib_lightgbm.so ${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.so
cp $BUILD_SOURCESDIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
fi
cd ${BUILD_REPOSITORY_LOCALPATH}/examples/python-guide
cd $BUILD_SOURCESDIRECTORY/examples/python-guide
sed -i'.bak' '/import lightgbm as lgb/a\
import matplotlib\
matplotlib.use\(\"Agg\"\)\
' plot_example.py # prevent interactive window mode
sed -i 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
for f in *.py; do python $f || exit -1; done # run all examples
fi
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment