Unverified Commit b044070e authored by Nikita Titov's avatar Nikita Titov Committed by GitHub
Browse files

[ci] run Dask examples on CI (#4064)

* Update Parallel-Learning-Guide.rst

* Update test.sh

* fix path

* address review comments
parent 96728a04
...@@ -224,7 +224,7 @@ import matplotlib\ ...@@ -224,7 +224,7 @@ import matplotlib\
matplotlib.use\(\"Agg\"\)\ matplotlib.use\(\"Agg\"\)\
' plot_example.py # prevent interactive window mode ' plot_example.py # prevent interactive window mode
sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
for f in *.py; do python $f || exit -1; done # run all examples for f in *.py **/*.py; do python $f || exit -1; done # run all examples
cd $BUILD_DIRECTORY/examples/python-guide/notebooks cd $BUILD_DIRECTORY/examples/python-guide/notebooks
conda install -q -y -n $CONDA_ENV ipywidgets notebook conda install -q -y -n $CONDA_ENV ipywidgets notebook
jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb || exit -1 # run all notebooks jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb || exit -1 # run all notebooks
......
...@@ -62,10 +62,14 @@ Dask ...@@ -62,10 +62,14 @@ Dask
LightGBM's Python package supports distributed learning via `Dask`_. This integration is maintained by LightGBM's maintainers. LightGBM's Python package supports distributed learning via `Dask`_. This integration is maintained by LightGBM's maintainers.
.. warning::
Dask integration is only tested on Linux.
Dask Examples Dask Examples
''''''''''''' '''''''''''''
For sample code using ``lightgbm.dask``, see `these Dask examples`_ . For sample code using ``lightgbm.dask``, see `these Dask examples`_.
Training with Dask Training with Dask
'''''''''''''''''' ''''''''''''''''''
......
...@@ -5,7 +5,6 @@ from sklearn.datasets import make_blobs ...@@ -5,7 +5,6 @@ from sklearn.datasets import make_blobs
import lightgbm as lgb import lightgbm as lgb
if __name__ == "__main__": if __name__ == "__main__":
print("loading data") print("loading data")
X, y = make_blobs(n_samples=1000, n_features=50, centers=2) X, y = make_blobs(n_samples=1000, n_features=50, centers=2)
......
...@@ -5,7 +5,6 @@ from sklearn.datasets import make_blobs ...@@ -5,7 +5,6 @@ from sklearn.datasets import make_blobs
import lightgbm as lgb import lightgbm as lgb
if __name__ == "__main__": if __name__ == "__main__":
print("loading data") print("loading data")
X, y = make_blobs(n_samples=1000, n_features=50, centers=3) X, y = make_blobs(n_samples=1000, n_features=50, centers=3)
......
...@@ -6,7 +6,6 @@ from sklearn.metrics import mean_squared_error ...@@ -6,7 +6,6 @@ from sklearn.metrics import mean_squared_error
import lightgbm as lgb import lightgbm as lgb
if __name__ == "__main__": if __name__ == "__main__":
print("loading data") print("loading data")
X, y = make_regression(n_samples=1000, n_features=50) X, y = make_regression(n_samples=1000, n_features=50)
......
import os
import dask.array as da import dask.array as da
import numpy as np import numpy as np
from distributed import Client, LocalCluster from distributed import Client, LocalCluster
...@@ -6,11 +8,12 @@ from sklearn.datasets import load_svmlight_file ...@@ -6,11 +8,12 @@ from sklearn.datasets import load_svmlight_file
import lightgbm as lgb import lightgbm as lgb
if __name__ == "__main__": if __name__ == "__main__":
print("loading data") print("loading data")
X, y = load_svmlight_file("../lambdarank/rank.train") X, y = load_svmlight_file(os.path.join(os.path.dirname(os.path.realpath(__file__)),
group = np.loadtxt("../lambdarank/rank.train.query") '../../lambdarank/rank.train'))
group = np.loadtxt(os.path.join(os.path.dirname(os.path.realpath(__file__)),
'../../lambdarank/rank.train.query'))
print("initializing a Dask cluster") print("initializing a Dask cluster")
......
...@@ -5,7 +5,6 @@ from sklearn.datasets import make_regression ...@@ -5,7 +5,6 @@ from sklearn.datasets import make_regression
import lightgbm as lgb import lightgbm as lgb
if __name__ == "__main__": if __name__ == "__main__":
print("loading data") print("loading data")
X, y = make_regression(n_samples=1000, n_features=50) X, y = make_regression(n_samples=1000, n_features=50)
......
...@@ -204,6 +204,10 @@ You can use ``python setup.py bdist_wheel`` instead of ``python setup.py install ...@@ -204,6 +204,10 @@ You can use ``python setup.py bdist_wheel`` instead of ``python setup.py install
Install Dask-package Install Dask-package
'''''''''''''''''''' ''''''''''''''''''''
.. warning::
Dask-package is only tested on Linux.
To install all additional dependencies required for Dask-package, you can append ``[dask]`` to LightGBM package name: To install all additional dependencies required for Dask-package, you can append ``[dask]`` to LightGBM package name:
.. code:: sh .. code:: sh
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment