Skip to content

GitLab

Unverified Commit a95459e3 authored by Minjie Wang's avatar Minjie Wang Committed by GitHub
Browse files

[CI] Improved CI (#141) 

* change ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* nx package

* update ci

* update ci

* update ci

* fix

* mx dockerfile by zhengda

* python3.6->3.5

* update ci image

* add tutorial test

* fix ci

* fix ssl problem

* minor change

* small fix on traversal utest

* fix syntax

* add matplotlib in image

* fix

* update ci

* update ci
parent 7cbb83db
Show whitespace changes
Inline Side-by-side
Showing with 247 additions and 186 deletions
Jenkinsfile
View file @ a95459e3
#!/usr/bin/env groovy
def init_git_submodule() {
sh 'git submodule init'
sh 'git submodule update'
sh "git submodule init"
sh "git submodule update"
}
def setup() {
sh 'easy_install nose'
init_git_submodule()
}
def build_dgl() {
sh 'if [ -d build ]; then rm -rf build; fi; mkdir build'
dir('python') {
sh 'python3 setup.py install'
sh "if [ -d build ]; then rm -rf build; fi; mkdir build"
dir("python") {
sh "python3 setup.py install"
}
dir ('build') {
sh 'cmake ..'
sh 'make -j4'
dir ("build") {
sh "cmake .."
sh "make -j4"
}
}
def pytorch_unit_test() {
withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) {
sh 'nosetests tests -v --with-xunit'
sh 'nosetests tests/pytorch -v --with-xunit'
sh 'nosetests tests/graph_index -v --with-xunit'
def pytorch_unit_test(dev) {
withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
sh "python3 -m nose -v --with-xunit tests"
sh "python3 -m nose -v --with-xunit tests/pytorch"
sh "python3 -m nose -v --with-xunit tests/graph_index"
}
}
def mxnet_unit_test() {
withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) {
sh 'nosetests tests/mxnet -v --with-xunit'
def mxnet_unit_test(dev) {
withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
sh "python3 -m nose -v --with-xunit tests/mxnet"
}
}
def example_test(dev) {
dir ('tests/scripts') {
withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) {
sh "./test_examples.sh ${dev}"
dir ("tests/scripts") {
withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
sh "./task_example_test.sh ${dev}"
}
}
}
pipeline {
agent none
stages {
stage('Lint Check') {
agent {
docker {
image 'lingfanyu/dgl-lint'
def pytorch_tutorials() {
dir ("tests/scripts") {
withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
sh "./task_tutorial_test.sh"
}
}
}
pipeline {
agent none
stages {
stage('CHECK') {
stage("Lint Check") {
agent { docker { image "dgllib/dgl-ci-lint" } }
steps {
init_git_submodule()
sh 'tests/scripts/task_lint.sh'
}
}
sh "tests/scripts/task_lint.sh"
}
}
stage('Build and Test on Pytorch') {
stage("Build") {
parallel {
stage('CPU') {
agent {
docker {
image 'lingfanyu/dgl-cpu'
}
}
stages {
stage('SETUP') {
stage("CPU Build") {
agent { docker { image "dgllib/dgl-ci-cpu" } }
steps {
setup()
}
}
stage('BUILD') {
steps {
build_dgl()
}
}
stage('UNIT TEST') {
steps {
pytorch_unit_test()
}
}
stage('EXAMPLE TEST') {
steps {
example_test('CPU')
}
}
}
post {
always {
junit '*.xml'
}
}
}
stage('GPU') {
stage("GPU Build") {
agent {
docker {
image 'lingfanyu/dgl-gpu'
args '--runtime nvidia'
image "dgllib/dgl-ci-gpu"
args "--runtime nvidia"
}
}
stages {
stage('SETUP') {
steps {
setup()
build_dgl()
}
}
stage('BUILD') {
stage("MXNet CPU Build (temp)") {
agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } }
steps {
setup()
build_dgl()
}
}
stage('UNIT TEST') {
steps {
pytorch_unit_test()
}
}
stage('EXAMPLE TEST') {
steps {
example_test('GPU')
stage("Test") {
parallel {
stage("Pytorch CPU") {
agent { docker { image "dgllib/dgl-ci-cpu" } }
stages {
stage("TH CPU unittest") {
steps { pytorch_unit_test("CPU") }
}
stage("TH CPU example test") {
steps { example_test("CPU") }
}
}
post {
always {
junit '*.xml'
}
always { junit "*.xml" }
}
}
}
}
stage('Build and Test on MXNet') {
parallel {
stage('CPU') {
stage("Pytorch GPU") {
agent {
docker {
image 'zhengda1936/dgl-mxnet-cpu:v3'
image "dgllib/dgl-ci-gpu"
args "--runtime nvidia"
}
}
stages {
stage('SETUP') {
steps {
setup()
// TODO: have GPU unittest
//stage("TH GPU unittest") {
// steps { pytorch_unit_test("GPU") }
//}
stage("TH GPU example test") {
steps { example_test("GPU") }
}
}
stage('BUILD') {
steps {
build_dgl()
}
}
stage('UNIT TEST') {
steps {
mxnet_unit_test()
}
}
stage('EXAMPLE TEST') {
steps {
example_test('CPU')
// TODO: have GPU unittest
//post {
// always { junit "*.xml" }
//}
}
stage("MXNet CPU") {
agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } }
stages {
stage("MX Unittest") {
steps { mxnet_unit_test("CPU") }
}
}
post {
always {
junit '*.xml'
always { junit "*.xml" }
}
}
}
}
stage("Doc") {
agent { docker { image "dgllib/dgl-ci-cpu" } }
steps {
pytorch_tutorials()
}
}
}
......
docker/Dockerfile.ci_cpu_mxnet 0 → 100644
View file @ a95459e3
# CI docker GPU env
FROM nvidia/cuda:9.0-cudnn7-devel
# Base scripts
RUN apt-get update --fix-missing
COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
COPY install/ubuntu_install_mxnet_cpu.sh /install/ubuntu_install_mxnet_cpu.sh
RUN bash /install/ubuntu_install_mxnet_cpu.sh
# Environment variables
ENV PATH=/usr/local/nvidia/bin:${PATH}
ENV PATH=/usr/local/cuda/bin:${PATH}
ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
docker/Dockerfile.ci_gpu_mxnet 0 → 100644
View file @ a95459e3
# CI docker GPU env
FROM nvidia/cuda:9.0-cudnn7-devel
# Base scripts
RUN apt-get update --fix-missing
COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
COPY install/ubuntu_install_mxnet_gpu.sh /install/ubuntu_install_mxnet_gpu.sh
RUN bash /install/ubuntu_install_mxnet_gpu.sh
# Environment variables
ENV PATH=/usr/local/nvidia/bin:${PATH}
ENV PATH=/usr/local/cuda/bin:${PATH}
ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
docker/README.md
View file @ a95459e3
......@@ -8,3 +8,9 @@ docker build -t dgl-gpu -f Dockerfile.ci_gpu .
### Lint image
docker build -t dgl-lint -f Dockerfile.ci_lint .
### CPU MXNet image
docker build -t dgl-mxnet-cpu -f Dockerfile.ci_cpu_mxnet .
### GPU MXNet image
docker build -t dgl-mxnet-gpu -f Dockerfile.ci_gpu_mxnet .
docker/install/ubuntu_install_core.sh
View file @ a95459e3
# install libraries for building c++ core on ubuntu
apt update && apt install -y --no-install-recommends --force-yes \
apt-utils git build-essential make cmake wget unzip sudo libz-dev libxml2-dev
apt-utils git build-essential make cmake wget unzip sudo \
libz-dev libxml2-dev libopenblas-dev libopencv-dev ca-certificates
docker/install/ubuntu_install_mxnet_cpu.sh 0 → 100644
View file @ a95459e3
git clone --recursive -b dgl_graph https://github.com/zheng-da/incubator-mxnet.git
cd incubator-mxnet
make -j4 USE_OPENCV=1 USE_BLAS=openblas USE_MKLDNN=1 USE_DIST_KVSTORE=1
pip3 install -e python
docker/install/ubuntu_install_mxnet_gpu.sh 0 → 100644
View file @ a95459e3
git clone --recursive -b dgl_graph https://github.com/zheng-da/incubator-mxnet.git
cd incubator-mxnet
make -j4 USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 USE_MKLDNN=1 USE_DIST_KVSTORE=1
pip3 install -e python
docker/install/ubuntu_install_python.sh
View file @ a95459e3
# install python and pip, don't modify this, modify install_python_package.sh
# apt-get update && apt-get install -y python-dev python-pip
apt-get update
apt-get install -y python-dev python3-dev
# python 3.6
apt-get update && yes | apt-get install software-properties-common
add-apt-repository ppa:jonathonf/python-3.6
apt-get update && apt-get install -y python3.6 python3.6-dev
rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3
# Install pip
# install pip
cd /tmp && wget https://bootstrap.pypa.io/get-pip.py
# python2 get-pip.py
python3.6 get-pip.py
python2 get-pip.py
python3 get-pip.py
# santiy check
python2 --version
python3 --version
pip2 --version
pip3 --version
docker/install/ubuntu_install_python_package.sh
View file @ a95459e3
# install libraries for python package on ubuntu
# pip2 install pylint numpy cython scipy nltk requests[security]
pip3 install pylint numpy cython scipy nltk requests[security]
pip2 install nose numpy cython scipy networkx matplotlib nltk requests[security]
pip3 install nose numpy cython scipy networkx matplotlib nltk requests[security]
# install DL Framework
# pip2 install torch torchvision
pip2 install torch torchvision
pip3 install torch torchvision
tests/pytorch/test_traversal.py
View file @ a95459e3
......@@ -16,13 +16,8 @@ np.random.seed(42)
def toset(x):
return set(x.tolist())
def test_bfs_nodes(n=1000):
g_nx = nx.random_tree(n)
g = dgl.DGLGraph()
g.from_networkx(g_nx)
src = random.choice(range(n))
def test_bfs(n=1000):
def _bfs_nx(g_nx, src):
edges = nx.bfs_edges(g_nx, src)
layers_nx = [set([src])]
edges_nx = []
......@@ -39,14 +34,24 @@ def test_bfs_nodes(n=1000):
edge_frontier = set([g.edge_id(u, v)])
layers_nx.append(frontier)
edges_nx.append(edge_frontier)
return layers_nx, edges_nx
g = dgl.DGLGraph()
a = sp.random(n, n, 10 / n, data_rvs=lambda n: np.ones(n))
g.from_scipy_sparse_matrix(a)
g_nx = g.to_networkx()
src = random.choice(range(n))
layers_nx, _ = _bfs_nx(g_nx, src)
layers_dgl = dgl.bfs_nodes_generator(g, src)
assert len(layers_dgl) == len(layers_nx)
assert all(toset(x) == y for x, y in zip(layers_dgl, layers_nx))
g_nx = nx.random_tree(n, seed=42)
g = dgl.DGLGraph()
g.from_networkx(g_nx)
src = 0
_, edges_nx = _bfs_nx(g_nx, src)
edges_dgl = dgl.bfs_edges_generator(g, src)
assert len(edges_dgl) == len(edges_nx)
assert all(toset(x) == y for x, y in zip(edges_dgl, edges_nx))
......@@ -114,6 +119,6 @@ def test_dfs_labeled_edges(n=1000, example=False):
if __name__ == '__main__':
test_bfs_nodes()
test_bfs()
test_topological_nodes()
test_dfs_labeled_edges()
tests/scripts/task_tutorial_test.sh 0 → 100755
View file @ a95459e3
#!/bin/bash
# The working directory for this script will be "tests/scripts"
function fail {
echo FAIL: $@
exit -1
}
export MPLBACKEND=Agg
for f in $(find "../../tutorials" -name "*.py")
do
echo "Running tutorial ${f} ..."
python3 $f || fail "run ${f}"
done
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment