Skip to content

GitLab

"...text-generation-inference.git" did not exist on "e52be9bba2a11f6b2145fcd21be0ba860b977a4d"
Unverified Commit a95459e3 authored by Minjie Wang's avatar Minjie Wang Committed by GitHub
Browse files

[CI] Improved CI (#141) 

* change ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* nx package

* update ci

* update ci

* update ci

* fix

* mx dockerfile by zhengda

* python3.6->3.5

* update ci image

* add tutorial test

* fix ci

* fix ssl problem

* minor change

* small fix on traversal utest

* fix syntax

* add matplotlib in image

* fix

* update ci

* update ci
parent 7cbb83db
Hide whitespace changes
Inline Side-by-side
Showing with 247 additions and 186 deletions
Jenkinsfile
View file @ a95459e3
#!/usr/bin/env groovy #!/usr/bin/env groovy
def init_git_submodule() { def init_git_submodule() {
sh 'git submodule init' sh "git submodule init"
sh 'git submodule update' sh "git submodule update"
} }
def setup() { def setup() {
sh 'easy_install nose' init_git_submodule()
init_git_submodule()
} }
def build_dgl() { def build_dgl() {
sh 'if [ -d build ]; then rm -rf build; fi; mkdir build' sh "if [ -d build ]; then rm -rf build; fi; mkdir build"
dir('python') { dir("python") {
sh 'python3 setup.py install' sh "python3 setup.py install"
} }
dir ('build') { dir ("build") {
sh 'cmake ..' sh "cmake .."
sh 'make -j4' sh "make -j4"
} }
} }
def pytorch_unit_test() { def pytorch_unit_test(dev) {
withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) { withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
sh 'nosetests tests -v --with-xunit' sh "python3 -m nose -v --with-xunit tests"
sh 'nosetests tests/pytorch -v --with-xunit' sh "python3 -m nose -v --with-xunit tests/pytorch"
sh 'nosetests tests/graph_index -v --with-xunit' sh "python3 -m nose -v --with-xunit tests/graph_index"
} }
} }
def mxnet_unit_test() { def mxnet_unit_test(dev) {
withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) { withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
sh 'nosetests tests/mxnet -v --with-xunit' sh "python3 -m nose -v --with-xunit tests/mxnet"
} }
} }
def example_test(dev) { def example_test(dev) {
dir ('tests/scripts') { dir ("tests/scripts") {
withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) { withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
sh "./test_examples.sh ${dev}" sh "./task_example_test.sh ${dev}"
} }
}
}
def pytorch_tutorials() {
dir ("tests/scripts") {
withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
sh "./task_tutorial_test.sh"
} }
}
} }
pipeline { pipeline {
agent none agent none
stages { stages {
stage('Lint Check') { stage("Lint Check") {
agent { agent { docker { image "dgllib/dgl-ci-lint" } }
docker { steps {
image 'lingfanyu/dgl-lint' init_git_submodule()
} sh "tests/scripts/task_lint.sh"
}
}
stage("Build") {
parallel {
stage("CPU Build") {
agent { docker { image "dgllib/dgl-ci-cpu" } }
steps {
setup()
build_dgl()
}
}
stage("GPU Build") {
agent {
docker {
image "dgllib/dgl-ci-gpu"
args "--runtime nvidia"
}
}
steps {
setup()
build_dgl()
}
}
stage("MXNet CPU Build (temp)") {
agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } }
steps {
setup()
build_dgl()
}
}
}
}
stage("Test") {
parallel {
stage("Pytorch CPU") {
agent { docker { image "dgllib/dgl-ci-cpu" } }
stages {
stage("TH CPU unittest") {
steps { pytorch_unit_test("CPU") }
} }
stages { stage("TH CPU example test") {
stage('CHECK') { steps { example_test("CPU") }
steps {
init_git_submodule()
sh 'tests/scripts/task_lint.sh'
}
}
} }
}
post {
always { junit "*.xml" }
}
} }
stage('Build and Test on Pytorch') { stage("Pytorch GPU") {
parallel { agent {
stage('CPU') { docker {
agent { image "dgllib/dgl-ci-gpu"
docker { args "--runtime nvidia"
image 'lingfanyu/dgl-cpu'
}
}
stages {
stage('SETUP') {
steps {
setup()
}
}
stage('BUILD') {
steps {
build_dgl()
}
}
stage('UNIT TEST') {
steps {
pytorch_unit_test()
}
}
stage('EXAMPLE TEST') {
steps {
example_test('CPU')
}
}
}
post {
always {
junit '*.xml'
}
}
}
stage('GPU') {
agent {
docker {
image 'lingfanyu/dgl-gpu'
args '--runtime nvidia'
}
}
stages {
stage('SETUP') {
steps {
setup()
}
}
stage('BUILD') {
steps {
build_dgl()
}
}
stage('UNIT TEST') {
steps {
pytorch_unit_test()
}
}
stage('EXAMPLE TEST') {
steps {
example_test('GPU')
}
}
}
post {
always {
junit '*.xml'
}
}
}
} }
}
stages {
// TODO: have GPU unittest
//stage("TH GPU unittest") {
// steps { pytorch_unit_test("GPU") }
//}
stage("TH GPU example test") {
steps { example_test("GPU") }
}
}
// TODO: have GPU unittest
//post {
// always { junit "*.xml" }
//}
} }
stage('Build and Test on MXNet') { stage("MXNet CPU") {
parallel { agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } }
stage('CPU') { stages {
agent { stage("MX Unittest") {
docker { steps { mxnet_unit_test("CPU") }
image 'zhengda1936/dgl-mxnet-cpu:v3'
}
}
stages {
stage('SETUP') {
steps {
setup()
}
}
stage('BUILD') {
steps {
build_dgl()
}
}
stage('UNIT TEST') {
steps {
mxnet_unit_test()
}
}
stage('EXAMPLE TEST') {
steps {
example_test('CPU')
}
}
}
post {
always {
junit '*.xml'
}
}
}
} }
}
post {
always { junit "*.xml" }
}
} }
}
}
stage("Doc") {
agent { docker { image "dgllib/dgl-ci-cpu" } }
steps {
pytorch_tutorials()
}
} }
}
} }
docker/Dockerfile.ci_cpu_mxnet 0 → 100644
View file @ a95459e3
# CI docker GPU env
FROM nvidia/cuda:9.0-cudnn7-devel
# Base scripts
RUN apt-get update --fix-missing
COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
COPY install/ubuntu_install_mxnet_cpu.sh /install/ubuntu_install_mxnet_cpu.sh
RUN bash /install/ubuntu_install_mxnet_cpu.sh
# Environment variables
ENV PATH=/usr/local/nvidia/bin:${PATH}
ENV PATH=/usr/local/cuda/bin:${PATH}
ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
docker/Dockerfile.ci_gpu_mxnet 0 → 100644
View file @ a95459e3
# CI docker GPU env
FROM nvidia/cuda:9.0-cudnn7-devel
# Base scripts
RUN apt-get update --fix-missing
COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
COPY install/ubuntu_install_mxnet_gpu.sh /install/ubuntu_install_mxnet_gpu.sh
RUN bash /install/ubuntu_install_mxnet_gpu.sh
# Environment variables
ENV PATH=/usr/local/nvidia/bin:${PATH}
ENV PATH=/usr/local/cuda/bin:${PATH}
ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
docker/README.md
View file @ a95459e3
...@@ -8,3 +8,9 @@ docker build -t dgl-gpu -f Dockerfile.ci_gpu . ...@@ -8,3 +8,9 @@ docker build -t dgl-gpu -f Dockerfile.ci_gpu .
### Lint image ### Lint image
docker build -t dgl-lint -f Dockerfile.ci_lint . docker build -t dgl-lint -f Dockerfile.ci_lint .
### CPU MXNet image
docker build -t dgl-mxnet-cpu -f Dockerfile.ci_cpu_mxnet .
### GPU MXNet image
docker build -t dgl-mxnet-gpu -f Dockerfile.ci_gpu_mxnet .
docker/install/ubuntu_install_core.sh
View file @ a95459e3
# install libraries for building c++ core on ubuntu # install libraries for building c++ core on ubuntu
apt update && apt install -y --no-install-recommends --force-yes \ apt update && apt install -y --no-install-recommends --force-yes \
apt-utils git build-essential make cmake wget unzip sudo libz-dev libxml2-dev apt-utils git build-essential make cmake wget unzip sudo \
libz-dev libxml2-dev libopenblas-dev libopencv-dev ca-certificates
docker/install/ubuntu_install_mxnet_cpu.sh 0 → 100644
View file @ a95459e3
git clone --recursive -b dgl_graph https://github.com/zheng-da/incubator-mxnet.git
cd incubator-mxnet
make -j4 USE_OPENCV=1 USE_BLAS=openblas USE_MKLDNN=1 USE_DIST_KVSTORE=1
pip3 install -e python
docker/install/ubuntu_install_mxnet_gpu.sh 0 → 100644
View file @ a95459e3
git clone --recursive -b dgl_graph https://github.com/zheng-da/incubator-mxnet.git
cd incubator-mxnet
make -j4 USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 USE_MKLDNN=1 USE_DIST_KVSTORE=1
pip3 install -e python
docker/install/ubuntu_install_python.sh
View file @ a95459e3
# install python and pip, don't modify this, modify install_python_package.sh # install python and pip, don't modify this, modify install_python_package.sh
# apt-get update && apt-get install -y python-dev python-pip apt-get update
apt-get install -y python-dev python3-dev
# python 3.6 # install pip
apt-get update && yes | apt-get install software-properties-common
add-apt-repository ppa:jonathonf/python-3.6
apt-get update && apt-get install -y python3.6 python3.6-dev
rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3
# Install pip
cd /tmp && wget https://bootstrap.pypa.io/get-pip.py cd /tmp && wget https://bootstrap.pypa.io/get-pip.py
# python2 get-pip.py python2 get-pip.py
python3.6 get-pip.py python3 get-pip.py
# santiy check
python2 --version
python3 --version
pip2 --version
pip3 --version
docker/install/ubuntu_install_python_package.sh
View file @ a95459e3
# install libraries for python package on ubuntu # install libraries for python package on ubuntu
# pip2 install pylint numpy cython scipy nltk requests[security] pip2 install nose numpy cython scipy networkx matplotlib nltk requests[security]
pip3 install pylint numpy cython scipy nltk requests[security] pip3 install nose numpy cython scipy networkx matplotlib nltk requests[security]
# install DL Framework # install DL Framework
# pip2 install torch torchvision pip2 install torch torchvision
pip3 install torch torchvision pip3 install torch torchvision
tests/pytorch/test_traversal.py
View file @ a95459e3
...@@ -16,37 +16,42 @@ np.random.seed(42) ...@@ -16,37 +16,42 @@ np.random.seed(42)
def toset(x): def toset(x):
return set(x.tolist()) return set(x.tolist())
def test_bfs_nodes(n=1000): def test_bfs(n=1000):
g_nx = nx.random_tree(n) def _bfs_nx(g_nx, src):
g = dgl.DGLGraph() edges = nx.bfs_edges(g_nx, src)
g.from_networkx(g_nx) layers_nx = [set([src])]
edges_nx = []
frontier = set()
edge_frontier = set()
for u, v in edges:
if u in layers_nx[-1]:
frontier.add(v)
edge_frontier.add(g.edge_id(u, v))
else:
layers_nx.append(frontier)
edges_nx.append(edge_frontier)
frontier = set([v])
edge_frontier = set([g.edge_id(u, v)])
layers_nx.append(frontier)
edges_nx.append(edge_frontier)
return layers_nx, edges_nx
g = dgl.DGLGraph()
a = sp.random(n, n, 10 / n, data_rvs=lambda n: np.ones(n))
g.from_scipy_sparse_matrix(a)
g_nx = g.to_networkx()
src = random.choice(range(n)) src = random.choice(range(n))
layers_nx, _ = _bfs_nx(g_nx, src)
edges = nx.bfs_edges(g_nx, src)
layers_nx = [set([src])]
edges_nx = []
frontier = set()
edge_frontier = set()
for u, v in edges:
if u in layers_nx[-1]:
frontier.add(v)
edge_frontier.add(g.edge_id(u, v))
else:
layers_nx.append(frontier)
edges_nx.append(edge_frontier)
frontier = set([v])
edge_frontier = set([g.edge_id(u, v)])
layers_nx.append(frontier)
edges_nx.append(edge_frontier)
layers_dgl = dgl.bfs_nodes_generator(g, src) layers_dgl = dgl.bfs_nodes_generator(g, src)
assert len(layers_dgl) == len(layers_nx) assert len(layers_dgl) == len(layers_nx)
assert all(toset(x) == y for x, y in zip(layers_dgl, layers_nx)) assert all(toset(x) == y for x, y in zip(layers_dgl, layers_nx))
g_nx = nx.random_tree(n, seed=42)
g = dgl.DGLGraph()
g.from_networkx(g_nx)
src = 0
_, edges_nx = _bfs_nx(g_nx, src)
edges_dgl = dgl.bfs_edges_generator(g, src) edges_dgl = dgl.bfs_edges_generator(g, src)
assert len(edges_dgl) == len(edges_nx) assert len(edges_dgl) == len(edges_nx)
assert all(toset(x) == y for x, y in zip(edges_dgl, edges_nx)) assert all(toset(x) == y for x, y in zip(edges_dgl, edges_nx))
...@@ -114,6 +119,6 @@ def test_dfs_labeled_edges(n=1000, example=False): ...@@ -114,6 +119,6 @@ def test_dfs_labeled_edges(n=1000, example=False):
if __name__ == '__main__': if __name__ == '__main__':
test_bfs_nodes() test_bfs()
test_topological_nodes() test_topological_nodes()
test_dfs_labeled_edges() test_dfs_labeled_edges()
tests/scripts/task_tutorial_test.sh 0 → 100755
View file @ a95459e3
#!/bin/bash
# The working directory for this script will be "tests/scripts"
function fail {
echo FAIL: $@
exit -1
}
export MPLBACKEND=Agg
for f in $(find "../../tutorials" -name "*.py")
do
echo "Running tutorial ${f} ..."
python3 $f || fail "run ${f}"
done
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment