[CI] Improved CI (#141)

* change ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* update ci

* nx package

* update ci

* update ci

* update ci

* fix

* mx dockerfile by zhengda

* python3.6->3.5

* update ci image

* add tutorial test

* fix ci

* fix ssl problem

* minor change

* small fix on traversal utest

* fix syntax

* add matplotlib in image

* fix

* update ci

* update ci

Jenkinsfile

 #!/usr/bin/env groovy
 
 def init_git_submodule() {
-    sh 'git submodule init'
-    sh 'git submodule update'
+  sh "git submodule init"
+  sh "git submodule update"
 }
 
 def setup() {
-    sh 'easy_install nose'
-    init_git_submodule()
+  init_git_submodule()
 }
 
 def build_dgl() {
-    sh 'if [ -d build ]; then rm -rf build; fi; mkdir build'
-    dir('python') {
-        sh 'python3 setup.py install'
-    }
-    dir ('build') {
-        sh 'cmake ..'
-        sh 'make -j4'
-    }
+  sh "if [ -d build ]; then rm -rf build; fi; mkdir build"
+  dir("python") {
+    sh "python3 setup.py install"
+  }
+  dir ("build") {
+    sh "cmake .."
+    sh "make -j4"
+  }
 }
 
-def pytorch_unit_test() {
-    withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) {
-        sh 'nosetests tests -v --with-xunit'
-        sh 'nosetests tests/pytorch -v --with-xunit'
-        sh 'nosetests tests/graph_index -v --with-xunit'
-    }
+def pytorch_unit_test(dev) {
+  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
+    sh "python3 -m nose -v --with-xunit tests"
+    sh "python3 -m nose -v --with-xunit tests/pytorch"
+    sh "python3 -m nose -v --with-xunit tests/graph_index"
+  }
 }
 
-def mxnet_unit_test() {
-    withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) {
-        sh 'nosetests tests/mxnet -v --with-xunit'
-    }
+def mxnet_unit_test(dev) {
+  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
+    sh "python3 -m nose -v --with-xunit tests/mxnet"
+  }
 }
 
 def example_test(dev) {
-    dir ('tests/scripts') {
-        withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) {
-            sh "./test_examples.sh ${dev}"
-        }
+  dir ("tests/scripts") {
+    withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
+      sh "./task_example_test.sh ${dev}"
+    }
+  }
+}
+
+def pytorch_tutorials() {
+  dir ("tests/scripts") {
+    withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
+      sh "./task_tutorial_test.sh"
     }
+  }
 }
 
 pipeline {
-    agent none
-    stages {
-        stage('Lint Check') {
-            agent {
-                docker {
-                    image 'lingfanyu/dgl-lint'
-                }
+  agent none
+  stages {
+    stage("Lint Check") {
+      agent { docker { image "dgllib/dgl-ci-lint" } }
+      steps {
+        init_git_submodule()
+        sh "tests/scripts/task_lint.sh"
+      }
+    }
+    stage("Build") {
+      parallel {
+        stage("CPU Build") {
+          agent { docker { image "dgllib/dgl-ci-cpu" } }
+          steps {
+            setup()
+            build_dgl()
+          }
+        }
+        stage("GPU Build") {
+          agent {
+            docker {
+              image "dgllib/dgl-ci-gpu"
+              args "--runtime nvidia"
+            }
+          }
+          steps {
+            setup()
+            build_dgl()
+          }
+        }
+        stage("MXNet CPU Build (temp)") {
+          agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } }
+          steps {
+            setup()
+            build_dgl()
+          }
+        }
+      }
+    }
+    stage("Test") {
+      parallel {
+        stage("Pytorch CPU") {
+          agent { docker { image "dgllib/dgl-ci-cpu" } }
+          stages {
+            stage("TH CPU unittest") {
+              steps { pytorch_unit_test("CPU") }
             }
-            stages {
-                stage('CHECK') {
-                    steps {
-                        init_git_submodule()
-                        sh 'tests/scripts/task_lint.sh'
-                    }
-                }
+            stage("TH CPU example test") {
+              steps { example_test("CPU") }
             }
+          }
+          post {
+            always { junit "*.xml" }
+          }
         }
-        stage('Build and Test on Pytorch') {
-            parallel {
-                stage('CPU') {
-                    agent {
-                        docker {
-                            image 'lingfanyu/dgl-cpu'
-                        }
-                    }
-                    stages {
-                        stage('SETUP') {
-                            steps {
-                                setup()
-                            }
-                        }
-                        stage('BUILD') {
-                            steps {
-                                build_dgl()
-                            }
-                        }
-                        stage('UNIT TEST') {
-                            steps {
-                                pytorch_unit_test()
-                            }
-                        }
-                        stage('EXAMPLE TEST') {
-                            steps {
-                                example_test('CPU')
-                            }
-                        }
-                    }
-                    post {
-                        always {
-                            junit '*.xml'
-                        }
-                    }
-                }
-                stage('GPU') {
-                    agent {
-                        docker {
-                            image 'lingfanyu/dgl-gpu'
-                            args '--runtime nvidia'
-                        }
-                    }
-                    stages {
-                        stage('SETUP') {
-                            steps {
-                                setup()
-                            }
-                        }
-                        stage('BUILD') {
-                            steps {
-                                build_dgl()
-                            }
-                        }
-                        stage('UNIT TEST') {
-                            steps {
-                                pytorch_unit_test()
-                            }
-                        }
-                        stage('EXAMPLE TEST') {
-                            steps {
-                                example_test('GPU')
-                            }
-                        }
-                    }
-                    post {
-                        always {
-                            junit '*.xml'
-                        }
-                    }
-                }
+        stage("Pytorch GPU") {
+          agent {
+            docker {
+              image "dgllib/dgl-ci-gpu"
+              args "--runtime nvidia"
             }
+          }
+          stages {
+            // TODO: have GPU unittest
+            //stage("TH GPU unittest") {
+            //  steps { pytorch_unit_test("GPU") }
+            //}
+            stage("TH GPU example test") {
+              steps { example_test("GPU") }
+            }
+          }
+          // TODO: have GPU unittest
+          //post {
+          //  always { junit "*.xml" }
+          //}
         }
-        stage('Build and Test on MXNet') {
-            parallel {
-                stage('CPU') {
-                    agent {
-                        docker {
-                            image 'zhengda1936/dgl-mxnet-cpu:v3'
-                        }
-                    }
-                    stages {
-                        stage('SETUP') {
-                            steps {
-                                setup()
-                            }
-                        }
-                        stage('BUILD') {
-                            steps {
-                                build_dgl()
-                            }
-                        }
-                        stage('UNIT TEST') {
-                            steps {
-                                mxnet_unit_test()
-                            }
-                        }
-                        stage('EXAMPLE TEST') {
-                            steps {
-                                example_test('CPU')
-                            }
-                        }
-                    }
-                    post {
-                        always {
-                            junit '*.xml'
-                        }
-                    }
-                }
+        stage("MXNet CPU") {
+          agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } }
+          stages {
+            stage("MX Unittest") {
+              steps { mxnet_unit_test("CPU") }
             }
+          }
+          post {
+            always { junit "*.xml" }
+          }
         }
+      }
+    }
+    stage("Doc") {
+      agent { docker { image "dgllib/dgl-ci-cpu" } }
+      steps {
+        pytorch_tutorials()
+      }
     }
+  }
 }

docker/Dockerfile.ci_cpu_mxnet

+# CI docker GPU env
+FROM nvidia/cuda:9.0-cudnn7-devel
+
+# Base scripts
+RUN apt-get update --fix-missing
+
+COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
+RUN bash /install/ubuntu_install_core.sh
+
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+
+COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
+RUN bash /install/ubuntu_install_python_package.sh
+
+COPY install/ubuntu_install_mxnet_cpu.sh /install/ubuntu_install_mxnet_cpu.sh
+RUN bash /install/ubuntu_install_mxnet_cpu.sh
+
+# Environment variables
+ENV PATH=/usr/local/nvidia/bin:${PATH}
+ENV PATH=/usr/local/cuda/bin:${PATH}
+ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
+ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
+ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
+ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}

docker/Dockerfile.ci_gpu_mxnet

+# CI docker GPU env
+FROM nvidia/cuda:9.0-cudnn7-devel
+
+# Base scripts
+RUN apt-get update --fix-missing
+
+COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
+RUN bash /install/ubuntu_install_core.sh
+
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+
+COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
+RUN bash /install/ubuntu_install_python_package.sh
+
+COPY install/ubuntu_install_mxnet_gpu.sh /install/ubuntu_install_mxnet_gpu.sh
+RUN bash /install/ubuntu_install_mxnet_gpu.sh
+
+ # Environment variables
+ENV PATH=/usr/local/nvidia/bin:${PATH}
+ENV PATH=/usr/local/cuda/bin:${PATH}
+ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
+ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
+ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
+ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}

docker/README.md

@@ -8,3 +8,9 @@ docker build -t dgl-gpu -f Dockerfile.ci_gpu .
 
 ### Lint image
 docker build -t dgl-lint -f Dockerfile.ci_lint .
+
+### CPU MXNet image
+docker build -t dgl-mxnet-cpu -f Dockerfile.ci_cpu_mxnet .
+
+### GPU MXNet image
+docker build -t dgl-mxnet-gpu -f Dockerfile.ci_gpu_mxnet .

docker/install/ubuntu_install_core.sh

 # install libraries for building c++ core on ubuntu
 apt update && apt install -y --no-install-recommends --force-yes \
-        apt-utils git build-essential make cmake wget unzip sudo libz-dev libxml2-dev
+        apt-utils git build-essential make cmake wget unzip sudo \
+        libz-dev libxml2-dev libopenblas-dev libopencv-dev ca-certificates

docker/install/ubuntu_install_mxnet_cpu.sh

+git clone --recursive -b dgl_graph https://github.com/zheng-da/incubator-mxnet.git
+cd incubator-mxnet
+make -j4 USE_OPENCV=1 USE_BLAS=openblas USE_MKLDNN=1 USE_DIST_KVSTORE=1
+pip3 install -e python

docker/install/ubuntu_install_mxnet_gpu.sh

+git clone --recursive -b dgl_graph https://github.com/zheng-da/incubator-mxnet.git
+cd incubator-mxnet
+make -j4 USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda \
+  USE_CUDNN=1 USE_MKLDNN=1 USE_DIST_KVSTORE=1
+pip3 install -e python

docker/install/ubuntu_install_python.sh

 # install python and pip, don't modify this, modify install_python_package.sh
-# apt-get update && apt-get install -y python-dev python-pip
+apt-get update
+apt-get install -y python-dev python3-dev
 
-# python 3.6
-apt-get update && yes | apt-get install software-properties-common
-add-apt-repository ppa:jonathonf/python-3.6
-apt-get update && apt-get install -y python3.6 python3.6-dev
-rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3
-
-# Install pip
+# install pip
 cd /tmp && wget https://bootstrap.pypa.io/get-pip.py
-# python2 get-pip.py
-python3.6 get-pip.py
+python2 get-pip.py
+python3 get-pip.py
+
+# santiy check
+python2 --version
+python3 --version
+pip2 --version
+pip3 --version

docker/install/ubuntu_install_python_package.sh

 # install libraries for python package on ubuntu
-# pip2 install pylint numpy cython scipy nltk requests[security]
-pip3 install pylint numpy cython scipy nltk requests[security]
+pip2 install nose numpy cython scipy networkx matplotlib nltk requests[security]
+pip3 install nose numpy cython scipy networkx matplotlib nltk requests[security]
 
 # install DL Framework
-# pip2 install torch torchvision
+pip2 install torch torchvision
 pip3 install torch torchvision

tests/pytorch/test_traversal.py

@@ -16,37 +16,42 @@ np.random.seed(42)
 def toset(x):
     return set(x.tolist())
 
-def test_bfs_nodes(n=1000):
-    g_nx = nx.random_tree(n)
-    g = dgl.DGLGraph()
-    g.from_networkx(g_nx)
+def test_bfs(n=1000):
+    def _bfs_nx(g_nx, src):
+        edges = nx.bfs_edges(g_nx, src)
+        layers_nx = [set([src])]
+        edges_nx = []
+        frontier = set()
+        edge_frontier = set()
+        for u, v in edges:
+            if u in layers_nx[-1]:
+                frontier.add(v)
+                edge_frontier.add(g.edge_id(u, v))
+            else:
+                layers_nx.append(frontier)
+                edges_nx.append(edge_frontier)
+                frontier = set([v])
+                edge_frontier = set([g.edge_id(u, v)])
+        layers_nx.append(frontier)
+        edges_nx.append(edge_frontier)
+        return layers_nx, edges_nx
 
+    g = dgl.DGLGraph()
+    a = sp.random(n, n, 10 / n, data_rvs=lambda n: np.ones(n))
+    g.from_scipy_sparse_matrix(a)
+    g_nx = g.to_networkx()
     src = random.choice(range(n))
-
-    edges = nx.bfs_edges(g_nx, src)
-    layers_nx = [set([src])]
-    edges_nx = []
-    frontier = set()
-    edge_frontier = set()
-    for u, v in edges:
-        if u in layers_nx[-1]:
-            frontier.add(v)
-            edge_frontier.add(g.edge_id(u, v))
-        else:
-            layers_nx.append(frontier)
-            edges_nx.append(edge_frontier)
-            frontier = set([v])
-            edge_frontier = set([g.edge_id(u, v)])
-    layers_nx.append(frontier)
-    edges_nx.append(edge_frontier)
-
+    layers_nx, _ = _bfs_nx(g_nx, src)
     layers_dgl = dgl.bfs_nodes_generator(g, src)
-
     assert len(layers_dgl) == len(layers_nx)
     assert all(toset(x) == y for x, y in zip(layers_dgl, layers_nx))
 
+    g_nx = nx.random_tree(n, seed=42)
+    g = dgl.DGLGraph()
+    g.from_networkx(g_nx)
+    src = 0
+    _, edges_nx = _bfs_nx(g_nx, src)
     edges_dgl = dgl.bfs_edges_generator(g, src)
-
     assert len(edges_dgl) == len(edges_nx)
     assert all(toset(x) == y for x, y in zip(edges_dgl, edges_nx))
 
@@ -114,6 +119,6 @@ def test_dfs_labeled_edges(n=1000, example=False):
 
 
 if __name__ == '__main__':
-    test_bfs_nodes()
+    test_bfs()
     test_topological_nodes()
     test_dfs_labeled_edges()

tests/scripts/task_tutorial_test.sh

+#!/bin/bash
+# The working directory for this script will be "tests/scripts"
+
+function fail {
+    echo FAIL: $@
+    exit -1
+}
+
+export MPLBACKEND=Agg
+
+for f in $(find "../../tutorials" -name "*.py")
+do
+    echo "Running tutorial ${f} ..."
+    python3 $f || fail "run ${f}"
+done