[CI] Fix CI bugs (#592)

* new jenkins script

* fix ci

* poke ci

* new config

* new config

* new config

* poke ci

* poke ci

* poke ci

* poke ci

* poke ci

* poke ci

* poke ci

* update docker image; poke ci

* poke ci

* poke ci

* poke ci

* poke ci

* poke ci

* poke ci

* poke ci

* poke ci

* poke ci

* poke ci

* poke ci

* update image

* update image

* fix

* Windows CI support

* typo

* typo*2

* missed sh

* typo*3

* missed dir

Jenkinsfile

 #!/usr/bin/env groovy
 
-def init_git_submodule() {
+dgl_linux_libs = "build/libdgl.so, build/runUnitTests, python/dgl/_ffi/_cy3/core.cpython-35m-x86_64-linux-gnu.so"
+// Currently DGL on Windows is not working with Cython yet
+dgl_windows_libs = "build\\dgl.dll, build\\runUnitTests.exe"
+
+def init_git() {
+  sh "rm -rf *"
+  checkout scm
   sh "git submodule init"
   sh "git submodule update"
+  sh "ls -lh"
 }
 
-def init_git_submodule_win64() {
+def init_git_win64() {
+  checkout scm
   bat "git submodule init"
   bat "git submodule update"
 }
 
-def build_dgl() {
+// pack libraries for later use
+def pack_lib(name, libs) {
+  echo "Packing ${libs} into ${name}"
+  stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs) {
+  unstash name
+  echo "Unpacked ${libs} from ${name}"
+}
+
+def build_dgl_linux(dev) {
+  init_git()
   sh "bash tests/scripts/build_dgl.sh"
+  pack_lib("dgl-${dev}", dgl_linux_libs)
 }
 
-def build_dgl_win64() {
+def build_dgl_win64(dev) {
   /* Assuming that Windows slaves are already configured with MSBuild VS2017,
    * CMake and Python/pip/setuptools etc. */
+  init_git_win64()
   bat "CALL tests\\scripts\\build_dgl.bat"
+  pack_lib("dgl-${dev}", dgl_windows_libs)
 }
 
-def cpp_unit_test_linux(){
+def cpp_unit_test_linux() {
+  init_git()
+  unpack_lib("dgl-cpu", dgl_linux_libs)
   sh "bash tests/scripts/task_cpp_unit_test.sh"
 }
 
-def cpp_unit_test_windows(){
+def cpp_unit_test_win64() {
+  init_git_win64()
+  unpack_lib("dgl-cpu", dgl_windows_libs)
   bat "CALL tests\\scripts\\task_cpp_unit_test.bat"
 }
 
 def unit_test(backend, dev) {
-  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python", "DGLBACKEND=${backend}"]) {
+  init_git()
+  unpack_lib("dgl-${dev}", dgl_linux_libs)
+  timeout(time: 2, unit: 'MINUTES') {
     sh "bash tests/scripts/task_unit_test.sh ${backend}"
   }
 }
 
 def unit_test_win64(backend, dev) {
-  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}\\build", "PYTHONPATH=${env.WORKSPACE}\\python", "DGLBACKEND=${backend}"]) {
+  init_git_win64()
+  unpack_lib("dgl-${dev}", dgl_windows_libs)
+  timeout(time: 2, unit: 'MINUTES') {
     bat "CALL tests\\scripts\\task_unit_test.bat ${backend}"
   }
 }
 
 def example_test(backend, dev) {
-  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python", "DGLBACKEND=${backend}"]) {
-    dir ("tests/scripts") {
-      sh "bash task_example_test.sh ${dev}"
-    }
+  init_git()
+  unpack_lib("dgl-${dev}", dgl_linux_libs)
+  timeout(time: 20, unit: 'MINUTES') {
+    sh "bash tests/scripts/task_example_test.sh ${dev}"
   }
 }
 
 def example_test_win64(backend, dev) {
-  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}\\build", "PYTHONPATH=${env.WORKSPACE}\\python", "DGLBACKEND=${backend}"]) {
-    dir ("tests\\scripts") {
-      bat "CALL task_example_test ${dev}"
-    }
+  init_git_win64()
+  unpack_lib("dgl-${dev}", dgl_windows_libs)
+  timeout(time: 20, unit: 'MINUTES') {
+    bat "CALL tests\\scripts\\task_example_test.bat ${dev}"
   }
 }
 
-def pytorch_tutorials() {
-  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
-    dir ("tests/scripts") {
-      sh "bash task_pytorch_tutorial_test.sh"
-    }
+def tutorial_test(backend) {
+  init_git()
+  unpack_lib("dgl-cpu", dgl_linux_libs)
+  timeout(time: 20, unit: 'MINUTES') {
+    sh "bash tests/scripts/task_${backend}_tutorial_test.sh"
   }
 }
 
-def mxnet_tutorials() {
-  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python", "DGLBACKEND=mxnet"]) {
-    dir("tests/scripts") {
-      sh "bash task_mxnet_tutorial_test.sh"
-    }
-  }
-}
 pipeline {
   agent none
   stages {
     stage("Lint Check") {
-      agent {
-        docker { image "dgllib/dgl-ci-lint" }
-      }
+      agent { docker { image "dgllib/dgl-ci-lint" } }
       steps {
-        init_git_submodule()
+        init_git()
         sh "bash tests/scripts/task_lint.sh"
       }
     }
     stage("Build") {
       parallel {
         stage("CPU Build") {
-          agent {
-            docker { image "dgllib/dgl-ci-cpu" }
-          }
+          agent { docker { image "dgllib/dgl-ci-cpu" } }
           steps {
-            init_git_submodule()
-            build_dgl()
+            build_dgl_linux("cpu")
           }
         }
         stage("GPU Build") {
@@ -102,85 +122,71 @@ pipeline {
             }
           }
           steps {
-            init_git_submodule()
-            build_dgl()
-          }
-        }
-        stage("MXNet CPU Build (temp)") {
-          agent {
-            docker { image "dgllib/dgl-ci-mxnet-cpu" }
-          }
-          steps {
-            init_git_submodule()
-            build_dgl()
+            sh "nvidia-smi"
+            build_dgl_linux("gpu")
           }
         }
-        stage("CPU Build (Win64/PyTorch)") {
-          agent {
-            label "windows"
-          }
+        stage("CPU Build (Win64)") {
+          // Windows build machines are manually added to Jenkins master with
+          // "windows" label as permanent agents.
+          agent { label "windows" }
           steps {
-            init_git_submodule_win64()
-            build_dgl_win64()
+            build_dgl_win64("cpu")
           }
         }
+        // Currently we don't have Windows GPU build machines
       }
     }
     stage("Test") {
       parallel {
-        stage("CPP Test"){
-          stages{
-            stage("CPP Unit Test Linux"){
-              agent {
-                docker {image "dgllib/dgl-ci-cpu"}
-              }
-              steps { 
-                init_git_submodule()
-                cpp_unit_test_linux() 
-              }
-            }
-            stage("CPP Unit Test Windows"){
-              agent {
-                label "windows"
-              }
-              steps {
-                init_git_submodule_win64()
-                cpp_unit_test_windows()
-              }
-            }
+        stage("C++ CPU") {
+          agent { docker { image "dgllib/dgl-ci-cpu" } }
+          steps {
+            cpp_unit_test_linux()
           }
         }
-        stage("Pytorch CPU") {
-          agent {
-            docker { image "dgllib/dgl-ci-cpu" }
+        stage("C++ CPU (Win64)") {
+          agent { label "windows" }
+          steps {
+            cpp_unit_test_win64()
           }
+        }
+        stage("Torch CPU") {
+          agent { docker { image "dgllib/dgl-ci-cpu" } }
           stages {
-            stage("TH CPU unittest") {
-              steps { unit_test("pytorch", "CPU") }
+            stage("Unit test") {
+              steps {
+                unit_test("pytorch", "cpu")
+              }
             }
-            stage("TH CPU example test") {
-              steps { example_test("pytorch", "CPU") }
+            stage("Example test") {
+              steps {
+                example_test("pytorch", "cpu")
+              }
+            }
+            stage("Tutorial test") {
+              steps {
+                tutorial_test("pytorch")
+              }
             }
-          }
-          post {
-            always { junit "*.xml" }
           }
         }
-        stage("Pytorch CPU (Windows)") {
+        stage("Torch CPU (Win64)") {
           agent { label "windows" }
           stages {
-            stage("TH CPU Win64 unittest") {
-              steps { unit_test_win64("pytorch", "CPU") }
+            stage("Unit test") {
+              steps {
+                unit_test_win64("pytorch", "cpu")
+              }
             }
-            stage("TH CPU Win64 example test") {
-              steps { example_test_win64("pytorch", "CPU") }
+            stage("Example test") {
+              steps {
+                example_test_win64("pytorch", "cpu")
+              }
             }
           }
-          post {
-            always { junit "*.xml" }
-          }
         }
-        stage("Pytorch GPU") {
+        stage("Torch GPU") {
           agent {
             docker {
               image "dgllib/dgl-ci-gpu"
@@ -188,55 +194,39 @@ pipeline {
             }
           }
           stages {
-            // TODO: have GPU unittest
-            //stage("TH GPU unittest") {
-            //  steps { pytorch_unit_test("GPU") }
-            //}
-            stage("TH GPU example test") {
-              steps { example_test("pytorch", "GPU") }
+            stage("Unit test") {
+              steps {
+                //unit_test("pytorch", "gpu")
+                sh "nvidia-smi"
+              }
+            }
+            stage("Example test") {
+              steps {
+                example_test("pytorch", "gpu")
+              }
             }
           }
-          // TODO: have GPU unittest
-          //post {
-          //  always { junit "*.xml" }
-          //}
         }
         stage("MXNet CPU") {
-          agent {
-            docker { image "dgllib/dgl-ci-mxnet-cpu" }
-          }
+          agent { docker { image "dgllib/dgl-ci-cpu" } }
           stages {
-            stage("MX Unittest") {
-              options {
-                  timeout(time: 5, unit: 'MINUTES') 
+            stage("Unit test") {
+              steps {
+                unit_test("mxnet", "cpu")
               }
-              steps { unit_test("mxnet", "CPU") }
             }
-          }
-          post {
-            always { junit "*.xml" }
-          }
-        }
-      }
-    }
-    stage("Doc") {
-      parallel {
-        stage("TH Tutorial") {
-          agent {
-            docker { image "dgllib/dgl-ci-cpu" }
-          }
-          steps {
-            pytorch_tutorials()
+            //stage("Example test") {
+            //  steps {
+            //    unit_test("pytorch", "cpu")
+            //  }
+            //}
+            //stage("Tutorial test") {
+            //  steps {
+            //    tutorial_test("mxnet")
+            //  }
+            //}
           }
         }
-        //stage("MX Tutorial") {
-        //  agent {
-        //    docker { image "dgllib/dgl-ci-mxnet-cpu" }
-        //  }
-        //  steps {
-        //    mxnet_tutorials()
-        //  }
-        //}
       }
     }
   }

docker/Dockerfile.ci_cpu

@@ -10,6 +10,14 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_build.sh /install/ubuntu_install_build.sh
 RUN bash /install/ubuntu_install_build.sh
 
+# ANTLR deps
+COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
+RUN bash /install/ubuntu_install_java.sh
+
+COPY install/ubuntu_install_antlr.sh /install/ubuntu_install_antlr.sh
+RUN bash /install/ubuntu_install_antlr.sh
+
+# python
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
 RUN bash /install/ubuntu_install_python.sh
 
@@ -18,3 +26,6 @@ RUN bash /install/ubuntu_install_python_package.sh
 
 COPY install/ubuntu_install_torch.sh /install/ubuntu_install_torch.sh
 RUN bash /install/ubuntu_install_torch.sh
+
+COPY install/ubuntu_install_mxnet_cpu.sh /install/ubuntu_install_mxnet_cpu.sh
+RUN bash /install/ubuntu_install_mxnet_cpu.sh

docker/Dockerfile.ci_gpu

@@ -10,6 +10,14 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_build.sh /install/ubuntu_install_build.sh
 RUN bash /install/ubuntu_install_build.sh
 
+# ANTLR deps
+COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
+RUN bash /install/ubuntu_install_java.sh
+
+COPY install/ubuntu_install_antlr.sh /install/ubuntu_install_antlr.sh
+RUN bash /install/ubuntu_install_antlr.sh
+
+# python
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
 RUN bash /install/ubuntu_install_python.sh
 

docker/install/ubuntu_install_antlr.sh

+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+cd /usr/local/lib
+wget -q https://www.antlr.org/download/antlr-4.7.1-complete.jar
+cd -

docker/install/ubuntu_install_java.sh

+#!/bin/bash
+
+set -o errexit -o nounset
+set -o pipefail
+
+apt-get update && apt-get install -y openjdk-8-jdk maven
+test -d "/usr/lib/jvm/java-8-openjdk-amd64/jre"
+echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre" >> /etc/profile

docker/install/ubuntu_install_mxnet_cpu.sh

-pip3 install mxnet==1.5.0b20190528
+pip3 install mxnet==1.5.0b20190523

docker/install/ubuntu_install_mxnet_gpu.sh

-pip3 install mxnet-cu90==1.5.0b20190528
+pip3 install mxnet-cu90==1.5.0b20190523

tests/scripts/task_cpp_unit_test.sh

 #!/bin/bash
 pushd build
 ./runUnitTests
+popd

tests/scripts/task_example_test.bat

 @ECHO OFF
 SETLOCAL EnableDelayedExpansion
 
+SET GCN_EXAMPLE_DIR=.\examples\pytorch
+
 IF x%1x==xx (
 	ECHO Must supply CPU or GPU
 	GOTO :FAIL
-) ELSE IF x%1x==xCPUx (
+) ELSE IF x%1x==xcpux (
 	SET DEV=-1
-) ELSE IF x%1x==xGPUx (
+) ELSE IF x%1x==xgpux (
 	SET DEV=0
 	SET CUDA_VISIBLE_DEVICES=0
 ) ELSE (
@@ -14,10 +16,15 @@ IF x%1x==xx (
 	GOTO :FAIL
 )
 
-PUSHD ..\..\examples\pytorch
+SET DGLBACKEND=pytorch
+SET DGL_LIBRARY_PATH=!CD!\build
+SET PYTHONPATH=!CD!\python;!PYTHONPATH!
+SET DGL_DOWNLOAD_DIR=!CD!
+
+PUSHD !GCN_EXAMPLE_DIR!
 python pagerank.py || GOTO :FAIL
-python gcn\gcn.py --dataset cora --gpu !dev! || GOTO :FAIL
-python gcn\gcn_spmv.py --dataset cora --gpu !dev! || GOTO :FAIL
+python gcn\gcn.py --dataset cora --gpu !DEV! || GOTO :FAIL
+python gcn\gcn_spmv.py --dataset cora --gpu !DEV! || GOTO :FAIL
 POPD
 ENDLOCAL
 EXIT /B

tests/scripts/task_example_test.sh

 #!/bin/bash
 
-GCN_EXAMPLE_DIR="../../examples/pytorch/"
+GCN_EXAMPLE_DIR="./examples/pytorch/"
 
 function fail {
     echo FAIL: $@
@@ -8,7 +8,7 @@ function fail {
 }
 
 function usage {
-    echo "Usage: $0 [CPU|GPU]"
+    echo "Usage: $0 [cpu|gpu]"
 }
 
 # check arguments
@@ -17,9 +17,9 @@ if [ $# -ne 1 ]; then
     fail "Error: must specify device"
 fi
 
-if [ "$1" == "CPU" ]; then
+if [ "$1" == "cpu" ]; then
     dev=-1
-elif [ "$1" == "GPU" ]; then
+elif [ "$1" == "gpu" ]; then
     export CUDA_VISIBLE_DEVICES=0
     dev=0
 else
@@ -27,9 +27,15 @@ else
     fail "Unknown device $1"
 fi
 
-pushd $GCN_EXAMPLE_DIR> /dev/null
+export DGLBACKEND=pytorch
+export DGL_LIBRARY_PATH=${PWD}/build
+export PYTHONPATH=${PWD}/python:$PYTHONPATH
+export DGL_DOWNLOAD_DIR=${PWD}
 
 # test
+
+pushd $GCN_EXAMPLE_DIR> /dev/null
+
 python3 pagerank.py || fail "run pagerank.py on $1"
 python3 gcn/gcn.py --dataset cora --gpu $dev || fail "run gcn/gcn.py on $1"
 python3 gcn/gcn_spmv.py --dataset cora --gpu $dev || fail "run gcn/gcn_spmv.py on $1"

tests/scripts/task_mxnet_tutorial_test.sh

 #!/bin/bash
 # The working directory for this script will be "tests/scripts"
 
-TUTORIAL_ROOT="../../tutorials"
+TUTORIAL_ROOT="./tutorials"
 
 function fail {
     echo FAIL: $@
     exit -1
 }
 
+export MPLBACKEND=Agg
+export DGLBACKEND=mxnet
+export DGL_LIBRARY_PATH=${PWD}/build
+export PYTHONPATH=${PWD}/python:$PYTHONPATH
+export DGL_DOWNLOAD_DIR=${PWD}
+
 pushd ${TUTORIAL_ROOT} > /dev/null
 # Install requirements
 pip3 install -r requirements.txt || fail "installing requirements"
 
 # Test
-export MPLBACKEND=Agg
 for f in $(find . -name "*_mx.py")
 do
     echo "Running tutorial ${f} ..."

tests/scripts/task_pytorch_tutorial_test.sh

 #!/bin/bash
 # The working directory for this script will be "tests/scripts"
 
-TUTORIAL_ROOT="../../tutorials"
+TUTORIAL_ROOT="./tutorials"
 
 function fail {
     echo FAIL: $@
     exit -1
 }
 
+export MPLBACKEND=Agg
+export DGLBACKEND=pytorch
+export DGL_LIBRARY_PATH=${PWD}/build
+export PYTHONPATH=${PWD}/python:$PYTHONPATH
+export DGL_DOWNLOAD_DIR=${PWD}
+
 pushd ${TUTORIAL_ROOT} > /dev/null
 # Install requirements
 pip3 install -r requirements.txt || fail "installing requirements"
 
 # Test
-export MPLBACKEND=Agg
 for f in $(find . -name "*.py" ! -name "*_mx.py")
 do
     echo "Running tutorial ${f} ..."

tests/scripts/task_unit_test.bat

@@ -8,9 +8,13 @@ IF x%1x==xx (
 	SET BACKEND=%1
 )
 
-SET PYTHONPATH=tests;!PYTHONPATH!
+SET PYTHONPATH=tests;!CD!\python;!PYTHONPATH!
+SET DGLBACKEND=!BACKEND!
+SET DGL_LIBRARY_PATH=!CD!\build
+SET DGL_DOWNLOAD_DIR=!CD!
 
 python -m nose -v --with-xunit tests\!BACKEND! || EXIT /B 1
 python -m nose -v --with-xunit tests\graph_index || EXIT /B 1
 python -m nose -v --with-xunit tests\compute || EXIT /B 1
+ENDLOCAL
 EXIT /B

tests/scripts/task_unit_test.sh

@@ -15,7 +15,10 @@ if [ $# -ne 1 ]; then
 fi
 
 BACKEND=$1
-export PYTHONPATH=tests:$PYTHONPATH
+export DGLBACKEND=$1
+export DGL_LIBRARY_PATH=${PWD}/build
+export PYTHONPATH=tests:${PWD}/python:$PYTHONPATH
+export DGL_DOWNLOAD_DIR=${PWD}
 
 python3 -m nose -v --with-xunit tests/$BACKEND || fail "backend-specific"
 python3 -m nose -v --with-xunit tests/graph_index || fail "graph_index"