Jenkinsfile 8.97 KB
Newer Older
Lingfan Yu's avatar
Lingfan Yu committed
1
2
#!/usr/bin/env groovy

VoVAllen's avatar
VoVAllen committed
3
dgl_linux_libs = "build/libdgl.so, build/runUnitTests, python/dgl/_ffi/_cy3/core.cpython-36m-x86_64-linux-gnu.so"
Minjie Wang's avatar
Minjie Wang committed
4
// Currently DGL on Windows is not working with Cython yet
Minjie Wang's avatar
Minjie Wang committed
5
dgl_win64_libs = "build\\dgl.dll, build\\runUnitTests.exe"
Minjie Wang's avatar
Minjie Wang committed
6
7
8
9

def init_git() {
  sh "rm -rf *"
  checkout scm
10
  sh "git submodule update --recursive --init"
Lingfan Yu's avatar
Lingfan Yu committed
11
12
}

Minjie Wang's avatar
Minjie Wang committed
13
14
def init_git_win64() {
  checkout scm
15
  bat "git submodule update --recursive --init"
Lingfan Yu's avatar
Lingfan Yu committed
16
17
}

Minjie Wang's avatar
Minjie Wang committed
18
19
20
21
22
23
24
25
26
27
28
29
30
31
// pack libraries for later use
def pack_lib(name, libs) {
  echo "Packing ${libs} into ${name}"
  stash includes: libs, name: name
}

// unpack libraries saved before
def unpack_lib(name, libs) {
  unstash name
  echo "Unpacked ${libs} from ${name}"
}

def build_dgl_linux(dev) {
  init_git()
32
  sh "bash tests/scripts/build_dgl.sh ${dev}"
Minjie Wang's avatar
Minjie Wang committed
33
  pack_lib("dgl-${dev}-linux", dgl_linux_libs)
Lingfan Yu's avatar
Lingfan Yu committed
34
35
}

Minjie Wang's avatar
Minjie Wang committed
36
def build_dgl_win64(dev) {
37
38
  /* Assuming that Windows slaves are already configured with MSBuild VS2017,
   * CMake and Python/pip/setuptools etc. */
Minjie Wang's avatar
Minjie Wang committed
39
  init_git_win64()
40
  bat "CALL tests\\scripts\\build_dgl.bat"
Minjie Wang's avatar
Minjie Wang committed
41
  pack_lib("dgl-${dev}-win64", dgl_win64_libs)
42
43
}

Minjie Wang's avatar
Minjie Wang committed
44
45
def cpp_unit_test_linux() {
  init_git()
Minjie Wang's avatar
Minjie Wang committed
46
  unpack_lib("dgl-cpu-linux", dgl_linux_libs)
VoVAllen's avatar
VoVAllen committed
47
48
49
  sh "bash tests/scripts/task_cpp_unit_test.sh"
}

Minjie Wang's avatar
Minjie Wang committed
50
51
def cpp_unit_test_win64() {
  init_git_win64()
Minjie Wang's avatar
Minjie Wang committed
52
  unpack_lib("dgl-cpu-win64", dgl_win64_libs)
VoVAllen's avatar
VoVAllen committed
53
54
55
  bat "CALL tests\\scripts\\task_cpp_unit_test.bat"
}

Minjie Wang's avatar
Minjie Wang committed
56
def unit_test_linux(backend, dev) {
Minjie Wang's avatar
Minjie Wang committed
57
  init_git()
Minjie Wang's avatar
Minjie Wang committed
58
  unpack_lib("dgl-${dev}-linux", dgl_linux_libs)
VoVAllen's avatar
VoVAllen committed
59
  timeout(time: 10, unit: 'MINUTES') {
60
    sh "bash tests/scripts/task_unit_test.sh ${backend} ${dev}"
Minjie Wang's avatar
Minjie Wang committed
61
  }
Lingfan Yu's avatar
Lingfan Yu committed
62
63
}

64
def unit_test_win64(backend, dev) {
Minjie Wang's avatar
Minjie Wang committed
65
  init_git_win64()
Minjie Wang's avatar
Minjie Wang committed
66
  unpack_lib("dgl-${dev}-win64", dgl_win64_libs)
Minjie Wang's avatar
Minjie Wang committed
67
  timeout(time: 2, unit: 'MINUTES') {
68
    bat "CALL tests\\scripts\\task_unit_test.bat ${backend}"
Minjie Wang's avatar
Minjie Wang committed
69
  }
Da Zheng's avatar
Da Zheng committed
70
71
}

Minjie Wang's avatar
Minjie Wang committed
72
def example_test_linux(backend, dev) {
Minjie Wang's avatar
Minjie Wang committed
73
  init_git()
Minjie Wang's avatar
Minjie Wang committed
74
  unpack_lib("dgl-${dev}-linux", dgl_linux_libs)
Minjie Wang's avatar
Minjie Wang committed
75
76
  timeout(time: 20, unit: 'MINUTES') {
    sh "bash tests/scripts/task_example_test.sh ${dev}"
Minjie Wang's avatar
Minjie Wang committed
77
78
79
  }
}

80
def example_test_win64(backend, dev) {
Minjie Wang's avatar
Minjie Wang committed
81
  init_git_win64()
Minjie Wang's avatar
Minjie Wang committed
82
  unpack_lib("dgl-${dev}-win64", dgl_win64_libs)
Minjie Wang's avatar
Minjie Wang committed
83
84
  timeout(time: 20, unit: 'MINUTES') {
    bat "CALL tests\\scripts\\task_example_test.bat ${dev}"
85
86
87
  }
}

Minjie Wang's avatar
Minjie Wang committed
88
def tutorial_test_linux(backend) {
Minjie Wang's avatar
Minjie Wang committed
89
  init_git()
Minjie Wang's avatar
Minjie Wang committed
90
  unpack_lib("dgl-cpu-linux", dgl_linux_libs)
Minjie Wang's avatar
Minjie Wang committed
91
92
  timeout(time: 20, unit: 'MINUTES') {
    sh "bash tests/scripts/task_${backend}_tutorial_test.sh"
Minjie Wang's avatar
Minjie Wang committed
93
  }
Lingfan Yu's avatar
Lingfan Yu committed
94
95
}

VoVAllen's avatar
VoVAllen committed
96

Minjie Wang's avatar
Minjie Wang committed
97
pipeline {
98
  agent any
Minjie Wang's avatar
Minjie Wang committed
99
100
  stages {
    stage("Lint Check") {
VoVAllen's avatar
VoVAllen committed
101
102
      agent { 
        docker {
103
          label "linux-c52x-node"
VoVAllen's avatar
VoVAllen committed
104
105
106
          image "dgllib/dgl-ci-lint" 
        }
      }
Minjie Wang's avatar
Minjie Wang committed
107
      steps {
Minjie Wang's avatar
Minjie Wang committed
108
        init_git()
109
        sh "bash tests/scripts/task_lint.sh"
Minjie Wang's avatar
Minjie Wang committed
110
      }
111
112
113
114
115
      post {
        always {
          cleanWs disableDeferredWipeout: true, deleteDirs: true
        }
      }
Minjie Wang's avatar
Minjie Wang committed
116
117
118
119
    }
    stage("Build") {
      parallel {
        stage("CPU Build") {
VoVAllen's avatar
VoVAllen committed
120
121
          agent { 
            docker {
122
              label "linux-c52x-node"
VoVAllen's avatar
VoVAllen committed
123
124
125
              image "dgllib/dgl-ci-cpu:conda" 
            }
          }
Minjie Wang's avatar
Minjie Wang committed
126
          steps {
Minjie Wang's avatar
Minjie Wang committed
127
            build_dgl_linux("cpu")
Minjie Wang's avatar
Minjie Wang committed
128
          }
129
130
131
132
133
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
Minjie Wang's avatar
Minjie Wang committed
134
135
136
137
        }
        stage("GPU Build") {
          agent {
            docker {
138
              label "linux-c52x-node"
VoVAllen's avatar
VoVAllen committed
139
              image "dgllib/dgl-ci-gpu:conda"
VoVAllen's avatar
VoVAllen committed
140
              args "-u root"
Minjie Wang's avatar
Minjie Wang committed
141
142
143
            }
          }
          steps {
Jinjing Zhou's avatar
Jinjing Zhou committed
144
            // sh "nvidia-smi"
Minjie Wang's avatar
Minjie Wang committed
145
            build_dgl_linux("gpu")
Minjie Wang's avatar
Minjie Wang committed
146
          }
147
148
149
150
151
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
Minjie Wang's avatar
Minjie Wang committed
152
        }
Minjie Wang's avatar
Minjie Wang committed
153
154
155
156
        stage("CPU Build (Win64)") {
          // Windows build machines are manually added to Jenkins master with
          // "windows" label as permanent agents.
          agent { label "windows" }
157
          steps {
Minjie Wang's avatar
Minjie Wang committed
158
            build_dgl_win64("cpu")
159
          }
160
161
162
163
164
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
165
        }
Minjie Wang's avatar
Minjie Wang committed
166
        // Currently we don't have Windows GPU build machines
Minjie Wang's avatar
Minjie Wang committed
167
168
169
170
      }
    }
    stage("Test") {
      parallel {
Minjie Wang's avatar
Minjie Wang committed
171
        stage("C++ CPU") {
VoVAllen's avatar
VoVAllen committed
172
173
          agent { 
            docker { 
174
              label "linux-c52x-node"
VoVAllen's avatar
VoVAllen committed
175
176
177
              image "dgllib/dgl-ci-cpu:conda"
            }
          }
Minjie Wang's avatar
Minjie Wang committed
178
179
          steps {
            cpp_unit_test_linux()
VoVAllen's avatar
VoVAllen committed
180
          }
181
182
183
184
185
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
VoVAllen's avatar
VoVAllen committed
186
        }
Minjie Wang's avatar
Minjie Wang committed
187
188
189
190
        stage("C++ CPU (Win64)") {
          agent { label "windows" }
          steps {
            cpp_unit_test_win64()
191
          }
192
193
194
195
196
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
Minjie Wang's avatar
Minjie Wang committed
197
        }
Jinjing Zhou's avatar
Jinjing Zhou committed
198
199
200
        stage("Tensorflow CPU") {
          agent { 
            docker {
201
              label "linux-c52x-node"
Jinjing Zhou's avatar
Jinjing Zhou committed
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
              image "dgllib/dgl-ci-cpu:conda" 
            }
          }
          stages {
            stage("Unit test") {
              steps {
                unit_test_linux("tensorflow", "cpu")
              }
            }
          }
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
        }
        stage("Tensorflow GPU") {
          agent { 
            docker { 
              label "linux-gpu-node"
              image "dgllib/dgl-ci-gpu:conda" 
              args "--runtime nvidia"
            }
          }
          stages {
            stage("Unit test") {
              steps {
                unit_test_linux("tensorflow", "gpu")
              }
            }
          }
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
        }
Minjie Wang's avatar
Minjie Wang committed
239
        stage("Torch CPU") {
VoVAllen's avatar
VoVAllen committed
240
241
          agent { 
            docker {
242
              label "linux-c52x-node"
VoVAllen's avatar
VoVAllen committed
243
244
245
              image "dgllib/dgl-ci-cpu:conda" 
            }
          }
Minjie Wang's avatar
Minjie Wang committed
246
          stages {
Minjie Wang's avatar
Minjie Wang committed
247
248
            stage("Unit test") {
              steps {
Minjie Wang's avatar
Minjie Wang committed
249
                unit_test_linux("pytorch", "cpu")
Minjie Wang's avatar
Minjie Wang committed
250
              }
Lingfan Yu's avatar
Lingfan Yu committed
251
            }
Minjie Wang's avatar
Minjie Wang committed
252
253
            stage("Example test") {
              steps {
Minjie Wang's avatar
Minjie Wang committed
254
                example_test_linux("pytorch", "cpu")
Minjie Wang's avatar
Minjie Wang committed
255
256
257
258
              }
            }
            stage("Tutorial test") {
              steps {
Minjie Wang's avatar
Minjie Wang committed
259
                tutorial_test_linux("pytorch")
Minjie Wang's avatar
Minjie Wang committed
260
              }
261
262
            }
          }
263
264
265
266
267
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
268
        }
Minjie Wang's avatar
Minjie Wang committed
269
        stage("Torch CPU (Win64)") {
270
271
          agent { label "windows" }
          stages {
Minjie Wang's avatar
Minjie Wang committed
272
273
274
275
            stage("Unit test") {
              steps {
                unit_test_win64("pytorch", "cpu")
              }
276
            }
Minjie Wang's avatar
Minjie Wang committed
277
278
279
280
            stage("Example test") {
              steps {
                example_test_win64("pytorch", "cpu")
              }
Lingfan Yu's avatar
Lingfan Yu committed
281
            }
Minjie Wang's avatar
Minjie Wang committed
282
          }
283
284
285
286
287
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
Lingfan Yu's avatar
Lingfan Yu committed
288
        }
Minjie Wang's avatar
Minjie Wang committed
289
        stage("Torch GPU") {
Minjie Wang's avatar
Minjie Wang committed
290
291
          agent {
            docker {
VoVAllen's avatar
VoVAllen committed
292
              label "linux-gpu-node"
VoVAllen's avatar
VoVAllen committed
293
              image "dgllib/dgl-ci-gpu:conda"
Minjie Wang's avatar
Minjie Wang committed
294
              args "--runtime nvidia"
Minjie Wang's avatar
Minjie Wang committed
295
            }
Minjie Wang's avatar
Minjie Wang committed
296
297
          }
          stages {
Minjie Wang's avatar
Minjie Wang committed
298
299
300
            stage("Unit test") {
              steps {
                sh "nvidia-smi"
301
                unit_test_linux("pytorch", "gpu")
Minjie Wang's avatar
Minjie Wang committed
302
303
304
305
              }
            }
            stage("Example test") {
              steps {
Minjie Wang's avatar
Minjie Wang committed
306
                example_test_linux("pytorch", "gpu")
Minjie Wang's avatar
Minjie Wang committed
307
              }
Minjie Wang's avatar
Minjie Wang committed
308
309
            }
          }
310
311
312
313
314
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
Minjie Wang's avatar
Minjie Wang committed
315
        }
Minjie Wang's avatar
Minjie Wang committed
316
        stage("MXNet CPU") {
VoVAllen's avatar
VoVAllen committed
317
318
          agent { 
            docker {
319
              label "linux-c52x-node"
VoVAllen's avatar
VoVAllen committed
320
321
322
              image "dgllib/dgl-ci-cpu:conda" 
            }
          }
Minjie Wang's avatar
Minjie Wang committed
323
          stages {
Minjie Wang's avatar
Minjie Wang committed
324
325
            stage("Unit test") {
              steps {
Minjie Wang's avatar
Minjie Wang committed
326
                unit_test_linux("mxnet", "cpu")
VoVAllen's avatar
VoVAllen committed
327
              }
Da Zheng's avatar
Da Zheng committed
328
            }
329
            //stage("Tutorial test") {
Minjie Wang's avatar
Minjie Wang committed
330
            //  steps {
331
            //    tutorial_test_linux("mxnet")
Minjie Wang's avatar
Minjie Wang committed
332
333
            //  }
            //}
334
335
336
337
          }
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
338
            }
339
340
          }
        }
341
342
343
        stage("MXNet GPU") {
          agent {
            docker {
VoVAllen's avatar
VoVAllen committed
344
              label "linux-gpu-node" 
VoVAllen's avatar
VoVAllen committed
345
              image "dgllib/dgl-ci-gpu:conda"
346
347
348
349
350
351
352
353
354
355
              args "--runtime nvidia"
            }
          }
          stages {
            stage("Unit test") {
              steps {
                sh "nvidia-smi"
                unit_test_linux("mxnet", "gpu")
              }
            }
356
357
358
359
          }
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
360
            }
361
362
          }
        }
Minjie Wang's avatar
Minjie Wang committed
363
      }
Minjie Wang's avatar
Minjie Wang committed
364
    }
Minjie Wang's avatar
Minjie Wang committed
365
  }
366
367
368
369
370
371
372
  post {
    always {
      node('windows') {
        bat "rmvirtualenv ${BUILD_TAG}"
      }
    }
  }
Minjie Wang's avatar
Minjie Wang committed
373
}