Jenkinsfile 9.73 KB
Newer Older
Lingfan Yu's avatar
Lingfan Yu committed
1
2
#!/usr/bin/env groovy

VoVAllen's avatar
VoVAllen committed
3
dgl_linux_libs = "build/libdgl.so, build/runUnitTests, python/dgl/_ffi/_cy3/core.cpython-36m-x86_64-linux-gnu.so"
Minjie Wang's avatar
Minjie Wang committed
4
// Currently DGL on Windows is not working with Cython yet
Minjie Wang's avatar
Minjie Wang committed
5
dgl_win64_libs = "build\\dgl.dll, build\\runUnitTests.exe"
Minjie Wang's avatar
Minjie Wang committed
6
7
8
9

def init_git() {
  sh "rm -rf *"
  checkout scm
10
  sh "git submodule update --recursive --init"
Lingfan Yu's avatar
Lingfan Yu committed
11
12
}

Minjie Wang's avatar
Minjie Wang committed
13
14
def init_git_win64() {
  checkout scm
15
  bat "git submodule update --recursive --init"
Lingfan Yu's avatar
Lingfan Yu committed
16
17
}

Minjie Wang's avatar
Minjie Wang committed
18
19
20
21
22
23
24
25
26
27
28
29
30
31
// pack libraries for later use
def pack_lib(name, libs) {
  echo "Packing ${libs} into ${name}"
  stash includes: libs, name: name
}

// unpack libraries saved before
def unpack_lib(name, libs) {
  unstash name
  echo "Unpacked ${libs} from ${name}"
}

def build_dgl_linux(dev) {
  init_git()
32
  sh "bash tests/scripts/build_dgl.sh ${dev}"
Minjie Wang's avatar
Minjie Wang committed
33
  pack_lib("dgl-${dev}-linux", dgl_linux_libs)
Lingfan Yu's avatar
Lingfan Yu committed
34
35
}

Minjie Wang's avatar
Minjie Wang committed
36
def build_dgl_win64(dev) {
37
38
  /* Assuming that Windows slaves are already configured with MSBuild VS2017,
   * CMake and Python/pip/setuptools etc. */
Minjie Wang's avatar
Minjie Wang committed
39
  init_git_win64()
40
  bat "CALL tests\\scripts\\build_dgl.bat"
Minjie Wang's avatar
Minjie Wang committed
41
  pack_lib("dgl-${dev}-win64", dgl_win64_libs)
42
43
}

Minjie Wang's avatar
Minjie Wang committed
44
45
def cpp_unit_test_linux() {
  init_git()
Minjie Wang's avatar
Minjie Wang committed
46
  unpack_lib("dgl-cpu-linux", dgl_linux_libs)
VoVAllen's avatar
VoVAllen committed
47
48
49
  sh "bash tests/scripts/task_cpp_unit_test.sh"
}

Minjie Wang's avatar
Minjie Wang committed
50
51
def cpp_unit_test_win64() {
  init_git_win64()
Minjie Wang's avatar
Minjie Wang committed
52
  unpack_lib("dgl-cpu-win64", dgl_win64_libs)
VoVAllen's avatar
VoVAllen committed
53
54
55
  bat "CALL tests\\scripts\\task_cpp_unit_test.bat"
}

Minjie Wang's avatar
Minjie Wang committed
56
def unit_test_linux(backend, dev) {
Minjie Wang's avatar
Minjie Wang committed
57
  init_git()
Minjie Wang's avatar
Minjie Wang committed
58
  unpack_lib("dgl-${dev}-linux", dgl_linux_libs)
VoVAllen's avatar
VoVAllen committed
59
  timeout(time: 10, unit: 'MINUTES') {
60
    sh "bash tests/scripts/task_unit_test.sh ${backend} ${dev}"
Minjie Wang's avatar
Minjie Wang committed
61
  }
Lingfan Yu's avatar
Lingfan Yu committed
62
63
}

64
def unit_test_win64(backend, dev) {
Minjie Wang's avatar
Minjie Wang committed
65
  init_git_win64()
Minjie Wang's avatar
Minjie Wang committed
66
  unpack_lib("dgl-${dev}-win64", dgl_win64_libs)
Minjie Wang's avatar
Minjie Wang committed
67
  timeout(time: 2, unit: 'MINUTES') {
68
    bat "CALL tests\\scripts\\task_unit_test.bat ${backend}"
Minjie Wang's avatar
Minjie Wang committed
69
  }
Da Zheng's avatar
Da Zheng committed
70
71
}

72
73
74
75
76
77
78
79
def kg_test_linux(backend, dev) {
  init_git()
  unpack_lib("dgl-${dev}-linux", dgl_linux_libs)
  timeout(time: 20, unit: 'MINUTES') {
    sh "bash tests/scripts/task_kg_test.sh ${backend} ${dev}"
  }
}

Minjie Wang's avatar
Minjie Wang committed
80
def example_test_linux(backend, dev) {
Minjie Wang's avatar
Minjie Wang committed
81
  init_git()
Minjie Wang's avatar
Minjie Wang committed
82
  unpack_lib("dgl-${dev}-linux", dgl_linux_libs)
Minjie Wang's avatar
Minjie Wang committed
83
84
  timeout(time: 20, unit: 'MINUTES') {
    sh "bash tests/scripts/task_example_test.sh ${dev}"
Minjie Wang's avatar
Minjie Wang committed
85
86
87
  }
}

88
def example_test_win64(backend, dev) {
Minjie Wang's avatar
Minjie Wang committed
89
  init_git_win64()
Minjie Wang's avatar
Minjie Wang committed
90
  unpack_lib("dgl-${dev}-win64", dgl_win64_libs)
Minjie Wang's avatar
Minjie Wang committed
91
92
  timeout(time: 20, unit: 'MINUTES') {
    bat "CALL tests\\scripts\\task_example_test.bat ${dev}"
93
94
95
  }
}

Minjie Wang's avatar
Minjie Wang committed
96
def tutorial_test_linux(backend) {
Minjie Wang's avatar
Minjie Wang committed
97
  init_git()
Minjie Wang's avatar
Minjie Wang committed
98
  unpack_lib("dgl-cpu-linux", dgl_linux_libs)
Minjie Wang's avatar
Minjie Wang committed
99
100
  timeout(time: 20, unit: 'MINUTES') {
    sh "bash tests/scripts/task_${backend}_tutorial_test.sh"
Minjie Wang's avatar
Minjie Wang committed
101
  }
Lingfan Yu's avatar
Lingfan Yu committed
102
103
}

Minjie Wang's avatar
Minjie Wang committed
104
pipeline {
105
  agent any
Minjie Wang's avatar
Minjie Wang committed
106
107
  stages {
    stage("Lint Check") {
Minjie Wang's avatar
Minjie Wang committed
108
      agent { docker { image "dgllib/dgl-ci-lint" } }
Minjie Wang's avatar
Minjie Wang committed
109
      steps {
Minjie Wang's avatar
Minjie Wang committed
110
        init_git()
111
        sh "bash tests/scripts/task_lint.sh"
Minjie Wang's avatar
Minjie Wang committed
112
      }
113
114
115
116
117
      post {
        always {
          cleanWs disableDeferredWipeout: true, deleteDirs: true
        }
      }
Minjie Wang's avatar
Minjie Wang committed
118
119
120
121
    }
    stage("Build") {
      parallel {
        stage("CPU Build") {
VoVAllen's avatar
VoVAllen committed
122
          agent { docker { image "dgllib/dgl-ci-cpu:conda" } }
Minjie Wang's avatar
Minjie Wang committed
123
          steps {
Minjie Wang's avatar
Minjie Wang committed
124
            build_dgl_linux("cpu")
Minjie Wang's avatar
Minjie Wang committed
125
          }
126
127
128
129
130
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
Minjie Wang's avatar
Minjie Wang committed
131
132
133
134
        }
        stage("GPU Build") {
          agent {
            docker {
VoVAllen's avatar
VoVAllen committed
135
              image "dgllib/dgl-ci-gpu:conda"
Minjie Wang's avatar
Minjie Wang committed
136
137
138
139
              args "--runtime nvidia"
            }
          }
          steps {
Minjie Wang's avatar
Minjie Wang committed
140
141
            sh "nvidia-smi"
            build_dgl_linux("gpu")
Minjie Wang's avatar
Minjie Wang committed
142
          }
143
144
145
146
147
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
Minjie Wang's avatar
Minjie Wang committed
148
        }
Minjie Wang's avatar
Minjie Wang committed
149
150
151
152
        stage("CPU Build (Win64)") {
          // Windows build machines are manually added to Jenkins master with
          // "windows" label as permanent agents.
          agent { label "windows" }
153
          steps {
Minjie Wang's avatar
Minjie Wang committed
154
            build_dgl_win64("cpu")
155
          }
156
157
158
159
160
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
161
        }
Minjie Wang's avatar
Minjie Wang committed
162
        // Currently we don't have Windows GPU build machines
Minjie Wang's avatar
Minjie Wang committed
163
164
165
166
      }
    }
    stage("Test") {
      parallel {
Minjie Wang's avatar
Minjie Wang committed
167
        stage("C++ CPU") {
VoVAllen's avatar
VoVAllen committed
168
          agent { docker { image "dgllib/dgl-ci-cpu:conda" } }
Minjie Wang's avatar
Minjie Wang committed
169
170
          steps {
            cpp_unit_test_linux()
VoVAllen's avatar
VoVAllen committed
171
          }
172
173
174
175
176
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
VoVAllen's avatar
VoVAllen committed
177
        }
Minjie Wang's avatar
Minjie Wang committed
178
179
180
181
        stage("C++ CPU (Win64)") {
          agent { label "windows" }
          steps {
            cpp_unit_test_win64()
182
          }
183
184
185
186
187
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
Minjie Wang's avatar
Minjie Wang committed
188
        }
VoVAllen's avatar
VoVAllen committed
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
        stage("Tensorflow CPU") {
          agent { docker { image "dgllib/dgl-ci-cpu:conda" } }
          stages {
            stage("Unit test") {
              steps {
                unit_test_linux("tensorflow", "cpu")
              }
            }
          }
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
        }
        stage("Tensorflow GPU") {
          agent { 
            docker { 
              image "dgllib/dgl-ci-gpu:conda" 
              args "--runtime nvidia"
            }
          }
          stages {
            stage("Unit test") {
              steps {
                unit_test_linux("tensorflow", "gpu")
              }
            }
          }
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
        }
Minjie Wang's avatar
Minjie Wang committed
224
        stage("Torch CPU") {
VoVAllen's avatar
VoVAllen committed
225
          agent { docker { image "dgllib/dgl-ci-cpu:conda" } }
Minjie Wang's avatar
Minjie Wang committed
226
          stages {
Minjie Wang's avatar
Minjie Wang committed
227
228
            stage("Unit test") {
              steps {
Minjie Wang's avatar
Minjie Wang committed
229
                unit_test_linux("pytorch", "cpu")
Minjie Wang's avatar
Minjie Wang committed
230
              }
Lingfan Yu's avatar
Lingfan Yu committed
231
            }
Minjie Wang's avatar
Minjie Wang committed
232
233
            stage("Example test") {
              steps {
Minjie Wang's avatar
Minjie Wang committed
234
                example_test_linux("pytorch", "cpu")
Minjie Wang's avatar
Minjie Wang committed
235
236
237
238
              }
            }
            stage("Tutorial test") {
              steps {
Minjie Wang's avatar
Minjie Wang committed
239
                tutorial_test_linux("pytorch")
Minjie Wang's avatar
Minjie Wang committed
240
              }
241
242
            }
          }
243
244
245
246
247
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
248
        }
Minjie Wang's avatar
Minjie Wang committed
249
        stage("Torch CPU (Win64)") {
250
251
          agent { label "windows" }
          stages {
Minjie Wang's avatar
Minjie Wang committed
252
253
254
255
            stage("Unit test") {
              steps {
                unit_test_win64("pytorch", "cpu")
              }
256
            }
Minjie Wang's avatar
Minjie Wang committed
257
258
259
260
            stage("Example test") {
              steps {
                example_test_win64("pytorch", "cpu")
              }
Lingfan Yu's avatar
Lingfan Yu committed
261
            }
Minjie Wang's avatar
Minjie Wang committed
262
          }
263
264
265
266
267
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
Lingfan Yu's avatar
Lingfan Yu committed
268
        }
Minjie Wang's avatar
Minjie Wang committed
269
        stage("Torch GPU") {
Minjie Wang's avatar
Minjie Wang committed
270
271
          agent {
            docker {
VoVAllen's avatar
VoVAllen committed
272
              image "dgllib/dgl-ci-gpu:conda"
Minjie Wang's avatar
Minjie Wang committed
273
              args "--runtime nvidia"
Minjie Wang's avatar
Minjie Wang committed
274
            }
Minjie Wang's avatar
Minjie Wang committed
275
276
          }
          stages {
Minjie Wang's avatar
Minjie Wang committed
277
278
279
            stage("Unit test") {
              steps {
                sh "nvidia-smi"
280
                unit_test_linux("pytorch", "gpu")
Minjie Wang's avatar
Minjie Wang committed
281
282
283
284
              }
            }
            stage("Example test") {
              steps {
Minjie Wang's avatar
Minjie Wang committed
285
                example_test_linux("pytorch", "gpu")
Minjie Wang's avatar
Minjie Wang committed
286
              }
Minjie Wang's avatar
Minjie Wang committed
287
288
            }
          }
289
290
291
292
293
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
Minjie Wang's avatar
Minjie Wang committed
294
        }
Minjie Wang's avatar
Minjie Wang committed
295
        stage("MXNet CPU") {
VoVAllen's avatar
VoVAllen committed
296
          agent { docker { image "dgllib/dgl-ci-cpu:conda" } }
Minjie Wang's avatar
Minjie Wang committed
297
          stages {
Minjie Wang's avatar
Minjie Wang committed
298
299
            stage("Unit test") {
              steps {
Minjie Wang's avatar
Minjie Wang committed
300
                unit_test_linux("mxnet", "cpu")
VoVAllen's avatar
VoVAllen committed
301
              }
Da Zheng's avatar
Da Zheng committed
302
            }
303
            //stage("Tutorial test") {
Minjie Wang's avatar
Minjie Wang committed
304
            //  steps {
305
            //    tutorial_test_linux("mxnet")
Minjie Wang's avatar
Minjie Wang committed
306
307
            //  }
            //}
308
309
310
311
          }
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
312
            }
313
314
          }
        }
315
316
317
        stage("MXNet GPU") {
          agent {
            docker {
VoVAllen's avatar
VoVAllen committed
318
              image "dgllib/dgl-ci-gpu:conda"
319
320
321
322
323
324
325
326
327
328
              args "--runtime nvidia"
            }
          }
          stages {
            stage("Unit test") {
              steps {
                sh "nvidia-smi"
                unit_test_linux("mxnet", "gpu")
              }
            }
329
330
331
332
          }
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
333
            }
334
335
          }
        }
Minjie Wang's avatar
Minjie Wang committed
336
      }
Minjie Wang's avatar
Minjie Wang committed
337
    }
338
339
340
    stage("App") {
      parallel {
        stage("Knowledge Graph CPU") {
VoVAllen's avatar
VoVAllen committed
341
          agent { docker { image "dgllib/dgl-ci-cpu:conda" } }
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
          stages {
            stage("Torch test") {
              steps {
                kg_test_linux("pytorch", "cpu")
              }
            }
            stage("MXNet test") {
              steps {
                kg_test_linux("mxnet", "cpu")
              }
            }
          }
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
        }
        stage("Knowledge Graph GPU") {
          agent {
            docker {
VoVAllen's avatar
VoVAllen committed
363
              image "dgllib/dgl-ci-gpu:conda"
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
              args "--runtime nvidia"
            }
          }
          stages {
            stage("Torch test") {
              steps {
                kg_test_linux("pytorch", "gpu")
              }
            }
            stage("MXNet test") {
              steps {
                kg_test_linux("mxnet", "gpu")
              }
            }
          }
          post {
            always {
              cleanWs disableDeferredWipeout: true, deleteDirs: true
            }
          }
        }
      }
    }
Minjie Wang's avatar
Minjie Wang committed
387
  }
388
389
390
391
392
393
394
  post {
    always {
      node('windows') {
        bat "rmvirtualenv ${BUILD_TAG}"
      }
    }
  }
Minjie Wang's avatar
Minjie Wang committed
395
}