Jenkinsfile 34.8 KB
Newer Older
JD's avatar
JD committed
1
2
3
4
5
6
7
8
9
10
11
12
13
def rocmnode(name) {
    return 'rocmtest && miopen && ' + name
}

def show_node_info() {
    sh """
        echo "NODE_NAME = \$NODE_NAME"
        lsb_release -sd
        uname -r
        ls /opt/ -la
    """
}

14
15
16
17
18
19
20
21
22
23
24
25
26
27
def nthreads() {
    def nproc = sh(returnStdout: true, script: 'nproc')
    echo "Number of cores: ${nproc}"
    def n = nproc.toInteger()
    if (n > 32){
        n /= 2
    }
    if (n > 64){
        n = 64
    }
    echo "Number of threads used for building: ${n}"
    return n
}

28
def runShell(String command){
29
    def responseCode = sh returnStatus: true, script: "${command} > tmp.txt"
30
31
32
33
    def output = readFile(file: "tmp.txt")
    return (output != "")
}

34
def getDockerImageName(){
35
    def img
36
    if (params.ROCMVERSION != "6.0"){
37
38
       if (params.COMPILER_VERSION == "") {
           img = "${env.CK_DOCKERHUB}:ck_ub20.04_rocm${params.ROCMVERSION}"
39
40
       }
       else{
41
42
43
44
45
46
47
          if (params.COMPILER_COMMIT == ""){
             img = "${env.CK_DOCKERHUB}:ck_ub20.04_rocm${params.ROCMVERSION}_${params.COMPILER_VERSION}"
          }
          else{
             def commit = "${params.COMPILER_COMMIT}"[0..6]
             img = "${env.CK_DOCKERHUB}:ck_ub20.04_rocm${params.ROCMVERSION}_${params.COMPILER_VERSION}_${commit}"
          }
48
       }
49
50
    }
    else{
51
52
       if (params.COMPILER_VERSION == "") {
           img = "${env.CK_DOCKERHUB_PRIVATE}:ck_ub20.04_rocm${params.ROCMVERSION}"
53
54
       }
       else{
55
56
57
58
59
60
61
          if (params.COMPILER_COMMIT == ""){
             img = "${env.CK_DOCKERHUB_PRIVATE}:ck_ub20.04_rocm${params.ROCMVERSION}_${params.COMPILER_VERSION}"
          }
          else{
             def commit = "${params.COMPILER_COMMIT}"[0..6]
             img = "${env.CK_DOCKERHUB_PRIVATE}:ck_ub20.04_rocm${params.ROCMVERSION}_${params.COMPILER_VERSION}_${commit}"
          }
62
       }
63
    }
64
65
66
    return img
}

67
def check_host() {
68
    if ("${env.CK_SCCACHE}" != "null"){
illsilin's avatar
illsilin committed
69
        def SCCACHE_SERVER="${env.CK_SCCACHE.split(':')[0]}"
70
        echo "sccache server: ${SCCACHE_SERVER}"
illsilin's avatar
illsilin committed
71
        sh '''ping -c 1 -p 6379 "${SCCACHE_SERVER}" | echo $? > tmp.txt'''
72
73
74
75
76
77
78
79
80
        def output = readFile(file: "tmp.txt")
        echo "tmp.txt contents: \$output"
        return (output != "0")
    }
    else{
        return 1
    }
}

81
82
83
84
85
86
def build_compiler(){
    def compiler
    if (params.BUILD_COMPILER == "hipcc"){
        compiler = '/opt/rocm/bin/hipcc'
    }
    else{
87
88
        if (params.COMPILER_VERSION == "amd-stg-open" || params.COMPILER_COMMIT != ""){
            compiler = "/llvm-project/build/bin/clang++"
89
90
        }
        else{
91
            compiler = "/opt/rocm/llvm/bin/clang++"
92
93
94
95
96
        }        
    }
    return compiler
}

97
98
def getDockerImage(Map conf=[:]){
    env.DOCKER_BUILDKIT=1
99
    def prefixpath = conf.get("prefixpath", "/opt/rocm")
100
    def no_cache = conf.get("no_cache", false)
101
    def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
    if(no_cache)
    {
        dockerArgs = dockerArgs + " --no-cache "
    }
    echo "Docker Args: ${dockerArgs}"
    def image = getDockerImageName()
    //Check if image exists 
    def retimage
    try 
    {
        echo "Pulling down image: ${image}"
        retimage = docker.image("${image}")
        retimage.pull()
    }
    catch(Exception ex)
    {
        error "Unable to locate image: ${image}"
    }
    return [retimage, image]
}

def buildDocker(install_prefix){
    show_node_info()
    env.DOCKER_BUILDKIT=1
    checkout scm
    def image_name = getDockerImageName()
    echo "Building Docker for ${image_name}"
129
    def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${install_prefix} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
130
131
132

    echo "Build Args: ${dockerArgs}"
    try{
133
134
135
136
137
138
139
140
141
142
143
        if(params.BUILD_DOCKER){
            //force building the new docker if that parameter is true
            echo "Building image: ${image_name}"
            retimage = docker.build("${image_name}", dockerArgs + ' .')
            retimage.push()
        }
        else{
            echo "Checking for image: ${image_name}"
            sh "docker manifest inspect --insecure ${image_name}"
            echo "Image: ${image_name} found!! Skipping building image"
        }
144
145
146
147
148
149
150
151
    }
    catch(Exception ex){
        echo "Unable to locate image: ${image_name}. Building image now"
        retimage = docker.build("${image_name}", dockerArgs + ' .')
        retimage.push()
    }
}

JD's avatar
JD committed
152
153
def cmake_build(Map conf=[:]){

154
    def compiler = build_compiler()
JD's avatar
JD committed
155
156
    def config_targets = conf.get("config_targets","check")
    def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined " + conf.get("extradebugflags", "")
157
    def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","")
JD's avatar
JD committed
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
    def prefixpath = conf.get("prefixpath","/opt/rocm")
    def setup_args = conf.get("setup_args","")

    if (prefixpath != "/usr/local"){
        setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} "
    }

    def build_type_debug = (conf.get("build_type",'release') == 'debug')

    //cmake_env can overwrite default CXX variables.
    def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","")

    def package_build = (conf.get("package_build","") == "true")

    if (package_build == true) {
        config_targets = "package"
    }

    if(conf.get("build_install","") == "true")
    {
        config_targets = 'install ' + config_targets
179
        setup_args = ' -DBUILD_DEV=On -DCMAKE_INSTALL_PREFIX=../install' + setup_args
JD's avatar
JD committed
180
181
182
    } else{
        setup_args = ' -DBUILD_DEV=On' + setup_args
    }
183
184
185
    if (params.DL_KERNELS){
        setup_args = setup_args + " -DDL_KERNELS=ON "
    }
JD's avatar
JD committed
186
187
188
189
190
191

    if(build_type_debug){
        setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args
    }else{
        setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args
    }
192
    if(env.CK_SCCACHE && params.USE_SCCACHE && check_host())
193
    {
194
        setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache " + setup_args
195
    }
JD's avatar
JD committed
196

illsilin's avatar
illsilin committed
197
198
    def pre_setup_cmd = """
            #!/bin/bash
JD's avatar
JD committed
199
200
201
202
203
204
205
            echo \$HSA_ENABLE_SDMA
            ulimit -c unlimited
            rm -rf build
            mkdir build
            rm -rf install
            mkdir install
            cd build
206
207
        """
    if(check_host() && params.USE_SCCACHE && "${env.CK_SCCACHE}" != "null") {
illsilin's avatar
illsilin committed
208
209
210
211
212
213
214
215
216
217
218
219
220
        def invocation_tag
        if ("${setup_args}".contains("gfx11")){
            invocation_tag="gfx11"
        }
        if ("${setup_args}".contains("gfx10")){
            invocation_tag="gfx10"
        }
        if ("${setup_args}".contains("gfx94")){
            invocation_tag="gfx94"
        }
        else {
            invocation_tag="gfx90"
        }
illsilin's avatar
illsilin committed
221
222
        pre_setup_cmd = pre_setup_cmd + """
            #!/bin/bash
223
224
225
226
227
228
229
230
231
            export ROCM_PATH=/opt/rocm
            export SCCACHE_ENABLED=true
            export SCCACHE_LOG_LEVEL=debug
            export SCCACHE_IDLE_TIMEOUT=14400
            export COMPILERS_HASH_DIR=/tmp/.sccache
            export SCCACHE_BIN=/usr/local/.cargo/bin/sccache
            export SCCACHE_EXTRAFILES=/tmp/.sccache/rocm_compilers_hash_file
            export SCCACHE_REDIS="redis://${env.CK_SCCACHE}"
            echo "connect = ${env.CK_SCCACHE}" >> ../script/redis-cli.conf
illsilin's avatar
illsilin committed
232
            export SCCACHE_C_CUSTOM_CACHE_BUSTER="${invocation_tag}"
illsilin's avatar
illsilin committed
233
            echo \$SCCACHE_C_CUSTOM_CACHE_BUSTER
234
235
            stunnel ../script/redis-cli.conf
            ../script/sccache_wrapper.sh --enforce_redis
JD's avatar
JD committed
236
        """
237
    }
JD's avatar
JD committed
238
    def setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args}   .. ")
Chao Liu's avatar
Chao Liu committed
239
    // reduce parallelism when compiling, clang uses too much memory
240
241
    def nt = nthreads()
    def build_cmd = conf.get("build_cmd", "${build_envs} dumb-init make  -j${nt} ${config_targets}")
JD's avatar
JD committed
242
243
244
245
246
247
248
249
250
251
252
253
254
    def execute_cmd = conf.get("execute_cmd", "")

    def cmd = conf.get("cmd", """
            ${pre_setup_cmd}
            ${setup_cmd}
            ${build_cmd}
            ${execute_cmd}
        """)

    echo cmd
    sh cmd

    // Only archive from master or develop
255
    if (package_build == true && (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "amd-master")) {
JD's avatar
JD committed
256
257
258
259
260
261
262
263
264
265
        archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true
    }
}

def buildHipClangJob(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

266
        def image = getDockerImageName() 
JD's avatar
JD committed
267
268
269
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
270
        def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
JD's avatar
JD committed
271
        if (conf.get("enforce_xnack_on", false)) {
272
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
JD's avatar
JD committed
273
        }
274
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
275
        if (params.COMPILER_VERSION == "amd-stg-open" || params.COMPILER_COMMIT != ""){
276
277
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
JD's avatar
JD committed
278
279
280
281

        def variant = env.STAGE_NAME

        def retimage
282
        (retimage, image) = getDockerImage(conf)
283
284

        gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') {
JD's avatar
JD committed
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
                timeout(time: 5, unit: 'HOURS')
                {
                    cmake_build(conf)
                }
            }
        }
        return retimage
}

def reboot(){
    build job: 'reboot-slaves', propagate: false , parameters: [string(name: 'server', value: "${env.NODE_NAME}"),]
}

def buildHipClangJobAndReboot(Map conf=[:]){
    try{
        buildHipClangJob(conf)
    }
    catch(e){
        echo "throwing error exception for the stage"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

315
316
317
318
319
320
def runCKProfiler(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

321
        def image = getDockerImageName()
322
323
324
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
325
        def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
326
        if (conf.get("enforce_xnack_on", false)) {
327
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
328
        }
329
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
330
        if (params.COMPILER_VERSION == "amd-stg-open" || params.COMPILER_COMMIT != ""){
331
332
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
333
334
335

        def variant = env.STAGE_NAME
        def retimage
336
337

        gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') {
338
            try {
339
                (retimage, image) = getDockerImage(conf)
340
341
                withDockerContainer(image: image, args: dockerOpts) {
                    timeout(time: 5, unit: 'MINUTES'){
342
343
                        sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo | tee clinfo.log'
                        if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
344
                            throw new Exception ("GPU not found")
345
346
347
348
                        }
                        else{
                            echo "GPU is OK"
                        }
349
350
351
                    }
                }
            }
352
353
354
355
356
357
358
359
            catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
                echo "The job was cancelled or aborted"
                throw e
            }
            catch(Exception ex) {
                retimage = docker.build("${image}", dockerArgs + " --no-cache .")
                withDockerContainer(image: image, args: dockerOpts) {
                    timeout(time: 5, unit: 'MINUTES'){
360
361
                        sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo | tee clinfo.log'
                        if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
362
                            throw new Exception ("GPU not found")
363
364
365
366
                        }
                        else{
                            echo "GPU is OK"
                        }
367
                    }
368
369
                }
            }
370
371

            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
372
                timeout(time: 24, unit: 'HOURS')
373
                {
374
375
376
377
378
379
380
381
382
                    sh """
                        rm -rf build
                        mkdir build
                    """
                    dir("build"){
                        unstash 'ckProfiler.tar.gz'
                        sh 'tar -xvf ckProfiler.tar.gz'
                    }

383
					dir("script"){
384
                        if (params.RUN_FULL_QA){
385
386
387
388
389
390
391
392
393
394
395
396
397
                            sh "./run_full_performance_tests.sh 1 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
                            archiveArtifacts "perf_batched_gemm.log"
                            archiveArtifacts "perf_grouped_gemm.log"
                            archiveArtifacts "perf_conv_fwd.log"
                            archiveArtifacts "perf_conv_bwd_data.log"
                            archiveArtifacts "perf_gemm_bilinear.log"
                            archiveArtifacts "perf_reduction.log"
                            archiveArtifacts "perf_splitK_gemm_verify.log"
                            archiveArtifacts "perf_splitK_gemm.log"
                            archiveArtifacts "perf_onnx_gemm.log"
398
                           // stash perf files to master
399
400
401
402
403
404
405
406
407
408
409
                            stash name: "perf_gemm.log"
                            stash name: "perf_resnet50_N256.log"
                            stash name: "perf_resnet50_N4.log"
                            stash name: "perf_batched_gemm.log"
                            stash name: "perf_grouped_gemm.log"
                            stash name: "perf_conv_fwd.log"
                            stash name: "perf_conv_bwd_data.log"
                            stash name: "perf_gemm_bilinear.log"
                            stash name: "perf_reduction.log"
                            stash name: "perf_splitK_gemm.log"
                            stash name: "perf_onnx_gemm.log"
410
                            //we will process results on the master node
411
412
                        }
                        else{
413
414
415
416
                            sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
417
                            // stash perf files to master
418
419
420
                            stash name: "perf_gemm.log"
                            stash name: "perf_resnet50_N256.log"
                            stash name: "perf_resnet50_N4.log"
421
                            //we will process the results on the master node
422
                        }
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
					}
                }
            }
        }
        return retimage
}

def runPerfTest(Map conf=[:]){
    try{
        runCKProfiler(conf)
    }
    catch(e){
        echo "throwing error exception in performance tests"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

446
447
448
449
450
451
452
453
454
455
456
457
458
459
def Build_CK(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

        def image = getDockerImageName() 
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
        def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
        if (conf.get("enforce_xnack_on", false)) {
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
        }
460
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
461
        if (params.COMPILER_VERSION == "amd-stg-open" || params.COMPILER_COMMIT != ""){
462
463
464
465
466
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }

        def variant = env.STAGE_NAME
        def retimage
Illia Silin's avatar
Illia Silin committed
467
        def navi_node = 0
468
469
470
471
472
473
474
475
476
477
478
479
480

        gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') {
            try {
                (retimage, image) = getDockerImage(conf)
                withDockerContainer(image: image, args: dockerOpts) {
                    timeout(time: 5, unit: 'MINUTES'){
                        sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo | tee clinfo.log'
                        if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
                            throw new Exception ("GPU not found")
                        }
                        else{
                            echo "GPU is OK"
                        }
481
                        if ( runShell('grep -n "gfx1030" clinfo.log') || runShell('grep -n "gfx1101" clinfo.log') ){
Illia Silin's avatar
Illia Silin committed
482
483
                            navi_node = 1
                        }
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
                    }
                }
            }
            catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
                echo "The job was cancelled or aborted"
                throw e
            }
            catch(Exception ex) {
                retimage = docker.build("${image}", dockerArgs + " --no-cache .")
                withDockerContainer(image: image, args: dockerOpts) {
                    timeout(time: 5, unit: 'MINUTES'){
                        sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo |tee clinfo.log'
                        if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
                            throw new Exception ("GPU not found")
                        }
                        else{
                            echo "GPU is OK"
                        }
502
                        if ( runShell('grep -n "gfx1030" clinfo.log') || runShell('grep -n "gfx1101" clinfo.log') ){
Illia Silin's avatar
Illia Silin committed
503
504
                            navi_node = 1
                        }
505
506
507
508
509
510
511
512
                    }
                }
            }
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
                timeout(time: 24, unit: 'HOURS')
                {
                    cmake_build(conf)
                    dir("build"){
513
                        //run tests and examples
514
                        sh 'make -j check'
Illia Silin's avatar
Illia Silin committed
515
                        if (navi_node == 0 ){
516
517
                            //we only need the ckProfiler to run the performance tests, so we pack and stash it
                            //do not stash profiler on Navi nodes
Illia Silin's avatar
Illia Silin committed
518
519
520
                           sh 'tar -zcvf ckProfiler.tar.gz bin/ckProfiler'
                           stash "ckProfiler.tar.gz"
                        }
521
522
523
524
525
                        if (params.RUN_FULL_QA){
                           // build deb packages
                           sh 'make -j package'
                           archiveArtifacts artifacts: 'composablekernel-ckprofiler_*.deb'
                           archiveArtifacts artifacts: 'composablekernel-tests_*.deb'
Illia Silin's avatar
Illia Silin committed
526
527
                           sh 'mv composablekernel-ckprofiler_*.deb ckprofiler_0.2.0_amd64.deb'
                           stash "ckprofiler_0.2.0_amd64.deb"
528
                        }
529
                    }
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
                    if (params.hipTensor_test && navi_node == 0 ){
                        //build and test hipTensor
                        sh """#!/bin/bash
                            rm -rf "${params.hipTensor_branch}".zip
                            rm -rf hipTensor-"${params.hipTensor_branch}"
                            wget https://github.com/ROCmSoftwarePlatform/hipTensor/archive/refs/heads/"${params.hipTensor_branch}".zip
                            unzip -o "${params.hipTensor_branch}".zip
                        """
                        dir("hipTensor-${params.hipTensor_branch}"){
                            sh """#!/bin/bash
                                mkdir -p build
                                ls -ltr
                                CC=hipcc CXX=hipcc cmake -Bbuild . -D CMAKE_PREFIX_PATH="/opt/rocm;${env.WORKSPACE}/install"
                                cmake --build build -- -j
                            """
                        }
                        dir("hipTensor-${params.hipTensor_branch}/build"){
                            sh 'ctest'
                        }
                    }
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
                }
            }
        }
        return retimage
}

def Build_CK_and_Reboot(Map conf=[:]){
    try{
        Build_CK(conf)
    }
    catch(e){
        echo "throwing error exception while building CK"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

572
573
574
def process_results(Map conf=[:]){
    env.HSA_ENABLE_SDMA=0
    checkout scm
575
    def image = getDockerImageName() 
576
577
578
579
580
    def prefixpath = "/opt/rocm"

    // Jenkins is complaining about the render group 
    def dockerOpts="--cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
    if (conf.get("enforce_xnack_on", false)) {
581
        dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
582
583
584
585
586
587
588
    }

    def variant = env.STAGE_NAME
    def retimage

    gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') {
        try {
589
            (retimage, image) = getDockerImage(conf)
590
591
592
593
594
595
596
597
598
599
600
601
602
        }
        catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
            echo "The job was cancelled or aborted"
            throw e
        }
    }

    withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
        timeout(time: 1, unit: 'HOURS'){
            try{
                dir("script"){
                    if (params.RUN_FULL_QA){
                        // unstash perf files to master
603
604
605
606
607
608
609
610
611
612
613
614
                        unstash "perf_gemm.log"
                        unstash "perf_resnet50_N256.log"
                        unstash "perf_resnet50_N4.log"
                        unstash "perf_batched_gemm.log"
                        unstash "perf_grouped_gemm.log"
                        unstash "perf_conv_fwd.log"
                        unstash "perf_conv_bwd_data.log"
                        unstash "perf_gemm_bilinear.log"
                        unstash "perf_reduction.log"
                        unstash "perf_splitK_gemm.log"
                        unstash "perf_onnx_gemm.log"
                        sh "./process_qa_data.sh"
Illia Silin's avatar
Illia Silin committed
615
616
                        unstash "ckprofiler_0.2.0_amd64.deb"
                        sh "sshpass -p ${env.ck_deb_pw} scp -o StrictHostKeyChecking=no ckprofiler_0.2.0_amd64.deb ${env.ck_deb_user}@${env.ck_deb_ip}:/var/www/html/composable_kernel/"
617
618
619
                    }
                    else{
                        // unstash perf files to master
620
621
622
623
                        unstash "perf_gemm.log"
                        unstash "perf_resnet50_N256.log"
                        unstash "perf_resnet50_N4.log"
                        sh "./process_perf_data.sh"
624
625
626
627
628
629
630
631
632
633
634
635
                    }
                }
            }
            catch(e){
                echo "throwing error exception while processing performance test results"
                echo 'Exception occurred: ' + e.toString()
                throw e
            }
        }
    }
}

636
//launch develop branch daily at 23:00 UT in FULL_QA mode and at 19:00 UT with latest staging compiler version
637
638
CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;ROCMVERSION=5.7;COMPILER_VERSION=
                                              0 21 * * * % ROCMVERSION=5.7;COMPILER_VERSION=;COMPILER_COMMIT=
639
                                              0 19 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-stg-open;COMPILER_COMMIT=;USE_SCCACHE=false''' : ""
640

JD's avatar
JD committed
641
642
pipeline {
    agent none
643
644
645
    triggers {
        parameterizedCron(CRON_SETTINGS)
    }
JD's avatar
JD committed
646
647
648
    options {
        parallelsAlwaysFailFast()
    }
649
    parameters {
650
651
        booleanParam(
            name: "BUILD_DOCKER",
652
            defaultValue: false,
653
654
655
            description: "Force building docker image (default: false), set to true if docker image needs to be updated.")
        string(
            name: 'ROCMVERSION', 
656
657
            defaultValue: '5.7', 
            description: 'Specify which ROCM version to use: 5.7 (default).')
658
659
        string(
            name: 'COMPILER_VERSION', 
660
661
            defaultValue: '', 
            description: 'Specify which version of compiler to use: release, amd-stg-open, or leave blank (default).')
662
663
        string(
            name: 'COMPILER_COMMIT', 
664
665
            defaultValue: '', 
            description: 'Specify which commit of compiler branch to use: leave blank to use the latest commit, or use 5541927df00eabd6a110180170eca7785d436ee3 (default) commit of amd-stg-open branch.')
666
667
668
669
        string(
            name: 'BUILD_COMPILER', 
            defaultValue: 'hipcc', 
            description: 'Specify whether to build CK with hipcc (default) or with clang.')
670
671
672
673
        booleanParam(
            name: "RUN_FULL_QA",
            defaultValue: false,
            description: "Select whether to run small set of performance tests (default) or full QA")
674
675
676
677
        booleanParam(
            name: "DL_KERNELS",
            defaultValue: false,
            description: "Select whether to build DL kernels (default: OFF)")
678
679
680
681
682
683
684
685
        booleanParam(
            name: "hipTensor_test",
            defaultValue: true,
            description: "Use the CK build to verify hipTensor build and tests (default: ON)")
        string(
            name: 'hipTensor_branch',
            defaultValue: 'mainline',
            description: 'Specify which branch of hipTensor to use (default: mainline)')
686
687
688
689
        booleanParam(
            name: "USE_SCCACHE",
            defaultValue: true,
            description: "Use the sccache for building CK (default: ON)")
690
691
692
693
694
695
696
697
    }
    environment{
        dbuser = "${dbuser}"
        dbpassword = "${dbpassword}"
        dbsship = "${dbsship}"
        dbsshport = "${dbsshport}"
        dbsshuser = "${dbsshuser}"
        dbsshpassword = "${dbsshpassword}"
698
        status_wrapper_creds = "${status_wrapper_creds}"
699
700
        gerrit_cred="${gerrit_cred}"
        DOCKER_BUILDKIT = "1"
701
    }
JD's avatar
JD committed
702
    stages{
703
704
705
706
707
708
        stage("Build Docker"){
            parallel{
                stage('Docker /opt/rocm'){
                    agent{ label rocmnode("nogpu") }
                    steps{
                        buildDocker('/opt/rocm')
709
                        cleanWs()
710
711
712
713
                    }
                }
            }
        }
JD's avatar
JD committed
714
715
716
717
718
        stage("Static checks") {
            parallel{
                stage('Clang Format') {
                    agent{ label rocmnode("nogpu") }
                    environment{
Illia Silin's avatar
Illia Silin committed
719
720
721
                        execute_cmd = "find .. -not -path \'*.git*\' -iname \'*.h\' \
                                -o -not -path \'*.git*\' -iname \'*.hpp\' \
                                -o -not -path \'*.git*\' -iname \'*.cpp\' \
JD's avatar
JD committed
722
723
724
725
726
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
727
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\'"
JD's avatar
JD committed
728
729
730
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
731
                        cleanWs()
JD's avatar
JD committed
732
733
734
735
                    }
                }
            }
        }
736
737
    
		stage("Build CK and run Tests")
738
739
740
        {
            parallel
            {
741
742
743
744
745
746
747
748
                stage("Build CK and run Tests on MI100/MI200/MI300")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_FULL_QA.toBoolean() }
                    }
                    agent{ label rocmnode("gfx908 || gfx90a") }
                    environment{
749
750
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx908;gfx90a;gfx940;gfx941;gfx942" """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -DGPU_TARGETS="gfx908;gfx90a;gfx940;gfx941;gfx942" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """ 
751
752
753
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
754
                        cleanWs()
755
756
                    }
                }
Illia Silin's avatar
Illia Silin committed
757
                stage("Build CK and run Tests on MI100/MI200")
758
                {
759
760
761
762
                    when {
                        beforeAgent true
                        expression { !params.RUN_FULL_QA.toBoolean() }
                    }
763
                    agent{ label rocmnode("gfx908 || gfx90a") }
764
                    environment{
Illia Silin's avatar
Illia Silin committed
765
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx908;gfx90a" """
766
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -DGPU_TARGETS="gfx908;gfx90a" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """ 
Illia Silin's avatar
Illia Silin committed
767
768
769
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
770
                        cleanWs()
Illia Silin's avatar
Illia Silin committed
771
772
                    }
                }
773
                stage("Build CK and run Tests on Navi21")
Illia Silin's avatar
Illia Silin committed
774
775
776
777
778
779
780
                {
                    when {
                        beforeAgent true
                        expression { !params.RUN_FULL_QA.toBoolean() }
                    }
                    agent{ label rocmnode("navi21") }
                    environment{
781
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1030" -DDL_KERNELS=ON """ 
782
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -DGPU_TARGETS="gfx1030" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """
783
                    }
784
785
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
786
                        cleanWs()
787
788
789
790
791
792
793
794
795
796
                    }
                }
                stage("Build CK and run Tests on Navi32")
                {
                    when {
                        beforeAgent true
                        expression { !params.RUN_FULL_QA.toBoolean() }
                    }
                    agent{ label rocmnode("navi32") }
                    environment{
797
798
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1101" -DDL_KERNELS=ON """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -DGPU_TARGETS="gfx1101" -DDL_KERNELS=ON -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """
799
                    }
800
                    steps{
801
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
802
                        cleanWs()
JD's avatar
JD committed
803
804
                    }
                }
805
806
            }
        }
807

808
809
810
811
        stage("Performance Tests")
        {
            parallel
            {
Illia Silin's avatar
Illia Silin committed
812
                stage("Run ckProfiler: gfx90*")
813
                {
814
815
                    when {
                        beforeAgent true
816
                        expression { !params.RUN_FULL_QA.toBoolean() }
817
                    }
818
                    options { retry(2) }
819
                    agent{ label rocmnode("gfx908 || gfx90a")}
820
                    environment{
Illia Silin's avatar
Illia Silin committed
821
                        setup_args = """ -DGPU_TARGETS="gfx908;gfx90a" -DBUILD_DEV=On """
822
                   }
823
                    steps{
824
                        runPerfTest(setup_args:setup_args, config_targets: "ckProfiler", no_reboot:true, build_type: 'Release')
825
                        cleanWs()
826
827
828
829
                    }
                }
                stage("Run ckProfiler: gfx90a")
                {
830
831
                    when {
                        beforeAgent true
832
                        expression { params.RUN_FULL_QA.toBoolean() }
833
                    }
834
                    options { retry(2) }
835
836
                    agent{ label rocmnode("gfx90a")}
                    environment{
Illia Silin's avatar
Illia Silin committed
837
                        setup_args = """ -DGPU_TARGETS="gfx90a" -DBUILD_DEV=On """
838
                    }
839
                    steps{
840
                        runPerfTest(setup_args:setup_args, config_targets: "ckProfiler", no_reboot:true, build_type: 'Release')
841
                        cleanWs()
842
843
844
845
                    }
                }
            }
        }
846
847
848
849
        stage("Process Performance Test Results")
        {
            parallel
            {
850
                stage("Process results"){
851
852
                    agent { label 'mici' }
                    steps{
853
                        process_results()
854
                        cleanWs()
855
856
857
858
                    }
                }
            }
        }
JD's avatar
JD committed
859
    }
860
}