Jenkinsfile 34.9 KB
Newer Older
JD's avatar
JD committed
1
2
3
4
5
6
7
8
9
10
11
12
13
def rocmnode(name) {
    return 'rocmtest && miopen && ' + name
}

def show_node_info() {
    sh """
        echo "NODE_NAME = \$NODE_NAME"
        lsb_release -sd
        uname -r
        ls /opt/ -la
    """
}

14
15
16
17
18
19
20
21
22
23
24
25
26
27
def nthreads() {
    def nproc = sh(returnStdout: true, script: 'nproc')
    echo "Number of cores: ${nproc}"
    def n = nproc.toInteger()
    if (n > 32){
        n /= 2
    }
    if (n > 64){
        n = 64
    }
    echo "Number of threads used for building: ${n}"
    return n
}

28
def runShell(String command){
29
    def responseCode = sh returnStatus: true, script: "${command} > tmp.txt"
30
31
32
33
    def output = readFile(file: "tmp.txt")
    return (output != "")
}

34
def getDockerImageName(){
35
    def img
36
    if (params.ROCMVERSION != "6.0"){
37
38
       if (params.COMPILER_VERSION == "") {
           img = "${env.CK_DOCKERHUB}:ck_ub20.04_rocm${params.ROCMVERSION}"
39
40
       }
       else{
41
42
43
44
45
46
47
          if (params.COMPILER_COMMIT == ""){
             img = "${env.CK_DOCKERHUB}:ck_ub20.04_rocm${params.ROCMVERSION}_${params.COMPILER_VERSION}"
          }
          else{
             def commit = "${params.COMPILER_COMMIT}"[0..6]
             img = "${env.CK_DOCKERHUB}:ck_ub20.04_rocm${params.ROCMVERSION}_${params.COMPILER_VERSION}_${commit}"
          }
48
       }
49
50
    }
    else{
51
52
       if (params.COMPILER_VERSION == "") {
           img = "${env.CK_DOCKERHUB_PRIVATE}:ck_ub20.04_rocm${params.ROCMVERSION}"
53
54
       }
       else{
55
56
57
58
59
60
61
          if (params.COMPILER_COMMIT == ""){
             img = "${env.CK_DOCKERHUB_PRIVATE}:ck_ub20.04_rocm${params.ROCMVERSION}_${params.COMPILER_VERSION}"
          }
          else{
             def commit = "${params.COMPILER_COMMIT}"[0..6]
             img = "${env.CK_DOCKERHUB_PRIVATE}:ck_ub20.04_rocm${params.ROCMVERSION}_${params.COMPILER_VERSION}_${commit}"
          }
62
       }
63
    }
64
65
66
    return img
}

67
def check_host() {
68
    if ("${env.CK_SCCACHE}" != "null"){
illsilin's avatar
illsilin committed
69
        def SCCACHE_SERVER="${env.CK_SCCACHE.split(':')[0]}"
70
        echo "sccache server: ${SCCACHE_SERVER}"
illsilin's avatar
illsilin committed
71
        sh '''ping -c 1 -p 6379 "${SCCACHE_SERVER}" | echo $? > tmp.txt'''
72
73
74
75
76
77
78
79
80
        def output = readFile(file: "tmp.txt")
        echo "tmp.txt contents: \$output"
        return (output != "0")
    }
    else{
        return 1
    }
}

81
82
83
84
85
86
def build_compiler(){
    def compiler
    if (params.BUILD_COMPILER == "hipcc"){
        compiler = '/opt/rocm/bin/hipcc'
    }
    else{
87
88
        if (params.COMPILER_VERSION == "amd-stg-open" || params.COMPILER_COMMIT != ""){
            compiler = "/llvm-project/build/bin/clang++"
89
90
        }
        else{
91
            compiler = "/opt/rocm/llvm/bin/clang++"
92
93
94
95
96
        }        
    }
    return compiler
}

97
98
def getDockerImage(Map conf=[:]){
    env.DOCKER_BUILDKIT=1
99
    def prefixpath = conf.get("prefixpath", "/opt/rocm")
100
    def no_cache = conf.get("no_cache", false)
101
    def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
    if(no_cache)
    {
        dockerArgs = dockerArgs + " --no-cache "
    }
    echo "Docker Args: ${dockerArgs}"
    def image = getDockerImageName()
    //Check if image exists 
    def retimage
    try 
    {
        echo "Pulling down image: ${image}"
        retimage = docker.image("${image}")
        retimage.pull()
    }
    catch(Exception ex)
    {
        error "Unable to locate image: ${image}"
    }
    return [retimage, image]
}

def buildDocker(install_prefix){
    show_node_info()
    env.DOCKER_BUILDKIT=1
    checkout scm
    def image_name = getDockerImageName()
    echo "Building Docker for ${image_name}"
129
    def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${install_prefix} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
130
131
132

    echo "Build Args: ${dockerArgs}"
    try{
133
134
135
136
137
138
139
140
141
142
143
        if(params.BUILD_DOCKER){
            //force building the new docker if that parameter is true
            echo "Building image: ${image_name}"
            retimage = docker.build("${image_name}", dockerArgs + ' .')
            retimage.push()
        }
        else{
            echo "Checking for image: ${image_name}"
            sh "docker manifest inspect --insecure ${image_name}"
            echo "Image: ${image_name} found!! Skipping building image"
        }
144
145
146
147
148
149
150
151
    }
    catch(Exception ex){
        echo "Unable to locate image: ${image_name}. Building image now"
        retimage = docker.build("${image_name}", dockerArgs + ' .')
        retimage.push()
    }
}

JD's avatar
JD committed
152
153
def cmake_build(Map conf=[:]){

154
    def compiler = build_compiler()
JD's avatar
JD committed
155
156
    def config_targets = conf.get("config_targets","check")
    def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined " + conf.get("extradebugflags", "")
157
    def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","")
JD's avatar
JD committed
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
    def prefixpath = conf.get("prefixpath","/opt/rocm")
    def setup_args = conf.get("setup_args","")

    if (prefixpath != "/usr/local"){
        setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} "
    }

    def build_type_debug = (conf.get("build_type",'release') == 'debug')

    //cmake_env can overwrite default CXX variables.
    def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","")

    def package_build = (conf.get("package_build","") == "true")

    if (package_build == true) {
        config_targets = "package"
    }

    if(conf.get("build_install","") == "true")
    {
        config_targets = 'install ' + config_targets
179
        setup_args = ' -DBUILD_DEV=On -DCMAKE_INSTALL_PREFIX=../install' + setup_args
JD's avatar
JD committed
180
181
182
    } else{
        setup_args = ' -DBUILD_DEV=On' + setup_args
    }
183
184
185
    if (params.DL_KERNELS){
        setup_args = setup_args + " -DDL_KERNELS=ON "
    }
JD's avatar
JD committed
186
187
188
189
190
191

    if(build_type_debug){
        setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args
    }else{
        setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args
    }
192
    if(env.CK_SCCACHE && params.USE_SCCACHE && check_host())
193
    {
194
        setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache " + setup_args
195
    }
JD's avatar
JD committed
196

illsilin's avatar
illsilin committed
197
    def pre_setup_cmd = """#!/bin/bash
JD's avatar
JD committed
198
199
200
201
202
203
204
            echo \$HSA_ENABLE_SDMA
            ulimit -c unlimited
            rm -rf build
            mkdir build
            rm -rf install
            mkdir install
            cd build
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
        """
    if(check_host() && params.USE_SCCACHE && "${env.CK_SCCACHE}" != "null") {
        pre_setup_cmd = pre_setup_cmd + """#!/bin/bash
            export ROCM_PATH=/opt/rocm
            export SCCACHE_ENABLED=true
            export SCCACHE_LOG_LEVEL=debug
            export SCCACHE_IDLE_TIMEOUT=14400
            export COMPILERS_HASH_DIR=/tmp/.sccache
            export SCCACHE_BIN=/usr/local/.cargo/bin/sccache
            export SCCACHE_EXTRAFILES=/tmp/.sccache/rocm_compilers_hash_file
            export SCCACHE_REDIS="redis://${env.CK_SCCACHE}"
            echo "connect = ${env.CK_SCCACHE}" >> ../script/redis-cli.conf
            echo "setup_args: ${setup_args}"
            if [[ "${setup_args}" =~ "gfx1100" ]]; then
                export SCCACHE_C_CUSTOM_CACHE_BUSTER=gfx11
            elif [[ "${setup_args}" =~ "gfx1030" ]]; then
                export SCCACHE_C_CUSTOM_CACHE_BUSTER=gfx10
            elif [[ "${setup_args}" =~ "gfx940" ]]; then
                export SCCACHE_C_CUSTOM_CACHE_BUSTER=gfx94
            else
                export SCCACHE_C_CUSTOM_CACHE_BUSTER=gfx90
226
            fi
227
228
229
            echo \$SCCACHE_C_CUSTOM_CACHE_BUSTER
            stunnel ../script/redis-cli.conf
            ../script/sccache_wrapper.sh --enforce_redis
JD's avatar
JD committed
230
        """
231
    }
JD's avatar
JD committed
232
    def setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args}   .. ")
Chao Liu's avatar
Chao Liu committed
233
    // reduce parallelism when compiling, clang uses too much memory
234
235
    def nt = nthreads()
    def build_cmd = conf.get("build_cmd", "${build_envs} dumb-init make  -j${nt} ${config_targets}")
JD's avatar
JD committed
236
237
238
239
240
241
242
243
244
245
246
247
248
    def execute_cmd = conf.get("execute_cmd", "")

    def cmd = conf.get("cmd", """
            ${pre_setup_cmd}
            ${setup_cmd}
            ${build_cmd}
            ${execute_cmd}
        """)

    echo cmd
    sh cmd

    // Only archive from master or develop
249
    if (package_build == true && (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "amd-master")) {
JD's avatar
JD committed
250
251
252
253
254
255
256
257
258
259
        archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true
    }
}

def buildHipClangJob(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

260
        def image = getDockerImageName() 
JD's avatar
JD committed
261
262
263
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
264
        def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
JD's avatar
JD committed
265
        if (conf.get("enforce_xnack_on", false)) {
266
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
JD's avatar
JD committed
267
        }
268
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
269
        if (params.COMPILER_VERSION == "amd-stg-open" || params.COMPILER_COMMIT != ""){
270
271
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
JD's avatar
JD committed
272
273
274
275

        def variant = env.STAGE_NAME

        def retimage
276
        (retimage, image) = getDockerImage(conf)
277
278

        gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') {
JD's avatar
JD committed
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
                timeout(time: 5, unit: 'HOURS')
                {
                    cmake_build(conf)
                }
            }
        }
        return retimage
}

def reboot(){
    build job: 'reboot-slaves', propagate: false , parameters: [string(name: 'server', value: "${env.NODE_NAME}"),]
}

def buildHipClangJobAndReboot(Map conf=[:]){
    try{
        buildHipClangJob(conf)
    }
    catch(e){
        echo "throwing error exception for the stage"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

309
310
311
312
313
314
def runCKProfiler(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

315
        def image = getDockerImageName()
316
317
318
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
319
        def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
320
        if (conf.get("enforce_xnack_on", false)) {
321
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
322
        }
323
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
324
        if (params.COMPILER_VERSION == "amd-stg-open" || params.COMPILER_COMMIT != ""){
325
326
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
327
328
329

        def variant = env.STAGE_NAME
        def retimage
330
331

        gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') {
332
            try {
333
                (retimage, image) = getDockerImage(conf)
334
335
                withDockerContainer(image: image, args: dockerOpts) {
                    timeout(time: 5, unit: 'MINUTES'){
336
337
                        sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo | tee clinfo.log'
                        if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
338
                            throw new Exception ("GPU not found")
339
340
341
342
                        }
                        else{
                            echo "GPU is OK"
                        }
343
344
345
                    }
                }
            }
346
347
348
349
350
351
352
353
            catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
                echo "The job was cancelled or aborted"
                throw e
            }
            catch(Exception ex) {
                retimage = docker.build("${image}", dockerArgs + " --no-cache .")
                withDockerContainer(image: image, args: dockerOpts) {
                    timeout(time: 5, unit: 'MINUTES'){
354
355
                        sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo | tee clinfo.log'
                        if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
356
                            throw new Exception ("GPU not found")
357
358
359
360
                        }
                        else{
                            echo "GPU is OK"
                        }
361
                    }
362
363
                }
            }
364
365

            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
366
                timeout(time: 24, unit: 'HOURS')
367
                {
368
369
370
371
372
373
374
375
376
                    sh """
                        rm -rf build
                        mkdir build
                    """
                    dir("build"){
                        unstash 'ckProfiler.tar.gz'
                        sh 'tar -xvf ckProfiler.tar.gz'
                    }

377
					dir("script"){
378
                        if (params.RUN_FULL_QA){
379
380
381
382
383
384
385
386
387
388
389
390
391
                            sh "./run_full_performance_tests.sh 1 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
                            archiveArtifacts "perf_batched_gemm.log"
                            archiveArtifacts "perf_grouped_gemm.log"
                            archiveArtifacts "perf_conv_fwd.log"
                            archiveArtifacts "perf_conv_bwd_data.log"
                            archiveArtifacts "perf_gemm_bilinear.log"
                            archiveArtifacts "perf_reduction.log"
                            archiveArtifacts "perf_splitK_gemm_verify.log"
                            archiveArtifacts "perf_splitK_gemm.log"
                            archiveArtifacts "perf_onnx_gemm.log"
392
                           // stash perf files to master
393
394
395
396
397
398
399
400
401
402
403
                            stash name: "perf_gemm.log"
                            stash name: "perf_resnet50_N256.log"
                            stash name: "perf_resnet50_N4.log"
                            stash name: "perf_batched_gemm.log"
                            stash name: "perf_grouped_gemm.log"
                            stash name: "perf_conv_fwd.log"
                            stash name: "perf_conv_bwd_data.log"
                            stash name: "perf_gemm_bilinear.log"
                            stash name: "perf_reduction.log"
                            stash name: "perf_splitK_gemm.log"
                            stash name: "perf_onnx_gemm.log"
404
                            //we will process results on the master node
405
406
                        }
                        else{
407
408
409
410
                            sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
411
                            // stash perf files to master
412
413
414
                            stash name: "perf_gemm.log"
                            stash name: "perf_resnet50_N256.log"
                            stash name: "perf_resnet50_N4.log"
415
                            //we will process the results on the master node
416
                        }
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
					}
                }
            }
        }
        return retimage
}

def runPerfTest(Map conf=[:]){
    try{
        runCKProfiler(conf)
    }
    catch(e){
        echo "throwing error exception in performance tests"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

440
441
442
443
444
445
446
447
448
449
450
451
452
453
def Build_CK(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

        def image = getDockerImageName() 
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
        def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
        if (conf.get("enforce_xnack_on", false)) {
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
        }
454
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
455
        if (params.COMPILER_VERSION == "amd-stg-open" || params.COMPILER_COMMIT != ""){
456
457
458
459
460
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }

        def variant = env.STAGE_NAME
        def retimage
Illia Silin's avatar
Illia Silin committed
461
        def navi_node = 0
462
463
464
465
466
467
468
469
470
471
472
473
474

        gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') {
            try {
                (retimage, image) = getDockerImage(conf)
                withDockerContainer(image: image, args: dockerOpts) {
                    timeout(time: 5, unit: 'MINUTES'){
                        sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo | tee clinfo.log'
                        if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
                            throw new Exception ("GPU not found")
                        }
                        else{
                            echo "GPU is OK"
                        }
475
                        if ( runShell('grep -n "gfx1030" clinfo.log') || runShell('grep -n "gfx1101" clinfo.log') ){
Illia Silin's avatar
Illia Silin committed
476
477
                            navi_node = 1
                        }
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
                    }
                }
            }
            catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
                echo "The job was cancelled or aborted"
                throw e
            }
            catch(Exception ex) {
                retimage = docker.build("${image}", dockerArgs + " --no-cache .")
                withDockerContainer(image: image, args: dockerOpts) {
                    timeout(time: 5, unit: 'MINUTES'){
                        sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo |tee clinfo.log'
                        if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
                            throw new Exception ("GPU not found")
                        }
                        else{
                            echo "GPU is OK"
                        }
496
                        if ( runShell('grep -n "gfx1030" clinfo.log') || runShell('grep -n "gfx1101" clinfo.log') ){
Illia Silin's avatar
Illia Silin committed
497
498
                            navi_node = 1
                        }
499
500
501
502
503
504
505
506
                    }
                }
            }
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
                timeout(time: 24, unit: 'HOURS')
                {
                    cmake_build(conf)
                    dir("build"){
507
                        //run tests and examples
508
                        sh 'make -j check'
Illia Silin's avatar
Illia Silin committed
509
                        if (navi_node == 0 ){
510
511
                            //we only need the ckProfiler to run the performance tests, so we pack and stash it
                            //do not stash profiler on Navi nodes
Illia Silin's avatar
Illia Silin committed
512
513
514
                           sh 'tar -zcvf ckProfiler.tar.gz bin/ckProfiler'
                           stash "ckProfiler.tar.gz"
                        }
515
516
517
518
519
                        if (params.RUN_FULL_QA){
                           // build deb packages
                           sh 'make -j package'
                           archiveArtifacts artifacts: 'composablekernel-ckprofiler_*.deb'
                           archiveArtifacts artifacts: 'composablekernel-tests_*.deb'
Illia Silin's avatar
Illia Silin committed
520
521
                           sh 'mv composablekernel-ckprofiler_*.deb ckprofiler_0.2.0_amd64.deb'
                           stash "ckprofiler_0.2.0_amd64.deb"
522
                        }
523
                    }
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
                    if (params.hipTensor_test && navi_node == 0 ){
                        //build and test hipTensor
                        sh """#!/bin/bash
                            rm -rf "${params.hipTensor_branch}".zip
                            rm -rf hipTensor-"${params.hipTensor_branch}"
                            wget https://github.com/ROCmSoftwarePlatform/hipTensor/archive/refs/heads/"${params.hipTensor_branch}".zip
                            unzip -o "${params.hipTensor_branch}".zip
                        """
                        dir("hipTensor-${params.hipTensor_branch}"){
                            sh """#!/bin/bash
                                mkdir -p build
                                ls -ltr
                                CC=hipcc CXX=hipcc cmake -Bbuild . -D CMAKE_PREFIX_PATH="/opt/rocm;${env.WORKSPACE}/install"
                                cmake --build build -- -j
                            """
                        }
                        dir("hipTensor-${params.hipTensor_branch}/build"){
                            sh 'ctest'
                        }
                    }
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
                }
            }
        }
        return retimage
}

def Build_CK_and_Reboot(Map conf=[:]){
    try{
        Build_CK(conf)
    }
    catch(e){
        echo "throwing error exception while building CK"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

566
567
568
def process_results(Map conf=[:]){
    env.HSA_ENABLE_SDMA=0
    checkout scm
569
    def image = getDockerImageName() 
570
571
572
573
574
    def prefixpath = "/opt/rocm"

    // Jenkins is complaining about the render group 
    def dockerOpts="--cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
    if (conf.get("enforce_xnack_on", false)) {
575
        dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
576
577
578
579
580
581
582
    }

    def variant = env.STAGE_NAME
    def retimage

    gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') {
        try {
583
            (retimage, image) = getDockerImage(conf)
584
585
586
587
588
589
590
591
592
593
594
595
596
        }
        catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
            echo "The job was cancelled or aborted"
            throw e
        }
    }

    withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
        timeout(time: 1, unit: 'HOURS'){
            try{
                dir("script"){
                    if (params.RUN_FULL_QA){
                        // unstash perf files to master
597
598
599
600
601
602
603
604
605
606
607
608
                        unstash "perf_gemm.log"
                        unstash "perf_resnet50_N256.log"
                        unstash "perf_resnet50_N4.log"
                        unstash "perf_batched_gemm.log"
                        unstash "perf_grouped_gemm.log"
                        unstash "perf_conv_fwd.log"
                        unstash "perf_conv_bwd_data.log"
                        unstash "perf_gemm_bilinear.log"
                        unstash "perf_reduction.log"
                        unstash "perf_splitK_gemm.log"
                        unstash "perf_onnx_gemm.log"
                        sh "./process_qa_data.sh"
Illia Silin's avatar
Illia Silin committed
609
610
                        unstash "ckprofiler_0.2.0_amd64.deb"
                        sh "sshpass -p ${env.ck_deb_pw} scp -o StrictHostKeyChecking=no ckprofiler_0.2.0_amd64.deb ${env.ck_deb_user}@${env.ck_deb_ip}:/var/www/html/composable_kernel/"
611
612
613
                    }
                    else{
                        // unstash perf files to master
614
615
616
617
                        unstash "perf_gemm.log"
                        unstash "perf_resnet50_N256.log"
                        unstash "perf_resnet50_N4.log"
                        sh "./process_perf_data.sh"
618
619
620
621
622
623
624
625
626
627
628
629
                    }
                }
            }
            catch(e){
                echo "throwing error exception while processing performance test results"
                echo 'Exception occurred: ' + e.toString()
                throw e
            }
        }
    }
}

630
//launch develop branch daily at 23:00 UT in FULL_QA mode and at 19:00 UT with latest staging compiler version
631
632
CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;ROCMVERSION=5.7;COMPILER_VERSION=
                                              0 21 * * * % ROCMVERSION=5.7;COMPILER_VERSION=;COMPILER_COMMIT=
633
                                              0 19 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-stg-open;COMPILER_COMMIT=;USE_SCCACHE=false''' : ""
634

JD's avatar
JD committed
635
636
pipeline {
    agent none
637
638
639
    triggers {
        parameterizedCron(CRON_SETTINGS)
    }
JD's avatar
JD committed
640
641
642
    options {
        parallelsAlwaysFailFast()
    }
643
    parameters {
644
645
        booleanParam(
            name: "BUILD_DOCKER",
646
            defaultValue: false,
647
648
649
            description: "Force building docker image (default: false), set to true if docker image needs to be updated.")
        string(
            name: 'ROCMVERSION', 
650
651
            defaultValue: '5.7', 
            description: 'Specify which ROCM version to use: 5.7 (default).')
652
653
        string(
            name: 'COMPILER_VERSION', 
654
655
            defaultValue: '', 
            description: 'Specify which version of compiler to use: release, amd-stg-open, or leave blank (default).')
656
657
        string(
            name: 'COMPILER_COMMIT', 
658
659
            defaultValue: '', 
            description: 'Specify which commit of compiler branch to use: leave blank to use the latest commit, or use 5541927df00eabd6a110180170eca7785d436ee3 (default) commit of amd-stg-open branch.')
660
661
662
663
        string(
            name: 'BUILD_COMPILER', 
            defaultValue: 'hipcc', 
            description: 'Specify whether to build CK with hipcc (default) or with clang.')
664
665
666
667
        booleanParam(
            name: "RUN_FULL_QA",
            defaultValue: false,
            description: "Select whether to run small set of performance tests (default) or full QA")
668
669
670
671
        booleanParam(
            name: "DL_KERNELS",
            defaultValue: false,
            description: "Select whether to build DL kernels (default: OFF)")
672
673
674
675
676
677
678
679
        booleanParam(
            name: "hipTensor_test",
            defaultValue: true,
            description: "Use the CK build to verify hipTensor build and tests (default: ON)")
        string(
            name: 'hipTensor_branch',
            defaultValue: 'mainline',
            description: 'Specify which branch of hipTensor to use (default: mainline)')
680
681
682
683
        booleanParam(
            name: "USE_SCCACHE",
            defaultValue: true,
            description: "Use the sccache for building CK (default: ON)")
684
685
686
687
688
689
690
691
    }
    environment{
        dbuser = "${dbuser}"
        dbpassword = "${dbpassword}"
        dbsship = "${dbsship}"
        dbsshport = "${dbsshport}"
        dbsshuser = "${dbsshuser}"
        dbsshpassword = "${dbsshpassword}"
692
        status_wrapper_creds = "${status_wrapper_creds}"
693
694
        gerrit_cred="${gerrit_cred}"
        DOCKER_BUILDKIT = "1"
695
    }
JD's avatar
JD committed
696
    stages{
697
698
699
700
701
702
        stage("Build Docker"){
            parallel{
                stage('Docker /opt/rocm'){
                    agent{ label rocmnode("nogpu") }
                    steps{
                        buildDocker('/opt/rocm')
703
                        cleanWs()
704
705
706
707
                    }
                }
            }
        }
JD's avatar
JD committed
708
709
710
711
712
        stage("Static checks") {
            parallel{
                stage('Clang Format') {
                    agent{ label rocmnode("nogpu") }
                    environment{
Illia Silin's avatar
Illia Silin committed
713
714
715
                        execute_cmd = "find .. -not -path \'*.git*\' -iname \'*.h\' \
                                -o -not -path \'*.git*\' -iname \'*.hpp\' \
                                -o -not -path \'*.git*\' -iname \'*.cpp\' \
JD's avatar
JD committed
716
717
718
719
720
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
721
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\'"
JD's avatar
JD committed
722
723
724
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
725
                        cleanWs()
JD's avatar
JD committed
726
727
728
729
                    }
                }
            }
        }
730
731
    
		stage("Build CK and run Tests")
732
733
734
        {
            parallel
            {
735
736
737
738
739
740
741
742
                stage("Build CK and run Tests on MI100/MI200/MI300")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_FULL_QA.toBoolean() }
                    }
                    agent{ label rocmnode("gfx908 || gfx90a") }
                    environment{
743
744
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx908;gfx90a;gfx940;gfx941;gfx942" """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -DGPU_TARGETS="gfx908;gfx90a;gfx940;gfx941;gfx942" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """ 
745
746
747
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
748
                        cleanWs()
749
750
                    }
                }
Illia Silin's avatar
Illia Silin committed
751
                stage("Build CK and run Tests on MI100/MI200")
752
                {
753
754
755
756
                    when {
                        beforeAgent true
                        expression { !params.RUN_FULL_QA.toBoolean() }
                    }
757
                    agent{ label rocmnode("gfx908 || gfx90a") }
758
                    environment{
Illia Silin's avatar
Illia Silin committed
759
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx908;gfx90a" """
760
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -DGPU_TARGETS="gfx908;gfx90a" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """ 
Illia Silin's avatar
Illia Silin committed
761
762
763
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
764
                        cleanWs()
Illia Silin's avatar
Illia Silin committed
765
766
                    }
                }
767
                stage("Build CK and run Tests on Navi21")
Illia Silin's avatar
Illia Silin committed
768
769
770
771
772
773
774
                {
                    when {
                        beforeAgent true
                        expression { !params.RUN_FULL_QA.toBoolean() }
                    }
                    agent{ label rocmnode("navi21") }
                    environment{
775
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1030" -DDL_KERNELS=ON """ 
776
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -DGPU_TARGETS="gfx1030" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """
777
                    }
778
779
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
780
                        cleanWs()
781
782
783
784
785
786
787
788
789
790
                    }
                }
                stage("Build CK and run Tests on Navi32")
                {
                    when {
                        beforeAgent true
                        expression { !params.RUN_FULL_QA.toBoolean() }
                    }
                    agent{ label rocmnode("navi32") }
                    environment{
791
792
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1101" -DDL_KERNELS=ON """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -DGPU_TARGETS="gfx1101" -DDL_KERNELS=ON -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """
793
                    }
794
                    steps{
795
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
796
                        cleanWs()
JD's avatar
JD committed
797
798
                    }
                }
799
800
            }
        }
801

802
803
804
805
        stage("Performance Tests")
        {
            parallel
            {
Illia Silin's avatar
Illia Silin committed
806
                stage("Run ckProfiler: gfx90*")
807
                {
808
809
                    when {
                        beforeAgent true
810
                        expression { !params.RUN_FULL_QA.toBoolean() }
811
                    }
812
                    options { retry(2) }
813
                    agent{ label rocmnode("gfx908 || gfx90a")}
814
                    environment{
Illia Silin's avatar
Illia Silin committed
815
                        setup_args = """ -DGPU_TARGETS="gfx908;gfx90a" -DBUILD_DEV=On """
816
                   }
817
                    steps{
818
                        runPerfTest(setup_args:setup_args, config_targets: "ckProfiler", no_reboot:true, build_type: 'Release')
819
                        cleanWs()
820
821
822
823
                    }
                }
                stage("Run ckProfiler: gfx90a")
                {
824
825
                    when {
                        beforeAgent true
826
                        expression { params.RUN_FULL_QA.toBoolean() }
827
                    }
828
                    options { retry(2) }
829
830
                    agent{ label rocmnode("gfx90a")}
                    environment{
Illia Silin's avatar
Illia Silin committed
831
                        setup_args = """ -DGPU_TARGETS="gfx90a" -DBUILD_DEV=On """
832
                    }
833
                    steps{
834
                        runPerfTest(setup_args:setup_args, config_targets: "ckProfiler", no_reboot:true, build_type: 'Release')
835
                        cleanWs()
836
837
838
839
                    }
                }
            }
        }
840
841
842
843
        stage("Process Performance Test Results")
        {
            parallel
            {
844
                stage("Process results"){
845
846
                    agent { label 'mici' }
                    steps{
847
                        process_results()
848
                        cleanWs()
849
850
851
852
                    }
                }
            }
        }
JD's avatar
JD committed
853
    }
854
}