Jenkinsfile 54.6 KB
Newer Older
JD's avatar
JD committed
1
def rocmnode(name) {
2
    return '(rocmtest || miopen) && (' + name + ')'
JD's avatar
JD committed
3
4
5
6
7
8
9
}

def show_node_info() {
    sh """
        echo "NODE_NAME = \$NODE_NAME"
        lsb_release -sd
        uname -r
10
        cat /sys/module/amdgpu/version
JD's avatar
JD committed
11
12
13
14
        ls /opt/ -la
    """
}

15
16
17
18
19
20
21
22
23
24
25
26
27
28
def nthreads() {
    def nproc = sh(returnStdout: true, script: 'nproc')
    echo "Number of cores: ${nproc}"
    def n = nproc.toInteger()
    if (n > 32){
        n /= 2
    }
    if (n > 64){
        n = 64
    }
    echo "Number of threads used for building: ${n}"
    return n
}

29
def runShell(String command){
30
    def responseCode = sh returnStatus: true, script: "${command} > tmp.txt"
31
32
33
34
    def output = readFile(file: "tmp.txt")
    return (output != "")
}

35
def getBaseDockerImageName(){
36
    def img
37
38
39
40
    if (params.USE_CUSTOM_DOCKER != ""){
        img = "${params.USE_CUSTOM_DOCKER}"
    }
    else{
41
42
        def ROCM_numeric = "${params.ROCMVERSION}" as float
        if ( ROCM_numeric < 6.4 ){
43
            img = "${env.CK_DOCKERHUB}:ck_ub22.04_rocm${params.ROCMVERSION}"
44
45
            }
        else{
46
            img = "${env.CK_DOCKERHUB_PRIVATE}:ck_ub22.04_rocm${params.ROCMVERSION}"
47
            }
48
49
50
51
52
53
54
55
56
        }
    return img
}

def getDockerImageName(){
    def img
    def base_name = getBaseDockerImageName()
    if (params.USE_CUSTOM_DOCKER != ""){
        img = "${params.USE_CUSTOM_DOCKER}"
57
58
    }
    else{
59
       if (params.COMPILER_VERSION == "") {
60
           img = "${base_name}"
61
62
       }
       else{
63
          if (params.COMPILER_COMMIT == ""){
64
             img = "${base_name}_${params.COMPILER_VERSION}"
65
66
67
          }
          else{
             def commit = "${params.COMPILER_COMMIT}"[0..6]
68
             img = "${base_name}_${params.COMPILER_VERSION}_${commit}"
69
          }
70
       }
71
    }
72
73
74
    return img
}

75
def check_host() {
76
77
78
79
    if ("${env.CK_SCCACHE}" != "null"){
        def SCCACHE_SERVER="${env.CK_SCCACHE.split(':')[0]}"
        echo "sccache server: ${SCCACHE_SERVER}"
        sh '''ping -c 1 -p 6379 "${SCCACHE_SERVER}" | echo $? > tmp.txt'''
80
81
82
83
84
85
86
87
88
        def output = readFile(file: "tmp.txt")
        echo "tmp.txt contents: \$output"
        return (output != "0")
    }
    else{
        return 1
    }
}

89
90
def build_compiler(){
    def compiler
91
    compiler = "${params.BUILD_COMPILER}"
92
93
94
    return compiler
}

95
96
def getDockerImage(Map conf=[:]){
    env.DOCKER_BUILDKIT=1
97
    def prefixpath = conf.get("prefixpath", "/opt/rocm")
98
    def no_cache = conf.get("no_cache", false)
99
    def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' --build-arg DISABLE_CACHE='git rev-parse ${params.COMPILER_VERSION}' "
100
101
102
103
104
    if(no_cache)
    {
        dockerArgs = dockerArgs + " --no-cache "
    }
    echo "Docker Args: ${dockerArgs}"
105
106
107
108
109
110
111
112
113
    def image
    if ( params.BUILD_LEGACY_OS && conf.get("docker_name", "") != "" ){
        image = conf.get("docker_name", "")
        echo "Using legacy docker: ${image}"
    }
    else{
        image = getDockerImageName()
        echo "Using default docker: ${image}"
    }
114
115
116
117
118
119
    //Check if image exists 
    def retimage
    try 
    {
        echo "Pulling down image: ${image}"
        retimage = docker.image("${image}")
120
121
122
        withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
            retimage.pull()
        }
123
124
125
126
127
128
129
130
131
132
133
134
135
    }
    catch(Exception ex)
    {
        error "Unable to locate image: ${image}"
    }
    return [retimage, image]
}

def buildDocker(install_prefix){
    show_node_info()
    env.DOCKER_BUILDKIT=1
    checkout scm
    def image_name = getDockerImageName()
136
    def base_image_name = getBaseDockerImageName()
137
    echo "Building Docker for ${image_name}"
138
    def dockerArgs = "--build-arg PREFIX=${install_prefix} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
139
    if(params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
140
141
142
143
        dockerArgs = dockerArgs + " --no-cache --build-arg BASE_DOCKER='${base_image_name}' -f Dockerfile.compiler . "
    }
    else{
        dockerArgs = dockerArgs + " -f Dockerfile . "
144
    }
145
146
    echo "Build Args: ${dockerArgs}"
    try{
147
148
149
        if(params.BUILD_DOCKER){
            //force building the new docker if that parameter is true
            echo "Building image: ${image_name}"
150
            retimage = docker.build("${image_name}", dockerArgs)
151
152
153
            withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
                retimage.push()
            }
154
            sh 'docker images -q -f dangling=true | xargs --no-run-if-empty docker rmi'
155
156
157
158
        }
        else{
            echo "Checking for image: ${image_name}"
            sh "docker manifest inspect --insecure ${image_name}"
159
            echo "Image: ${image_name} found! Skipping building image"
160
        }
161
162
163
164
    }
    catch(Exception ex){
        echo "Unable to locate image: ${image_name}. Building image now"
        retimage = docker.build("${image_name}", dockerArgs + ' .')
165
166
167
        withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
            retimage.push()
        }
168
169
170
    }
}

JD's avatar
JD committed
171
172
def cmake_build(Map conf=[:]){

173
    def compiler = build_compiler()
JD's avatar
JD committed
174
175
    def config_targets = conf.get("config_targets","check")
    def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined " + conf.get("extradebugflags", "")
176
    def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","")
JD's avatar
JD committed
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
    def prefixpath = conf.get("prefixpath","/opt/rocm")
    def setup_args = conf.get("setup_args","")

    if (prefixpath != "/usr/local"){
        setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} "
    }

    def build_type_debug = (conf.get("build_type",'release') == 'debug')

    //cmake_env can overwrite default CXX variables.
    def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","")

    def package_build = (conf.get("package_build","") == "true")

    if (package_build == true) {
        config_targets = "package"
    }

    if(conf.get("build_install","") == "true")
    {
        config_targets = 'install ' + config_targets
198
        setup_args = ' -DBUILD_DEV=On -DCMAKE_INSTALL_PREFIX=../install' + setup_args
JD's avatar
JD committed
199
200
201
    } else{
        setup_args = ' -DBUILD_DEV=On' + setup_args
    }
202
203
204
    if (params.DL_KERNELS){
        setup_args = setup_args + " -DDL_KERNELS=ON "
    }
JD's avatar
JD committed
205
206
207
208
209
210
211
212

    if(build_type_debug){
        setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args
    }else{
        setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args
    }

    def pre_setup_cmd = """
213
            #!/bin/bash
JD's avatar
JD committed
214
215
216
217
218
219
220
221
            echo \$HSA_ENABLE_SDMA
            ulimit -c unlimited
            rm -rf build
            mkdir build
            rm -rf install
            mkdir install
            cd build
        """
222
    def invocation_tag=""
Illia Silin's avatar
Illia Silin committed
223
224
225
    if (setup_args.contains("gfx12")){
        invocation_tag="gfx12"
    }
226
227
228
229
230
231
232
233
234
235
236
237
    if (setup_args.contains("gfx11")){
        invocation_tag="gfx11"
    }
    if (setup_args.contains("gfx10")){
        invocation_tag="gfx10"
    }
    if (setup_args.contains("gfx90")){
        invocation_tag="gfx90"
    }
    if (setup_args.contains("gfx94")){
        invocation_tag="gfx94"
    }
238
239
    echo "invocation tag: ${invocation_tag}"
    def redis_pre_setup_cmd = pre_setup_cmd
240
    if(check_host() && params.USE_SCCACHE && "${env.CK_SCCACHE}" != "null" && "${invocation_tag}" != "") {
241
        redis_pre_setup_cmd = pre_setup_cmd + """
242
243
244
245
246
247
248
249
250
251
252
253
254
            #!/bin/bash
            export ROCM_PATH=/opt/rocm
            export SCCACHE_ENABLED=true
            export SCCACHE_LOG_LEVEL=debug
            export SCCACHE_IDLE_TIMEOUT=14400
            export COMPILERS_HASH_DIR=/tmp/.sccache
            export SCCACHE_BIN=/usr/local/.cargo/bin/sccache
            export SCCACHE_EXTRAFILES=/tmp/.sccache/rocm_compilers_hash_file
            export SCCACHE_REDIS="redis://${env.CK_SCCACHE}"
            echo "connect = ${env.CK_SCCACHE}" >> ../script/redis-cli.conf
            export SCCACHE_C_CUSTOM_CACHE_BUSTER="${invocation_tag}"
            echo \$SCCACHE_C_CUSTOM_CACHE_BUSTER
            stunnel ../script/redis-cli.conf
255
            ../script/sccache_wrapper.sh --enforce_redis
256
        """
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
        try {
            def cmd1 = conf.get("cmd1", """
                    ${redis_pre_setup_cmd}
                """)
            sh cmd1
            setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache " + setup_args
        }
        catch(Exception err){
            echo "could not connect to redis server: ${err.getMessage()}. will not use sccache."
            def cmd2 = conf.get("cmd2", """
                    ${pre_setup_cmd}
                """)
            sh cmd2
        }
    }
    else{
        def cmd3 = conf.get("cmd3",  """
                ${pre_setup_cmd}
            """)
        sh cmd3
277
    }
278

Chao Liu's avatar
Chao Liu committed
279
    // reduce parallelism when compiling, clang uses too much memory
280
    def nt = nthreads()
Paul Fultz II's avatar
Paul Fultz II committed
281
    def cmd
282
283
    def setup_cmd
    def build_cmd
JD's avatar
JD committed
284
    def execute_cmd = conf.get("execute_cmd", "")
Paul Fultz II's avatar
Paul Fultz II committed
285
    if(!setup_args.contains("NO_CK_BUILD")){
286
287
288
289
290
291
292
        if (setup_args.contains("gfx90a") && params.NINJA_BUILD_TRACE){
            echo "running ninja build trace"
            setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake -G Ninja ${setup_args}   .. ")
            build_cmd = conf.get("build_cmd", "${build_envs} ninja -j${nt} ${config_targets}")
        }
        else{
            setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args}   .. ")
293
            build_cmd = conf.get("build_cmd", "${build_envs} make -j${nt} ${config_targets}")
294
        }
Paul Fultz II's avatar
Paul Fultz II committed
295
        cmd = conf.get("cmd", """
JD's avatar
JD committed
296
297
298
299
            ${setup_cmd}
            ${build_cmd}
            ${execute_cmd}
        """)
Paul Fultz II's avatar
Paul Fultz II committed
300
301
302
303
304
305
    }
    else{
        cmd = conf.get("cmd", """
            ${execute_cmd}
        """)
    }
JD's avatar
JD committed
306
307

    echo cmd
308
309

    dir("build"){
310
        //build CK
311
        sh cmd
312
313
        //run tests except when NO_CK_BUILD or BUILD_LEGACY_OS are set
        if(!setup_args.contains("NO_CK_BUILD") && !params.BUILD_LEGACY_OS){
314
315
316
317
318
319
320
321
322
            if (setup_args.contains("gfx90a") && params.NINJA_BUILD_TRACE){
                sh "/ninjatracing/ninjatracing .ninja_log > ck_build_trace.json"
                archiveArtifacts "ck_build_trace.json"
                sh "ninja test"
            }
            else{
                sh "make check"
            }
        }
323
    }
JD's avatar
JD committed
324
325

    // Only archive from master or develop
326
    if (package_build == true && (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "amd-master")) {
JD's avatar
JD committed
327
328
        archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true
    }
329
330
331
332
333
334
335
336
337
    //check the node gpu architecture
    def arch_type = 0
    sh 'rocminfo | tee rocminfo.log'
    if ( runShell('grep -n "gfx90a" rocminfo.log') ){
        arch_type = 1
    }
    else if ( runShell('grep -n "gfx942" rocminfo.log') ) {
        arch_type = 2
    }
338
    if (params.RUN_CK_TILE_FMHA_TESTS){
339
        try{
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
            archiveArtifacts "perf_fmha_*.log"
            if (arch_type == 1){
                stash includes: "perf_fmha_**_gfx90a.log", name: "perf_fmha_log_gfx90a"
            }
            else if (arch_type == 2){
                stash includes: "perf_fmha_**_gfx942.log", name: "perf_fmha_log_gfx942"
            }
        }
        catch(Exception err){
            echo "could not locate the requested artifacts: ${err.getMessage()}. will skip the stashing."
        }
    }
    if (params.RUN_CK_TILE_GEMM_TESTS){
        try{
            archiveArtifacts "perf_tile_gemm_*.log"
            if (arch_type == 1){
                stash includes: "perf_tile_gemm_**_fp16_gfx90a.log", name: "perf_tile_gemm_log_gfx90a"
            }
            else if (arch_type == 2){
                stash includes: "perf_tile_gemm_**_fp16_gfx942.log", name: "perf_tile_gemm_log_gfx942"
            }
361
362
363
364
365
        }
        catch(Exception err){
            echo "could not locate the requested artifacts: ${err.getMessage()}. will skip the stashing."
        }
    }
JD's avatar
JD committed
366
367
368
369
370
371
372
373
}

def buildHipClangJob(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

374
375
376
377
378
379
380
381
382
        def image
        if ( params.BUILD_LEGACY_OS  && conf.get("docker_name", "") != "" ){
            image = conf.get("docker_name", "")
            echo "Using legacy docker: ${image}"
        }
        else{
            image = getDockerImageName()
            echo "Using default docker: ${image}"
        }
JD's avatar
JD committed
383
384
385
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
386
        def dockerOpts="-u root --device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
JD's avatar
JD committed
387
        if (conf.get("enforce_xnack_on", false)) {
388
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
JD's avatar
JD committed
389
        }
390
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
391
        if (params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
392
393
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
394
395
396
397
        def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3')
        def render_id = sh(returnStdout: true, script: 'getent group render | cut -d: -f3')
        dockerOpts = dockerOpts + " --group-add=${video_id} --group-add=${render_id} "
        echo "Docker flags: ${dockerOpts}"
JD's avatar
JD committed
398
399
400
401

        def variant = env.STAGE_NAME

        def retimage
402
        (retimage, image) = getDockerImage(conf)
403

404
        gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
JD's avatar
JD committed
405
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
406
                timeout(time: 20, unit: 'HOURS')
JD's avatar
JD committed
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
                {
                    cmake_build(conf)
                }
            }
        }
        return retimage
}

def reboot(){
    build job: 'reboot-slaves', propagate: false , parameters: [string(name: 'server', value: "${env.NODE_NAME}"),]
}

def buildHipClangJobAndReboot(Map conf=[:]){
    try{
        buildHipClangJob(conf)
    }
    catch(e){
        echo "throwing error exception for the stage"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

435
436
437
438
def Build_CK(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
439
        env.DOCKER_BUILDKIT=1
440
441
        checkout scm

442
443
444
445
446
447
448
449
450
451
        def image
        if ( params.BUILD_LEGACY_OS  && conf.get("docker_name", "") != "" ){
            image = conf.get("docker_name", "")
            echo "Using legacy docker: ${image}"
        }
        else{
            image = getDockerImageName()
            echo "Using default docker: ${image}"
        }

452
453
454
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
455
        def dockerOpts="-u root --device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
456
457
458
        if (conf.get("enforce_xnack_on", false)) {
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
        }
459
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
460
        if (params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
461
462
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
463
464
465
        if(params.BUILD_LEGACY_OS){
            dockerOpts = dockerOpts + " --env LD_LIBRARY_PATH='/opt/Python-3.8.13/lib' "
        }
466
467
468
469
        def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3')
        def render_id = sh(returnStdout: true, script: 'getent group render | cut -d: -f3')
        dockerOpts = dockerOpts + " --group-add=${video_id} --group-add=${render_id} "
        echo "Docker flags: ${dockerOpts}"
470
471
472

        def variant = env.STAGE_NAME
        def retimage
473

474
        gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
475
476
477
            try {
                (retimage, image) = getDockerImage(conf)
                withDockerContainer(image: image, args: dockerOpts) {
Illia Silin's avatar
Illia Silin committed
478
                    timeout(time: 2, unit: 'MINUTES'){
479
480
                        sh 'rocminfo | tee rocminfo.log'
                        if ( !runShell('grep -n "gfx" rocminfo.log') ){
481
482
483
484
485
486
487
488
489
490
491
492
493
                            throw new Exception ("GPU not found")
                        }
                        else{
                            echo "GPU is OK"
                        }
                    }
                }
            }
            catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
                echo "The job was cancelled or aborted"
                throw e
            }
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
494
                timeout(time: 20, unit: 'HOURS')
495
                {
Illia Silin's avatar
Illia Silin committed
496
                    //check whether to run performance tests on this node
497
                    def arch_type = 0
498
                    sh 'rocminfo | tee rocminfo.log'
499
500
501
502
503
504
505
506
507
508
509
510
511
512
                    if ( runShell('grep -n "gfx90a" rocminfo.log') ){
                        arch_type = 1
                    }
                    else if ( runShell('grep -n "gfx942" rocminfo.log') ) {
                        arch_type = 2
                    }
                    else if ( runShell('grep -n "gfx1030" rocminfo.log') ) {
                        arch_type = 3
                    }
                    else if ( runShell('grep -n "gfx1101" rocminfo.log') ) {
                        arch_type = 4
                    }
                    else if ( runShell('grep -n "gfx1201" rocminfo.log') ) {
                        arch_type = 5
513
                    }
514
                    cmake_build(conf)
515
                    if ( !params.BUILD_LEGACY_OS && arch_type == 1 ){
516
517
518
519
520
521
                            echo "Run inductor codegen tests"
                            sh """
                                  pip install --verbose .
                                  pytest python/test/test_gen_instances.py
                            """
                    }
522
                    dir("build"){
523
524
525
                        if (params.RUN_FULL_QA && arch_type == 1 ){
                            // build deb packages for all gfx9 targets on gfx90a system and prepare to export
                            echo "Build ckProfiler package"
526
527
528
                            sh 'make -j package'
                            archiveArtifacts artifacts: 'composablekernel-ckprofiler_*.deb'
                            sh 'mv composablekernel-ckprofiler_*.deb ckprofiler_0.2.0_amd64.deb'
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
                            stash includes: "ckprofiler_0.2.0_amd64.deb", name: "ckprofiler_0.2.0_amd64.deb"
                        }
                    }
                    // run performance tests, stash the logs, results will be processed on the master node
					dir("script"){
                        if (params.RUN_PERFORMANCE_TESTS){
                        if (params.RUN_FULL_QA && arch_type == 1){
                            // run full tests on gfx90a
                            echo "Run full performance tests"
                            sh "./run_full_performance_tests.sh 0 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
                            archiveArtifacts "perf_batched_gemm.log"
                            archiveArtifacts "perf_grouped_gemm.log"
                            archiveArtifacts "perf_grouped_conv_fwd.log"
                            archiveArtifacts "perf_grouped_conv_bwd_data.log"
                            archiveArtifacts "perf_grouped_conv_bwd_weight.log"
                            archiveArtifacts "perf_gemm_bilinear.log"
                            archiveArtifacts "perf_reduction.log"
                            archiveArtifacts "perf_splitK_gemm.log"
                            archiveArtifacts "perf_onnx_gemm.log"
                            archiveArtifacts "perf_mixed_gemm.log"
                            stash includes: "perf_**.log", name: "perf_log"
                        }
                        else if ( arch_type == 1 ){
                            // run standard tests on gfx90a
                            echo "Run performance tests"
                            sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_onnx_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
                            stash includes: "perf_**.log", name: "perf_log"
                        }
                        // disable performance tests on gfx1030 for now.
                        //else if ( arch_type == 3){
                            // run basic tests on gfx1030
                        //    echo "Run gemm performance tests"
                        //    sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx10"
                        //    archiveArtifacts "perf_onnx_gemm_gfx10.log"
                        //    stash includes: "perf_onnx_gemm_gfx10.log", name: "perf_log_gfx10"
                        //}
                        else if ( arch_type == 4){
                            // run basic tests on gfx11
                            echo "Run gemm performance tests"
                            sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx11"
                            archiveArtifacts "perf_onnx_gemm_gfx11.log"
                            stash includes: "perf_onnx_gemm_gfx11.log", name: "perf_log_gfx11"
                        }
                        else if ( arch_type == 5 ){
                            // run basic tests on gfx12
                            echo "Run gemm performance tests"
                            sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx12"
                            archiveArtifacts "perf_onnx_gemm_gfx12.log"
                            stash includes: "perf_onnx_gemm_gfx12.log", name: "perf_log_gfx12"
                        }                        
586
                        }
587
                    }
588
589
                    if (params.hipTensor_test && arch_type == 1 ){
                        // build and test hipTensor on gfx90a node
590
591
592
                        sh """#!/bin/bash
                            rm -rf "${params.hipTensor_branch}".zip
                            rm -rf hipTensor-"${params.hipTensor_branch}"
593
                            wget https://github.com/ROCm/hipTensor/archive/refs/heads/"${params.hipTensor_branch}".zip
594
595
596
597
598
599
                            unzip -o "${params.hipTensor_branch}".zip
                        """
                        dir("hipTensor-${params.hipTensor_branch}"){
                            sh """#!/bin/bash
                                mkdir -p build
                                ls -ltr
600
                                CC=hipcc CXX=hipcc cmake -Bbuild . -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install"
601
                                cmake --build build -- -j
602
                                ctest --test-dir build
603
604
605
                            """
                        }
                    }
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
                }
            }
        }
        return retimage
}

def Build_CK_and_Reboot(Map conf=[:]){
    try{
        Build_CK(conf)
    }
    catch(e){
        echo "throwing error exception while building CK"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

628
629
630
def process_results(Map conf=[:]){
    env.HSA_ENABLE_SDMA=0
    checkout scm
631
    def image = getDockerImageName() 
632
633
634
    def prefixpath = "/opt/rocm"

    // Jenkins is complaining about the render group 
635
    def dockerOpts="--cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
636
    if (conf.get("enforce_xnack_on", false)) {
637
        dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
638
639
640
641
642
    }

    def variant = env.STAGE_NAME
    def retimage

643
    gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
644
        try {
645
            (retimage, image) = getDockerImage(conf)
646
647
648
649
650
651
652
653
        }
        catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
            echo "The job was cancelled or aborted"
            throw e
        }
    }

    withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
654
        timeout(time: 15, unit: 'MINUTES'){
655
656
            try{
                dir("script"){
657
                    if (params.RUN_CK_TILE_FMHA_TESTS){
658
                        try{
659
660
                            unstash "perf_fmha_log_gfx942"
                            unstash "perf_fmha_log_gfx90a"
661
662
663
664
665
                        }
                        catch(Exception err){
                            echo "could not locate the FMHA performance logs: ${err.getMessage()}."
                        }
                    }
666
667
668
669
670
671
672
673
674
                    if (params.RUN_CK_TILE_GEMM_TESTS){
                        try{
                            unstash "perf_tile_gemm_log_gfx942"
                            unstash "perf_tile_gemm_log_gfx90a"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM performance logs: ${err.getMessage()}."
                        }
                    }
675
676
                    if (params.RUN_FULL_QA){
                        // unstash perf files to master
677
678
                        unstash "ckprofiler_0.2.0_amd64.deb"
                        sh "sshpass -p ${env.ck_deb_pw} scp -o StrictHostKeyChecking=no ckprofiler_0.2.0_amd64.deb ${env.ck_deb_user}@${env.ck_deb_ip}:/var/www/html/composable_kernel/"
679
680
681
682
683
684
685
686
                        unstash "perf_log"
                        try{
                            unstash "perf_log_gfx11"
                            unstash "perf_log_gfx12"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM gfx11/gfx12 performance logs: ${err.getMessage()}."
                        }
687
                        sh "./process_qa_data.sh"
688
689
690
                    }
                    else{
                        // unstash perf files to master
691
692
693
694
695
696
697
698
                        unstash "perf_log"
                        try{
                            unstash "perf_log_gfx11"
                            unstash "perf_log_gfx12"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM gfx11/gfx12 performance logs: ${err.getMessage()}."
                        }
699
                        sh "./process_perf_data.sh"
700
701
702
703
                    }
                }
            }
            catch(e){
704
                echo "Throwing error exception while processing performance test results"
705
706
707
                echo 'Exception occurred: ' + e.toString()
                throw e
            }
708
709
710
            finally{
                echo "Finished processing performance test results"
            }
711
712
713
714
        }
    }
}

715
//launch develop branch daily at 23:00 UT in FULL_QA mode and at 19:00 UT with latest staging compiler version
716
717
CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;ROCMVERSION=6.3;RUN_CK_TILE_FMHA_TESTS=true;RUN_CK_TILE_GEMM_TESTS=true
                                              0 21 * * * % ROCMVERSION=6.3;hipTensor_test=true;RUN_CODEGEN_TESTS=true
718
719
                                              0 19 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-staging;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true
                                              0 17 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-mainline;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true
720
721
                                              0 15 * * * % BUILD_INSTANCES_ONLY=true;RUN_PERFORMANCE_TESTS=false;USE_SCCACHE=false
                                              0 13 * * * % BUILD_LEGACY_OS=true''' : ""
722

JD's avatar
JD committed
723
724
pipeline {
    agent none
725
726
727
    triggers {
        parameterizedCron(CRON_SETTINGS)
    }
JD's avatar
JD committed
728
729
730
    options {
        parallelsAlwaysFailFast()
    }
731
    parameters {
732
733
        booleanParam(
            name: "BUILD_DOCKER",
734
            defaultValue: false,
735
            description: "Force building docker image (default: false), set to true if docker image needs to be updated.")
736
737
738
        string(
            name: 'USE_CUSTOM_DOCKER',
            defaultValue: '',
Paul Fultz II's avatar
Paul Fultz II committed
739
            description: 'If you want to use a custom docker image, please specify it here (default: leave blank).')
740
741
        string(
            name: 'ROCMVERSION', 
742
743
            defaultValue: '6.3',
            description: 'Specify which ROCM version to use: 6.3 (default).')
744
745
        string(
            name: 'COMPILER_VERSION', 
746
            defaultValue: '', 
747
            description: 'Specify which version of compiler to use: release, amd-staging, amd-mainline, or leave blank (default).')
748
749
        string(
            name: 'COMPILER_COMMIT', 
750
            defaultValue: '', 
751
            description: 'Specify which commit of compiler branch to use: leave blank to use the latest commit (default), or use some specific commit of llvm-project branch.')
752
753
        string(
            name: 'BUILD_COMPILER', 
754
755
            defaultValue: '/opt/rocm/llvm/bin/clang++', 
            description: 'Build CK with /opt/rocm/bin/hipcc, /llvm-project/build/bin/clang++, or with /opt/rocm/llvm/bin/clang++ (default).')
756
757
758
759
        booleanParam(
            name: "RUN_FULL_QA",
            defaultValue: false,
            description: "Select whether to run small set of performance tests (default) or full QA")
760
761
762
763
        booleanParam(
            name: "DL_KERNELS",
            defaultValue: false,
            description: "Select whether to build DL kernels (default: OFF)")
764
765
        booleanParam(
            name: "hipTensor_test",
766
767
            defaultValue: false,
            description: "Use the CK build to verify hipTensor build and tests (default: OFF)")
768
769
        string(
            name: 'hipTensor_branch',
770
771
            defaultValue: 'mainline',
            description: 'Specify which branch of hipTensor to use (default: mainline)')
772
773
774
775
        booleanParam(
            name: "USE_SCCACHE",
            defaultValue: true,
            description: "Use the sccache for building CK (default: ON)")
Illia Silin's avatar
Illia Silin committed
776
777
778
779
        booleanParam(
            name: "RUN_CPPCHECK",
            defaultValue: false,
            description: "Run the cppcheck static analysis (default: OFF)")
780
781
        booleanParam(
            name: "RUN_PERFORMANCE_TESTS",
782
783
            defaultValue: true,
            description: "Run the performance tests (default: ON)")
784
785
786
787
        booleanParam(
            name: "RUN_GROUPED_CONV_LARGE_CASES_TESTS",
            defaultValue: false,
            description: "Run the grouped conv large cases tests (default: OFF)")
788
789
790
791
        booleanParam(
            name: "RUN_CODEGEN_TESTS",
            defaultValue: false,
            description: "Run codegen tests (default: OFF)")
792
        booleanParam(
793
            name: "RUN_CK_TILE_FMHA_TESTS",
794
            defaultValue: false,
795
796
797
            description: "Run the ck_tile FMHA tests (default: OFF)")
        booleanParam(
            name: "RUN_CK_TILE_GEMM_TESTS",
798
            defaultValue: true,
799
            description: "Run the ck_tile GEMM tests (default: ON)")
800
801
802
803
        booleanParam(
            name: "BUILD_INSTANCES_ONLY",
            defaultValue: false,
            description: "Test building instances for various architectures simultaneously (default: OFF)")
Illia Silin's avatar
Illia Silin committed
804
805
        booleanParam(
            name: "BUILD_GFX12",
806
807
            defaultValue: true,
            description: "Build CK and run tests on gfx12 (default: ON)")
808
809
810
811
        booleanParam(
            name: "NINJA_BUILD_TRACE",
            defaultValue: false,
            description: "Generate a ninja build trace (default: OFF)")
812
813
814
815
        booleanParam(
            name: "BUILD_LEGACY_OS",
            defaultValue: false,
            description: "Try building CK with legacy OS dockers: RHEL8 and SLES15 (default: OFF)")
816
817
818
819
820
821
822
823
    }
    environment{
        dbuser = "${dbuser}"
        dbpassword = "${dbpassword}"
        dbsship = "${dbsship}"
        dbsshport = "${dbsshport}"
        dbsshuser = "${dbsshuser}"
        dbsshpassword = "${dbsshpassword}"
824
        ck_git_creds = "${ck_git_creds}"
825
826
        gerrit_cred="${gerrit_cred}"
        DOCKER_BUILDKIT = "1"
827
    }
JD's avatar
JD committed
828
    stages{
829
830
831
832
833
834
        stage("Build Docker"){
            parallel{
                stage('Docker /opt/rocm'){
                    agent{ label rocmnode("nogpu") }
                    steps{
                        buildDocker('/opt/rocm')
835
                        cleanWs()
836
837
838
839
                    }
                }
            }
        }
JD's avatar
JD committed
840
841
        stage("Static checks") {
            parallel{
Illia Silin's avatar
Illia Silin committed
842
843
844
845
846
847
848
                stage('Clang Format and Cppcheck') {
                    when {
                        beforeAgent true
                        expression { params.RUN_CPPCHECK.toBoolean() }
                    }
                    agent{ label rocmnode("nogpu") }
                    environment{
849
                        setup_args = "NO_CK_BUILD"
Illia Silin's avatar
Illia Silin committed
850
851
852
853
854
855
856
857
858
                        execute_cmd = "find .. -not -path \'*.git*\' -iname \'*.h\' \
                                -o -not -path \'*.git*\' -iname \'*.hpp\' \
                                -o -not -path \'*.git*\' -iname \'*.cpp\' \
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\' && \
859
860
861
862
863
                                /cppcheck/build/bin/cppcheck ../* -v -j \$(nproc) -I ../include -I ../profiler/include -I ../library/include \
                                -D CK_ENABLE_FP64 -D CK_ENABLE_FP32 -D CK_ENABLE_FP16 -D CK_ENABLE_FP8 -D CK_ENABLE_BF16 -D CK_ENABLE_BF8 -D CK_ENABLE_INT8 -D DL_KERNELS \
                                -D __gfx908__ -D __gfx90a__ -D __gfx940__ -D __gfx941__ -D __gfx942__ -D __gfx1030__ -D __gfx1100__ -D __gfx1101__ -D __gfx1102__ \
                                -U __gfx803__ -U __gfx900__ -U __gfx906__ -U CK_EXPERIMENTAL_BIT_INT_EXTENSION_INT4 \
                                --file-filter=*.cpp --force --enable=all --output-file=ck_cppcheck.log"
Illia Silin's avatar
Illia Silin committed
864
865
                    }
                    steps{
866
                        buildHipClangJobAndReboot(setup_args:setup_args, setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
Illia Silin's avatar
Illia Silin committed
867
868
869
870
                        archiveArtifacts "build/ck_cppcheck.log"
                        cleanWs()
                    }
                }
JD's avatar
JD committed
871
                stage('Clang Format') {
Illia Silin's avatar
Illia Silin committed
872
873
874
875
                    when {
                        beforeAgent true
                        expression { !params.RUN_CPPCHECK.toBoolean() }
                    }
JD's avatar
JD committed
876
877
                    agent{ label rocmnode("nogpu") }
                    environment{
878
                        setup_args = "NO_CK_BUILD"
Illia Silin's avatar
Illia Silin committed
879
880
881
                        execute_cmd = "find .. -not -path \'*.git*\' -iname \'*.h\' \
                                -o -not -path \'*.git*\' -iname \'*.hpp\' \
                                -o -not -path \'*.git*\' -iname \'*.cpp\' \
JD's avatar
JD committed
882
883
884
885
886
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
887
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\'"
JD's avatar
JD committed
888
889
                    }
                    steps{
890
                        buildHipClangJobAndReboot(setup_args:setup_args, setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
891
                        cleanWs()
JD's avatar
JD committed
892
893
894
895
                    }
                }
            }
        }
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
        stage("Run Grouped Conv Large Case Tests")
        {
            parallel
            {
                stage("Run Grouped Conv Large Case Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_GROUPED_CONV_LARGE_CASES_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a")}
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
                                           make -j64 test_grouped_convnd_fwd_large_cases_xdl && \
                                           ./bin/test_grouped_convnd_fwd_large_cases_xdl"""
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
        }
        stage("Run Codegen Tests")
        {
            parallel
            {
                stage("Run Codegen Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CODEGEN_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a")}
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ CXX=/opt/rocm/llvm/bin/clang++ cmake ../codegen && \
                                           make -j64 check"""
                    }
936
937
938
939
940
941
942
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
        }
943
        stage("Run CK_TILE_FMHA Tests")
944
945
946
        {
            parallel
            {
947
                stage("Run CK_TILE_FMHA Tests on gfx90a")
948
949
950
                {
                    when {
                        beforeAgent true
951
                        expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() }
952
953
954
955
956
957
958
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
                                           make -j64 tile_example_fmha_fwd tile_example_fmha_bwd && \
                                           cd ../ &&
959
                                           example/ck_tile/01_fmha/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
960
                    }
961
962
963
964
965
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
966
                stage("Run CK_TILE_FMHA Tests on gfx942")
967
968
969
                {
                    when {
                        beforeAgent true
970
                        expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() }
971
972
973
974
975
976
977
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
                                           make -j64 tile_example_fmha_fwd tile_example_fmha_bwd && \
                                           cd ../ &&
978
                                           example/ck_tile/01_fmha/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
979
                    }
980
981
982
983
984
985
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
        }
        stage("Run CK_TILE_GEMM Tests")
        {
            parallel
            {
                stage("Run CK_TILE_GEMM Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_GEMM_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
1001
                                           make -j64 tile_example_gemm_basic tile_example_gemm_universal && \
1002
1003
                                           cd ../ &&
                                           example/ck_tile/03_gemm/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
1004
                    }
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
                stage("Run CK_TILE_GEMM Tests on gfx942")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_GEMM_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
1020
                                           make -j64 tile_example_gemm_basic tile_example_gemm_universal && \
1021
1022
                                           cd ../ &&
                                           example/ck_tile/03_gemm/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
1023
                    }
1024
1025
1026
1027
1028
1029
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
Paul Fultz II's avatar
Paul Fultz II committed
1030
        }
1031

1032
		stage("Build CK and run Tests")
1033
1034
1035
        {
            parallel
            {
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
                stage("Build CK with RHEL8")
                {
                    when {
                        beforeAgent true
                        expression { params.BUILD_LEGACY_OS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        def docker_name = "${env.CK_DOCKERHUB_PRIVATE}:ck_rhel8_rocm6.3"
                        setup_args = """ -DGPU_TARGETS="gfx942" \
                                         -DCMAKE_CXX_FLAGS=" -O3 " \
                                         -DCK_USE_ALTERNATIVE_PYTHON=/opt/Python-3.8.13/bin/python3.8 """
                        execute_args = " "
1049
                    }
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: " ", no_reboot:true, build_type: 'Release', docker_name: docker_name)
                        cleanWs()
                    }
                }
                stage("Build CK with SLES15")
                {
                    when {
                        beforeAgent true
                        expression { params.BUILD_LEGACY_OS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        def docker_name = "${env.CK_DOCKERHUB_PRIVATE}:ck_sles15_rocm6.3"
                        setup_args = """ -DGPU_TARGETS="gfx942" \
                                         -DCMAKE_CXX_FLAGS=" -O3 " \
                                         -DCK_USE_ALTERNATIVE_PYTHON=/opt/Python-3.8.13/bin/python3.8 """
                        execute_args = " "
1068
                    }
1069
1070
1071
1072
1073
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: " ", no_reboot:true, build_type: 'Release', docker_name: docker_name)
                        cleanWs()
                    }
                }
1074
                stage("Build CK for all gfx9 targets")
1075
1076
1077
                {
                    when {
                        beforeAgent true
1078
                        expression { params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1079
                    }
1080
                    agent{ label rocmnode("gfx90a") }
1081
                    environment{
1082
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install \
1083
                                         -DGPU_TARGETS="gfx908;gfx90a;gfx942" \
1084
                                         -DCMAKE_CXX_FLAGS=" -O3 " """
1085
1086
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
1087
                                           -DGPU_TARGETS="gfx908;gfx90a;gfx942" \
1088
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1089
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1090
1091
                    }
                    steps{
1092
1093
1094
1095
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
                        cleanWs()
                    }
                }
Illia Silin's avatar
Illia Silin committed
1096
                stage("Build CK and run Tests on gfx942")
1097
1098
1099
                {
                    when {
                        beforeAgent true
1100
                        expression { params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1101
1102
1103
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
1104
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx942" -DCMAKE_CXX_FLAGS=" -O3 " """
1105
1106
1107
1108
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx942" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1109
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1110
1111
                    }
                    steps{
1112
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1113
                        cleanWs()
1114
1115
                    }
                }
Illia Silin's avatar
Illia Silin committed
1116
                stage("Build CK and run Tests on gfx90a")
1117
                {
1118
1119
                    when {
                        beforeAgent true
1120
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1121
                    }
1122
                    agent{ label rocmnode("gfx90a") }
1123
                    environment{
1124
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx90a" -DCMAKE_CXX_FLAGS=" -O3 " """
1125
1126
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
1127
                                           -DGPU_TARGETS="gfx90a" \
1128
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1129
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
Illia Silin's avatar
Illia Silin committed
1130
1131
1132
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1133
                        cleanWs()
Illia Silin's avatar
Illia Silin committed
1134
1135
                    }
                }
1136
1137
1138
1139
                stage("Build CK instances for different targets")
                {
                    when {
                        beforeAgent true
1140
                        expression { params.BUILD_INSTANCES_ONLY.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1141
1142
1143
1144
1145
1146
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        execute_args = """ cmake -D CMAKE_PREFIX_PATH=/opt/rocm \
                                           -D CMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -D CMAKE_BUILD_TYPE=Release \
1147
                                           -D GPU_ARCHS="gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102"  \
1148
                                           -D CMAKE_CXX_FLAGS=" -O3 " .. && make -j64 """
1149
                    }
1150
1151
1152
1153
1154
                    steps{
                        buildHipClangJobAndReboot(setup_cmd: "",  build_cmd: "", no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
Illia Silin's avatar
Illia Silin committed
1155
                stage("Build CK and run Tests on gfx1030")
Illia Silin's avatar
Illia Silin committed
1156
1157
1158
                {
                    when {
                        beforeAgent true
1159
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
Illia Silin's avatar
Illia Silin committed
1160
                    }
Illia Silin's avatar
Illia Silin committed
1161
                    agent{ label rocmnode("gfx1030") }
Illia Silin's avatar
Illia Silin committed
1162
                    environment{
1163
1164
1165
1166
1167
1168
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1030" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """ 
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1030" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1169
                    }
1170
1171
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1172
                        cleanWs()
1173
1174
                    }
                }
Illia Silin's avatar
Illia Silin committed
1175
                stage("Build CK and run Tests on gfx1101")
1176
1177
1178
                {
                    when {
                        beforeAgent true
1179
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1180
                    }
Illia Silin's avatar
Illia Silin committed
1181
                    agent{ label rocmnode("gfx1101") }
1182
                    environment{
1183
1184
1185
1186
1187
1188
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1101" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1101" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1189
                    }
1190
                    steps{
1191
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1192
                        cleanWs()
JD's avatar
JD committed
1193
1194
                    }
                }
Illia Silin's avatar
Illia Silin committed
1195
1196
1197
1198
                stage("Build CK and run Tests on gfx1201")
                {
                    when {
                        beforeAgent true
1199
                        expression { params.BUILD_GFX12.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
Illia Silin's avatar
Illia Silin committed
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
                    }
                    agent{ label rocmnode("gfx1201") }
                    environment{
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1201" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1201" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
                        cleanWs()
                    }
                }
1215
1216
            }
        }
1217
1218
1219
1220
        stage("Process Performance Test Results")
        {
            parallel
            {
1221
                stage("Process results"){
1222
1223
                    when {
                        beforeAgent true
1224
                        expression { params.RUN_PERFORMANCE_TESTS.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1225
                    }
1226
1227
                    agent { label 'mici' }
                    steps{
1228
                        process_results()
1229
                        cleanWs()
1230
1231
1232
1233
                    }
                }
            }
        }
JD's avatar
JD committed
1234
    }
1235
}