Jenkinsfile 57.6 KB
Newer Older
JD's avatar
JD committed
1
def rocmnode(name) {
2
    return '(rocmtest || miopen) && (' + name + ')'
JD's avatar
JD committed
3
4
5
6
7
8
9
}

def show_node_info() {
    sh """
        echo "NODE_NAME = \$NODE_NAME"
        lsb_release -sd
        uname -r
10
        cat /sys/module/amdgpu/version
JD's avatar
JD committed
11
12
13
14
        ls /opt/ -la
    """
}

15
16
17
18
19
20
21
22
23
24
25
26
27
28
def nthreads() {
    def nproc = sh(returnStdout: true, script: 'nproc')
    echo "Number of cores: ${nproc}"
    def n = nproc.toInteger()
    if (n > 32){
        n /= 2
    }
    if (n > 64){
        n = 64
    }
    echo "Number of threads used for building: ${n}"
    return n
}

29
def runShell(String command){
30
    def responseCode = sh returnStatus: true, script: "${command} > tmp.txt"
31
32
33
34
    def output = readFile(file: "tmp.txt")
    return (output != "")
}

35
def getBaseDockerImageName(){
36
    def img
37
38
39
40
    if (params.USE_CUSTOM_DOCKER != ""){
        img = "${params.USE_CUSTOM_DOCKER}"
    }
    else{
41
42
        def ROCM_numeric = "${params.ROCMVERSION}" as float
        if ( ROCM_numeric < 6.4 ){
43
            img = "${env.CK_DOCKERHUB}:ck_ub22.04_rocm${params.ROCMVERSION}"
44
45
            }
        else{
46
            img = "${env.CK_DOCKERHUB_PRIVATE}:ck_ub22.04_rocm${params.ROCMVERSION}"
47
            }
48
49
50
51
52
53
54
55
56
        }
    return img
}

def getDockerImageName(){
    def img
    def base_name = getBaseDockerImageName()
    if (params.USE_CUSTOM_DOCKER != ""){
        img = "${params.USE_CUSTOM_DOCKER}"
57
58
    }
    else{
59
       if (params.COMPILER_VERSION == "") {
60
           img = "${base_name}"
61
62
       }
       else{
63
          if (params.COMPILER_COMMIT == ""){
64
             img = "${base_name}_${params.COMPILER_VERSION}"
65
66
67
          }
          else{
             def commit = "${params.COMPILER_COMMIT}"[0..6]
68
             img = "${base_name}_${params.COMPILER_VERSION}_${commit}"
69
          }
70
       }
71
    }
72
73
74
    return img
}

75
def check_host() {
76
77
78
79
    if ("${env.CK_SCCACHE}" != "null"){
        def SCCACHE_SERVER="${env.CK_SCCACHE.split(':')[0]}"
        echo "sccache server: ${SCCACHE_SERVER}"
        sh '''ping -c 1 -p 6379 "${SCCACHE_SERVER}" | echo $? > tmp.txt'''
80
81
82
83
84
85
86
87
88
        def output = readFile(file: "tmp.txt")
        echo "tmp.txt contents: \$output"
        return (output != "0")
    }
    else{
        return 1
    }
}

89
90
def build_compiler(){
    def compiler
91
    compiler = "${params.BUILD_COMPILER}"
92
93
94
    return compiler
}

95
96
def getDockerImage(Map conf=[:]){
    env.DOCKER_BUILDKIT=1
97
    def prefixpath = conf.get("prefixpath", "/opt/rocm")
98
    def no_cache = conf.get("no_cache", false)
99
    def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' --build-arg DISABLE_CACHE='git rev-parse ${params.COMPILER_VERSION}' "
100
101
102
103
104
    if(no_cache)
    {
        dockerArgs = dockerArgs + " --no-cache "
    }
    echo "Docker Args: ${dockerArgs}"
105
106
107
108
109
110
111
112
113
    def image
    if ( params.BUILD_LEGACY_OS && conf.get("docker_name", "") != "" ){
        image = conf.get("docker_name", "")
        echo "Using legacy docker: ${image}"
    }
    else{
        image = getDockerImageName()
        echo "Using default docker: ${image}"
    }
114
115
116
117
118
119
    //Check if image exists 
    def retimage
    try 
    {
        echo "Pulling down image: ${image}"
        retimage = docker.image("${image}")
120
121
122
        withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
            retimage.pull()
        }
123
124
125
126
127
128
129
130
131
132
133
134
135
    }
    catch(Exception ex)
    {
        error "Unable to locate image: ${image}"
    }
    return [retimage, image]
}

def buildDocker(install_prefix){
    show_node_info()
    env.DOCKER_BUILDKIT=1
    checkout scm
    def image_name = getDockerImageName()
136
    def base_image_name = getBaseDockerImageName()
137
    echo "Building Docker for ${image_name}"
138
    def dockerArgs = "--build-arg PREFIX=${install_prefix} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
139
    if(params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
140
141
142
143
        dockerArgs = dockerArgs + " --no-cache --build-arg BASE_DOCKER='${base_image_name}' -f Dockerfile.compiler . "
    }
    else{
        dockerArgs = dockerArgs + " -f Dockerfile . "
144
    }
145
146
    echo "Build Args: ${dockerArgs}"
    try{
147
148
149
        if(params.BUILD_DOCKER){
            //force building the new docker if that parameter is true
            echo "Building image: ${image_name}"
150
            retimage = docker.build("${image_name}", dockerArgs)
151
152
153
            withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
                retimage.push()
            }
154
            sh 'docker images -q -f dangling=true | xargs --no-run-if-empty docker rmi'
155
156
157
158
        }
        else{
            echo "Checking for image: ${image_name}"
            sh "docker manifest inspect --insecure ${image_name}"
159
            echo "Image: ${image_name} found! Skipping building image"
160
        }
161
162
163
164
    }
    catch(Exception ex){
        echo "Unable to locate image: ${image_name}. Building image now"
        retimage = docker.build("${image_name}", dockerArgs + ' .')
165
166
167
        withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
            retimage.push()
        }
168
169
170
    }
}

JD's avatar
JD committed
171
172
def cmake_build(Map conf=[:]){

173
    def compiler = build_compiler()
JD's avatar
JD committed
174
175
    def config_targets = conf.get("config_targets","check")
    def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined " + conf.get("extradebugflags", "")
176
    def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","")
JD's avatar
JD committed
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
    def prefixpath = conf.get("prefixpath","/opt/rocm")
    def setup_args = conf.get("setup_args","")

    if (prefixpath != "/usr/local"){
        setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} "
    }

    def build_type_debug = (conf.get("build_type",'release') == 'debug')

    //cmake_env can overwrite default CXX variables.
    def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","")

    def package_build = (conf.get("package_build","") == "true")

    if (package_build == true) {
        config_targets = "package"
    }

    if(conf.get("build_install","") == "true")
    {
        config_targets = 'install ' + config_targets
198
        setup_args = ' -DBUILD_DEV=On -DCMAKE_INSTALL_PREFIX=../install' + setup_args
JD's avatar
JD committed
199
200
201
    } else{
        setup_args = ' -DBUILD_DEV=On' + setup_args
    }
202
203
204
    if (params.DL_KERNELS){
        setup_args = setup_args + " -DDL_KERNELS=ON "
    }
JD's avatar
JD committed
205
206
207
208
209
210
211
212

    if(build_type_debug){
        setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args
    }else{
        setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args
    }

    def pre_setup_cmd = """
213
            #!/bin/bash
JD's avatar
JD committed
214
215
216
217
218
219
220
221
            echo \$HSA_ENABLE_SDMA
            ulimit -c unlimited
            rm -rf build
            mkdir build
            rm -rf install
            mkdir install
            cd build
        """
222
    def invocation_tag=""
Illia Silin's avatar
Illia Silin committed
223
224
225
    if (setup_args.contains("gfx12")){
        invocation_tag="gfx12"
    }
226
227
228
229
230
231
232
233
234
235
236
237
    if (setup_args.contains("gfx11")){
        invocation_tag="gfx11"
    }
    if (setup_args.contains("gfx10")){
        invocation_tag="gfx10"
    }
    if (setup_args.contains("gfx90")){
        invocation_tag="gfx90"
    }
    if (setup_args.contains("gfx94")){
        invocation_tag="gfx94"
    }
238
239
    echo "invocation tag: ${invocation_tag}"
    def redis_pre_setup_cmd = pre_setup_cmd
240
    if(check_host() && params.USE_SCCACHE && "${env.CK_SCCACHE}" != "null" && "${invocation_tag}" != "") {
241
        redis_pre_setup_cmd = pre_setup_cmd + """
242
243
244
245
246
247
248
249
250
251
252
253
254
            #!/bin/bash
            export ROCM_PATH=/opt/rocm
            export SCCACHE_ENABLED=true
            export SCCACHE_LOG_LEVEL=debug
            export SCCACHE_IDLE_TIMEOUT=14400
            export COMPILERS_HASH_DIR=/tmp/.sccache
            export SCCACHE_BIN=/usr/local/.cargo/bin/sccache
            export SCCACHE_EXTRAFILES=/tmp/.sccache/rocm_compilers_hash_file
            export SCCACHE_REDIS="redis://${env.CK_SCCACHE}"
            echo "connect = ${env.CK_SCCACHE}" >> ../script/redis-cli.conf
            export SCCACHE_C_CUSTOM_CACHE_BUSTER="${invocation_tag}"
            echo \$SCCACHE_C_CUSTOM_CACHE_BUSTER
            stunnel ../script/redis-cli.conf
255
            ../script/sccache_wrapper.sh --enforce_redis
256
        """
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
        try {
            def cmd1 = conf.get("cmd1", """
                    ${redis_pre_setup_cmd}
                """)
            sh cmd1
            setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache " + setup_args
        }
        catch(Exception err){
            echo "could not connect to redis server: ${err.getMessage()}. will not use sccache."
            def cmd2 = conf.get("cmd2", """
                    ${pre_setup_cmd}
                """)
            sh cmd2
        }
    }
    else{
        def cmd3 = conf.get("cmd3",  """
                ${pre_setup_cmd}
            """)
        sh cmd3
277
    }
278

Chao Liu's avatar
Chao Liu committed
279
    // reduce parallelism when compiling, clang uses too much memory
280
    def nt = nthreads()
Paul Fultz II's avatar
Paul Fultz II committed
281
    def cmd
282
283
    def setup_cmd
    def build_cmd
JD's avatar
JD committed
284
    def execute_cmd = conf.get("execute_cmd", "")
Paul Fultz II's avatar
Paul Fultz II committed
285
    if(!setup_args.contains("NO_CK_BUILD")){
286
287
288
289
290
291
292
        if (setup_args.contains("gfx90a") && params.NINJA_BUILD_TRACE){
            echo "running ninja build trace"
            setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake -G Ninja ${setup_args}   .. ")
            build_cmd = conf.get("build_cmd", "${build_envs} ninja -j${nt} ${config_targets}")
        }
        else{
            setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args}   .. ")
293
            build_cmd = conf.get("build_cmd", "${build_envs} make -j${nt} ${config_targets}")
294
        }
Paul Fultz II's avatar
Paul Fultz II committed
295
        cmd = conf.get("cmd", """
JD's avatar
JD committed
296
297
298
299
            ${setup_cmd}
            ${build_cmd}
            ${execute_cmd}
        """)
Paul Fultz II's avatar
Paul Fultz II committed
300
301
302
303
304
305
    }
    else{
        cmd = conf.get("cmd", """
            ${execute_cmd}
        """)
    }
JD's avatar
JD committed
306
307

    echo cmd
308
309

    dir("build"){
310
        //build CK
311
        sh cmd
312
313
        //run tests except when NO_CK_BUILD or BUILD_LEGACY_OS are set
        if(!setup_args.contains("NO_CK_BUILD") && !params.BUILD_LEGACY_OS){
314
315
316
317
318
319
320
321
322
            if (setup_args.contains("gfx90a") && params.NINJA_BUILD_TRACE){
                sh "/ninjatracing/ninjatracing .ninja_log > ck_build_trace.json"
                archiveArtifacts "ck_build_trace.json"
                sh "ninja test"
            }
            else{
                sh "make check"
            }
        }
323
    }
JD's avatar
JD committed
324
325

    // Only archive from master or develop
326
    if (package_build == true && (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "amd-master")) {
JD's avatar
JD committed
327
328
        archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true
    }
329
330
331
332
333
334
335
336
337
    //check the node gpu architecture
    def arch_type = 0
    sh 'rocminfo | tee rocminfo.log'
    if ( runShell('grep -n "gfx90a" rocminfo.log') ){
        arch_type = 1
    }
    else if ( runShell('grep -n "gfx942" rocminfo.log') ) {
        arch_type = 2
    }
338
    if (params.RUN_CK_TILE_FMHA_TESTS){
339
        try{
340
341
342
343
344
345
346
347
348
349
350
351
            archiveArtifacts "perf_fmha_*.log"
            if (arch_type == 1){
                stash includes: "perf_fmha_**_gfx90a.log", name: "perf_fmha_log_gfx90a"
            }
            else if (arch_type == 2){
                stash includes: "perf_fmha_**_gfx942.log", name: "perf_fmha_log_gfx942"
            }
        }
        catch(Exception err){
            echo "could not locate the requested artifacts: ${err.getMessage()}. will skip the stashing."
        }
    }
352
353
354
355
356
357
358
359
360
361
362
363
364
365
    if (params.RUN_CK_TILE_FLEX_ATTENTION_TESTS){
        try{
            archiveArtifacts "perf_tile_flex_attn_*.log"
            if (arch_type == 1){
                stash includes: "perf_tile_flex_attn_**_fp16_gfx90a.log", name: "perf_tile_flex_attn_log_gfx90a"
            }
            else if (arch_type == 2){
                stash includes: "perf_tile_flex_attn_**_fp16_gfx942.log", name: "perf_tile_flex_attn__gfx942"
            }
        }
        catch(Exception err){
            echo "could not locate the requested artifacts: ${err.getMessage()}. will skip the stashing."
        }
    }
366
367
368
369
370
371
372
373
374
    if (params.RUN_CK_TILE_GEMM_TESTS){
        try{
            archiveArtifacts "perf_tile_gemm_*.log"
            if (arch_type == 1){
                stash includes: "perf_tile_gemm_**_fp16_gfx90a.log", name: "perf_tile_gemm_log_gfx90a"
            }
            else if (arch_type == 2){
                stash includes: "perf_tile_gemm_**_fp16_gfx942.log", name: "perf_tile_gemm_log_gfx942"
            }
375
376
377
378
379
        }
        catch(Exception err){
            echo "could not locate the requested artifacts: ${err.getMessage()}. will skip the stashing."
        }
    }
JD's avatar
JD committed
380
381
382
383
384
385
386
387
}

def buildHipClangJob(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

388
389
390
391
392
393
394
395
396
        def image
        if ( params.BUILD_LEGACY_OS  && conf.get("docker_name", "") != "" ){
            image = conf.get("docker_name", "")
            echo "Using legacy docker: ${image}"
        }
        else{
            image = getDockerImageName()
            echo "Using default docker: ${image}"
        }
JD's avatar
JD committed
397
398
399
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
400
        def dockerOpts="-u root --device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
JD's avatar
JD committed
401
        if (conf.get("enforce_xnack_on", false)) {
402
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
JD's avatar
JD committed
403
        }
404
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
405
        if (params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
406
407
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
408
409
410
411
        def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3')
        def render_id = sh(returnStdout: true, script: 'getent group render | cut -d: -f3')
        dockerOpts = dockerOpts + " --group-add=${video_id} --group-add=${render_id} "
        echo "Docker flags: ${dockerOpts}"
JD's avatar
JD committed
412
413
414
415

        def variant = env.STAGE_NAME

        def retimage
416
        (retimage, image) = getDockerImage(conf)
417

418
        gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
JD's avatar
JD committed
419
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
420
                timeout(time: 20, unit: 'HOURS')
JD's avatar
JD committed
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
                {
                    cmake_build(conf)
                }
            }
        }
        return retimage
}

def reboot(){
    build job: 'reboot-slaves', propagate: false , parameters: [string(name: 'server', value: "${env.NODE_NAME}"),]
}

def buildHipClangJobAndReboot(Map conf=[:]){
    try{
        buildHipClangJob(conf)
    }
    catch(e){
        echo "throwing error exception for the stage"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

449
450
451
452
def Build_CK(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
453
        env.DOCKER_BUILDKIT=1
454
455
        checkout scm

456
457
458
459
460
461
462
463
464
465
        def image
        if ( params.BUILD_LEGACY_OS  && conf.get("docker_name", "") != "" ){
            image = conf.get("docker_name", "")
            echo "Using legacy docker: ${image}"
        }
        else{
            image = getDockerImageName()
            echo "Using default docker: ${image}"
        }

466
467
468
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
469
        def dockerOpts="-u root --device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
470
471
472
        if (conf.get("enforce_xnack_on", false)) {
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
        }
473
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
474
        if (params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
475
476
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
477
478
479
        if(params.BUILD_LEGACY_OS){
            dockerOpts = dockerOpts + " --env LD_LIBRARY_PATH='/opt/Python-3.8.13/lib' "
        }
480
481
482
483
        def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3')
        def render_id = sh(returnStdout: true, script: 'getent group render | cut -d: -f3')
        dockerOpts = dockerOpts + " --group-add=${video_id} --group-add=${render_id} "
        echo "Docker flags: ${dockerOpts}"
484
485
486

        def variant = env.STAGE_NAME
        def retimage
487

488
        gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
489
490
491
            try {
                (retimage, image) = getDockerImage(conf)
                withDockerContainer(image: image, args: dockerOpts) {
Illia Silin's avatar
Illia Silin committed
492
                    timeout(time: 2, unit: 'MINUTES'){
493
494
                        sh 'rocminfo | tee rocminfo.log'
                        if ( !runShell('grep -n "gfx" rocminfo.log') ){
495
496
497
498
499
500
501
502
503
504
505
506
507
                            throw new Exception ("GPU not found")
                        }
                        else{
                            echo "GPU is OK"
                        }
                    }
                }
            }
            catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
                echo "The job was cancelled or aborted"
                throw e
            }
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
508
                timeout(time: 20, unit: 'HOURS')
509
                {
Illia Silin's avatar
Illia Silin committed
510
                    //check whether to run performance tests on this node
511
                    def arch_type = 0
512
                    sh 'rocminfo | tee rocminfo.log'
513
514
515
516
517
518
519
520
521
522
523
524
525
526
                    if ( runShell('grep -n "gfx90a" rocminfo.log') ){
                        arch_type = 1
                    }
                    else if ( runShell('grep -n "gfx942" rocminfo.log') ) {
                        arch_type = 2
                    }
                    else if ( runShell('grep -n "gfx1030" rocminfo.log') ) {
                        arch_type = 3
                    }
                    else if ( runShell('grep -n "gfx1101" rocminfo.log') ) {
                        arch_type = 4
                    }
                    else if ( runShell('grep -n "gfx1201" rocminfo.log') ) {
                        arch_type = 5
527
                    }
528
                    cmake_build(conf)
529
                    if ( !params.BUILD_LEGACY_OS && arch_type == 1 ){
530
531
532
533
534
535
                            echo "Run inductor codegen tests"
                            sh """
                                  pip install --verbose .
                                  pytest python/test/test_gen_instances.py
                            """
                    }
536
                    dir("build"){
537
538
539
                        if (params.RUN_FULL_QA && arch_type == 1 ){
                            // build deb packages for all gfx9 targets on gfx90a system and prepare to export
                            echo "Build ckProfiler package"
540
541
542
                            sh 'make -j package'
                            archiveArtifacts artifacts: 'composablekernel-ckprofiler_*.deb'
                            sh 'mv composablekernel-ckprofiler_*.deb ckprofiler_0.2.0_amd64.deb'
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
                            stash includes: "ckprofiler_0.2.0_amd64.deb", name: "ckprofiler_0.2.0_amd64.deb"
                        }
                    }
                    // run performance tests, stash the logs, results will be processed on the master node
					dir("script"){
                        if (params.RUN_PERFORMANCE_TESTS){
                        if (params.RUN_FULL_QA && arch_type == 1){
                            // run full tests on gfx90a
                            echo "Run full performance tests"
                            sh "./run_full_performance_tests.sh 0 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
                            archiveArtifacts "perf_batched_gemm.log"
                            archiveArtifacts "perf_grouped_gemm.log"
                            archiveArtifacts "perf_grouped_conv_fwd.log"
                            archiveArtifacts "perf_grouped_conv_bwd_data.log"
                            archiveArtifacts "perf_grouped_conv_bwd_weight.log"
                            archiveArtifacts "perf_gemm_bilinear.log"
                            archiveArtifacts "perf_reduction.log"
                            archiveArtifacts "perf_splitK_gemm.log"
                            archiveArtifacts "perf_onnx_gemm.log"
                            archiveArtifacts "perf_mixed_gemm.log"
                            stash includes: "perf_**.log", name: "perf_log"
                        }
                        else if ( arch_type == 1 ){
                            // run standard tests on gfx90a
                            echo "Run performance tests"
                            sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_onnx_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
                            stash includes: "perf_**.log", name: "perf_log"
                        }
                        // disable performance tests on gfx1030 for now.
                        //else if ( arch_type == 3){
                            // run basic tests on gfx1030
                        //    echo "Run gemm performance tests"
                        //    sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx10"
                        //    archiveArtifacts "perf_onnx_gemm_gfx10.log"
                        //    stash includes: "perf_onnx_gemm_gfx10.log", name: "perf_log_gfx10"
                        //}
                        else if ( arch_type == 4){
                            // run basic tests on gfx11
                            echo "Run gemm performance tests"
                            sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx11"
                            archiveArtifacts "perf_onnx_gemm_gfx11.log"
                            stash includes: "perf_onnx_gemm_gfx11.log", name: "perf_log_gfx11"
                        }
                        else if ( arch_type == 5 ){
                            // run basic tests on gfx12
                            echo "Run gemm performance tests"
                            sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx12"
                            archiveArtifacts "perf_onnx_gemm_gfx12.log"
                            stash includes: "perf_onnx_gemm_gfx12.log", name: "perf_log_gfx12"
                        }                        
600
                        }
601
                    }
602
603
                    if (params.hipTensor_test && arch_type == 1 ){
                        // build and test hipTensor on gfx90a node
604
605
606
                        sh """#!/bin/bash
                            rm -rf "${params.hipTensor_branch}".zip
                            rm -rf hipTensor-"${params.hipTensor_branch}"
607
                            wget https://github.com/ROCm/hipTensor/archive/refs/heads/"${params.hipTensor_branch}".zip
608
609
610
611
612
613
                            unzip -o "${params.hipTensor_branch}".zip
                        """
                        dir("hipTensor-${params.hipTensor_branch}"){
                            sh """#!/bin/bash
                                mkdir -p build
                                ls -ltr
614
                                CC=hipcc CXX=hipcc cmake -Bbuild . -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install"
615
                                cmake --build build -- -j
616
                                ctest --test-dir build
617
618
619
                            """
                        }
                    }
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
                }
            }
        }
        return retimage
}

def Build_CK_and_Reboot(Map conf=[:]){
    try{
        Build_CK(conf)
    }
    catch(e){
        echo "throwing error exception while building CK"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

642
643
644
def process_results(Map conf=[:]){
    env.HSA_ENABLE_SDMA=0
    checkout scm
645
    def image = getDockerImageName() 
646
647
648
    def prefixpath = "/opt/rocm"

    // Jenkins is complaining about the render group 
649
    def dockerOpts="--cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
650
    if (conf.get("enforce_xnack_on", false)) {
651
        dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
652
653
654
655
656
    }

    def variant = env.STAGE_NAME
    def retimage

657
    gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
658
        try {
659
            (retimage, image) = getDockerImage(conf)
660
661
662
663
664
665
666
667
        }
        catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
            echo "The job was cancelled or aborted"
            throw e
        }
    }

    withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
668
        timeout(time: 15, unit: 'MINUTES'){
669
670
            try{
                dir("script"){
671
                    if (params.RUN_CK_TILE_FMHA_TESTS){
672
                        try{
673
674
                            unstash "perf_fmha_log_gfx942"
                            unstash "perf_fmha_log_gfx90a"
675
676
677
678
679
                        }
                        catch(Exception err){
                            echo "could not locate the FMHA performance logs: ${err.getMessage()}."
                        }
                    }
680
681
682
683
684
685
686
687
688
                    if (params.RUN_CK_TILE_GEMM_TESTS){
                        try{
                            unstash "perf_tile_gemm_log_gfx942"
                            unstash "perf_tile_gemm_log_gfx90a"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM performance logs: ${err.getMessage()}."
                        }
                    }
689
690
                    if (params.RUN_FULL_QA){
                        // unstash perf files to master
691
692
                        unstash "ckprofiler_0.2.0_amd64.deb"
                        sh "sshpass -p ${env.ck_deb_pw} scp -o StrictHostKeyChecking=no ckprofiler_0.2.0_amd64.deb ${env.ck_deb_user}@${env.ck_deb_ip}:/var/www/html/composable_kernel/"
693
694
695
696
697
698
699
700
                        unstash "perf_log"
                        try{
                            unstash "perf_log_gfx11"
                            unstash "perf_log_gfx12"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM gfx11/gfx12 performance logs: ${err.getMessage()}."
                        }
701
                        sh "./process_qa_data.sh"
702
703
704
                    }
                    else{
                        // unstash perf files to master
705
706
707
708
709
710
711
712
                        unstash "perf_log"
                        try{
                            unstash "perf_log_gfx11"
                            unstash "perf_log_gfx12"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM gfx11/gfx12 performance logs: ${err.getMessage()}."
                        }
713
                        sh "./process_perf_data.sh"
714
715
716
717
                    }
                }
            }
            catch(e){
718
                echo "Throwing error exception while processing performance test results"
719
720
721
                echo 'Exception occurred: ' + e.toString()
                throw e
            }
722
723
724
            finally{
                echo "Finished processing performance test results"
            }
725
726
727
728
        }
    }
}

729
//launch develop branch daily at 23:00 UT in FULL_QA mode and at 19:00 UT with latest staging compiler version
730
CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;ROCMVERSION=6.3;RUN_CK_TILE_FMHA_TESTS=true;RUN_CK_TILE_FLEX_ATTENTION_TESTS=true;RUN_CK_TILE_GEMM_TESTS=true
731
                                              0 21 * * * % ROCMVERSION=6.3;hipTensor_test=true;RUN_CODEGEN_TESTS=true
732
733
                                              0 19 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-staging;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true
                                              0 17 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-mainline;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true
734
735
                                              0 15 * * * % BUILD_INSTANCES_ONLY=true;RUN_PERFORMANCE_TESTS=false;USE_SCCACHE=false
                                              0 13 * * * % BUILD_LEGACY_OS=true''' : ""
736

JD's avatar
JD committed
737
738
pipeline {
    agent none
739
740
741
    triggers {
        parameterizedCron(CRON_SETTINGS)
    }
JD's avatar
JD committed
742
743
744
    options {
        parallelsAlwaysFailFast()
    }
745
    parameters {
746
747
        booleanParam(
            name: "BUILD_DOCKER",
748
            defaultValue: false,
749
            description: "Force building docker image (default: false), set to true if docker image needs to be updated.")
750
751
752
        string(
            name: 'USE_CUSTOM_DOCKER',
            defaultValue: '',
Paul Fultz II's avatar
Paul Fultz II committed
753
            description: 'If you want to use a custom docker image, please specify it here (default: leave blank).')
754
755
        string(
            name: 'ROCMVERSION', 
756
757
            defaultValue: '6.3',
            description: 'Specify which ROCM version to use: 6.3 (default).')
758
759
        string(
            name: 'COMPILER_VERSION', 
760
            defaultValue: '', 
761
            description: 'Specify which version of compiler to use: release, amd-staging, amd-mainline, or leave blank (default).')
762
763
        string(
            name: 'COMPILER_COMMIT', 
764
            defaultValue: '', 
765
            description: 'Specify which commit of compiler branch to use: leave blank to use the latest commit (default), or use some specific commit of llvm-project branch.')
766
767
        string(
            name: 'BUILD_COMPILER', 
768
769
            defaultValue: '/opt/rocm/llvm/bin/clang++', 
            description: 'Build CK with /opt/rocm/bin/hipcc, /llvm-project/build/bin/clang++, or with /opt/rocm/llvm/bin/clang++ (default).')
770
771
772
773
        booleanParam(
            name: "RUN_FULL_QA",
            defaultValue: false,
            description: "Select whether to run small set of performance tests (default) or full QA")
774
775
776
777
        booleanParam(
            name: "DL_KERNELS",
            defaultValue: false,
            description: "Select whether to build DL kernels (default: OFF)")
778
779
        booleanParam(
            name: "hipTensor_test",
780
781
            defaultValue: false,
            description: "Use the CK build to verify hipTensor build and tests (default: OFF)")
782
783
        string(
            name: 'hipTensor_branch',
784
785
            defaultValue: 'mainline',
            description: 'Specify which branch of hipTensor to use (default: mainline)')
786
787
788
789
        booleanParam(
            name: "USE_SCCACHE",
            defaultValue: true,
            description: "Use the sccache for building CK (default: ON)")
Illia Silin's avatar
Illia Silin committed
790
791
792
793
        booleanParam(
            name: "RUN_CPPCHECK",
            defaultValue: false,
            description: "Run the cppcheck static analysis (default: OFF)")
794
795
        booleanParam(
            name: "RUN_PERFORMANCE_TESTS",
796
797
            defaultValue: true,
            description: "Run the performance tests (default: ON)")
798
799
800
801
        booleanParam(
            name: "RUN_GROUPED_CONV_LARGE_CASES_TESTS",
            defaultValue: false,
            description: "Run the grouped conv large cases tests (default: OFF)")
802
803
804
805
        booleanParam(
            name: "RUN_CODEGEN_TESTS",
            defaultValue: false,
            description: "Run codegen tests (default: OFF)")
806
        booleanParam(
807
            name: "RUN_CK_TILE_FMHA_TESTS",
808
            defaultValue: false,
809
            description: "Run the ck_tile FMHA tests (default: OFF)")
810
        booleanParam(
Aviral Goel's avatar
Aviral Goel committed
811
            name: "RUN_CK_TILE_FLEX_ATTENTION_TESTS",
Aviral Goel's avatar
Aviral Goel committed
812
            defaultValue: true,
Aviral Goel's avatar
Aviral Goel committed
813
            description: "Run the ck_tile FLEX ATTENTION tests (default: ON)")
814
815
        booleanParam(
            name: "RUN_CK_TILE_GEMM_TESTS",
816
            defaultValue: true,
817
            description: "Run the ck_tile GEMM tests (default: ON)")
818
819
820
821
        booleanParam(
            name: "BUILD_INSTANCES_ONLY",
            defaultValue: false,
            description: "Test building instances for various architectures simultaneously (default: OFF)")
Illia Silin's avatar
Illia Silin committed
822
823
        booleanParam(
            name: "BUILD_GFX12",
824
825
            defaultValue: true,
            description: "Build CK and run tests on gfx12 (default: ON)")
826
827
828
829
        booleanParam(
            name: "NINJA_BUILD_TRACE",
            defaultValue: false,
            description: "Generate a ninja build trace (default: OFF)")
830
831
832
833
        booleanParam(
            name: "BUILD_LEGACY_OS",
            defaultValue: false,
            description: "Try building CK with legacy OS dockers: RHEL8 and SLES15 (default: OFF)")
834
835
836
837
838
839
840
841
    }
    environment{
        dbuser = "${dbuser}"
        dbpassword = "${dbpassword}"
        dbsship = "${dbsship}"
        dbsshport = "${dbsshport}"
        dbsshuser = "${dbsshuser}"
        dbsshpassword = "${dbsshpassword}"
842
        ck_git_creds = "${ck_git_creds}"
843
844
        gerrit_cred="${gerrit_cred}"
        DOCKER_BUILDKIT = "1"
845
    }
JD's avatar
JD committed
846
    stages{
847
848
849
850
851
852
        stage("Build Docker"){
            parallel{
                stage('Docker /opt/rocm'){
                    agent{ label rocmnode("nogpu") }
                    steps{
                        buildDocker('/opt/rocm')
853
                        cleanWs()
854
855
856
857
                    }
                }
            }
        }
JD's avatar
JD committed
858
859
        stage("Static checks") {
            parallel{
Illia Silin's avatar
Illia Silin committed
860
861
862
863
864
865
866
                stage('Clang Format and Cppcheck') {
                    when {
                        beforeAgent true
                        expression { params.RUN_CPPCHECK.toBoolean() }
                    }
                    agent{ label rocmnode("nogpu") }
                    environment{
867
                        setup_args = "NO_CK_BUILD"
Illia Silin's avatar
Illia Silin committed
868
869
870
871
872
873
874
875
876
                        execute_cmd = "find .. -not -path \'*.git*\' -iname \'*.h\' \
                                -o -not -path \'*.git*\' -iname \'*.hpp\' \
                                -o -not -path \'*.git*\' -iname \'*.cpp\' \
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\' && \
877
878
879
880
881
                                /cppcheck/build/bin/cppcheck ../* -v -j \$(nproc) -I ../include -I ../profiler/include -I ../library/include \
                                -D CK_ENABLE_FP64 -D CK_ENABLE_FP32 -D CK_ENABLE_FP16 -D CK_ENABLE_FP8 -D CK_ENABLE_BF16 -D CK_ENABLE_BF8 -D CK_ENABLE_INT8 -D DL_KERNELS \
                                -D __gfx908__ -D __gfx90a__ -D __gfx940__ -D __gfx941__ -D __gfx942__ -D __gfx1030__ -D __gfx1100__ -D __gfx1101__ -D __gfx1102__ \
                                -U __gfx803__ -U __gfx900__ -U __gfx906__ -U CK_EXPERIMENTAL_BIT_INT_EXTENSION_INT4 \
                                --file-filter=*.cpp --force --enable=all --output-file=ck_cppcheck.log"
Illia Silin's avatar
Illia Silin committed
882
883
                    }
                    steps{
884
                        buildHipClangJobAndReboot(setup_args:setup_args, setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
Illia Silin's avatar
Illia Silin committed
885
886
887
888
                        archiveArtifacts "build/ck_cppcheck.log"
                        cleanWs()
                    }
                }
JD's avatar
JD committed
889
                stage('Clang Format') {
Illia Silin's avatar
Illia Silin committed
890
891
892
893
                    when {
                        beforeAgent true
                        expression { !params.RUN_CPPCHECK.toBoolean() }
                    }
JD's avatar
JD committed
894
895
                    agent{ label rocmnode("nogpu") }
                    environment{
896
                        setup_args = "NO_CK_BUILD"
Illia Silin's avatar
Illia Silin committed
897
898
899
                        execute_cmd = "find .. -not -path \'*.git*\' -iname \'*.h\' \
                                -o -not -path \'*.git*\' -iname \'*.hpp\' \
                                -o -not -path \'*.git*\' -iname \'*.cpp\' \
JD's avatar
JD committed
900
901
902
903
904
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
905
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\'"
JD's avatar
JD committed
906
907
                    }
                    steps{
908
                        buildHipClangJobAndReboot(setup_args:setup_args, setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
909
                        cleanWs()
JD's avatar
JD committed
910
911
912
913
                    }
                }
            }
        }
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
        stage("Run Grouped Conv Large Case Tests")
        {
            parallel
            {
                stage("Run Grouped Conv Large Case Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_GROUPED_CONV_LARGE_CASES_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a")}
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
                                           make -j64 test_grouped_convnd_fwd_large_cases_xdl && \
                                           ./bin/test_grouped_convnd_fwd_large_cases_xdl"""
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
        }
        stage("Run Codegen Tests")
        {
            parallel
            {
                stage("Run Codegen Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CODEGEN_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a")}
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ CXX=/opt/rocm/llvm/bin/clang++ cmake ../codegen && \
                                           make -j64 check"""
                    }
954
955
956
957
958
959
960
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
        }
961
        stage("Run CK_TILE_FMHA Tests")
962
963
964
        {
            parallel
            {
965
                stage("Run CK_TILE_FMHA Tests on gfx90a")
966
967
968
                {
                    when {
                        beforeAgent true
969
                        expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() }
970
971
972
973
974
975
976
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
                                           make -j64 tile_example_fmha_fwd tile_example_fmha_bwd && \
                                           cd ../ &&
977
                                           example/ck_tile/01_fmha/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
978
                    }
979
980
981
982
983
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
984
                stage("Run CK_TILE_FMHA Tests on gfx942")
985
986
987
                {
                    when {
                        beforeAgent true
988
                        expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() }
989
990
991
992
993
994
995
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
                                           make -j64 tile_example_fmha_fwd tile_example_fmha_bwd && \
                                           cd ../ &&
996
                                           example/ck_tile/01_fmha/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
997
                    }
998
999
1000
1001
1002
1003
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
1004
        }
Aviral Goel's avatar
Aviral Goel committed
1005
        stage("Run RUN_CK_TILE_FLEX_ATTENTION_TESTS Test")
1006
        {
Aviral Goel's avatar
Aviral Goel committed
1007

1008
            parallel
1009
            {   
Aviral Goel's avatar
Aviral Goel committed
1010
                stage("Run RUN_CK_TILE_FLEX_ATTENTION_TESTS Tests on gfx90a")
1011
1012
1013
                {
                    when {
                        beforeAgent true
1014
                        expression { params.RUN_CK_TILE_FLEX_ATTENTION_TESTS.toBoolean() } 
1015
1016
1017
1018
1019
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
Aviral Goel's avatar
Aviral Goel committed
1020
                                           make -j64 tile_example_flexattn_fwd && \
1021
1022
1023
1024
1025
1026
1027
1028
                                           cd ../ &&
                                           example/ck_tile/18_flexattn/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
                stage("Run RUN_CK_TILE_FLEX_ATTENTION_TESTS Tests on gfx942")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_FLEX_ATTENTION_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
Aviral Goel's avatar
Aviral Goel committed
1039
                                           make -j64 tile_example_flexattn_fwd && \
1040
                                           cd ../ &&
Aviral Goel's avatar
Aviral Goel committed
1041
                                           example/ck_tile/18_flexattn/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
1042
1043
1044
1045
1046
1047
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
1048
1049
            }
        }
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
        stage("Run CK_TILE_GEMM Tests")
        {
            parallel
            {
                stage("Run CK_TILE_GEMM Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_GEMM_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
1064
                                           make -j64 tile_example_gemm_basic tile_example_gemm_universal && \
1065
1066
                                           cd ../ &&
                                           example/ck_tile/03_gemm/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
1067
                    }
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
                stage("Run CK_TILE_GEMM Tests on gfx942")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_GEMM_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
1083
                                           make -j64 tile_example_gemm_basic tile_example_gemm_universal && \
1084
1085
                                           cd ../ &&
                                           example/ck_tile/03_gemm/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
1086
                    }
1087
1088
1089
1090
1091
1092
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
Paul Fultz II's avatar
Paul Fultz II committed
1093
        }
1094

1095
		stage("Build CK and run Tests")
1096
1097
1098
        {
            parallel
            {
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
                stage("Build CK with RHEL8")
                {
                    when {
                        beforeAgent true
                        expression { params.BUILD_LEGACY_OS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        def docker_name = "${env.CK_DOCKERHUB_PRIVATE}:ck_rhel8_rocm6.3"
                        setup_args = """ -DGPU_TARGETS="gfx942" \
                                         -DCMAKE_CXX_FLAGS=" -O3 " \
                                         -DCK_USE_ALTERNATIVE_PYTHON=/opt/Python-3.8.13/bin/python3.8 """
                        execute_args = " "
1112
                    }
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: " ", no_reboot:true, build_type: 'Release', docker_name: docker_name)
                        cleanWs()
                    }
                }
                stage("Build CK with SLES15")
                {
                    when {
                        beforeAgent true
                        expression { params.BUILD_LEGACY_OS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        def docker_name = "${env.CK_DOCKERHUB_PRIVATE}:ck_sles15_rocm6.3"
                        setup_args = """ -DGPU_TARGETS="gfx942" \
                                         -DCMAKE_CXX_FLAGS=" -O3 " \
                                         -DCK_USE_ALTERNATIVE_PYTHON=/opt/Python-3.8.13/bin/python3.8 """
                        execute_args = " "
1131
                    }
1132
1133
1134
1135
1136
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: " ", no_reboot:true, build_type: 'Release', docker_name: docker_name)
                        cleanWs()
                    }
                }
1137
                stage("Build CK for all gfx9 targets")
1138
1139
1140
                {
                    when {
                        beforeAgent true
1141
                        expression { params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1142
                    }
1143
                    agent{ label rocmnode("gfx90a") }
1144
                    environment{
1145
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install \
1146
                                         -DGPU_TARGETS="gfx908;gfx90a;gfx942" \
1147
                                         -DCMAKE_CXX_FLAGS=" -O3 " """
1148
1149
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
1150
                                           -DGPU_TARGETS="gfx908;gfx90a;gfx942" \
1151
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1152
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1153
1154
                    }
                    steps{
1155
1156
1157
1158
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
                        cleanWs()
                    }
                }
Illia Silin's avatar
Illia Silin committed
1159
                stage("Build CK and run Tests on gfx942")
1160
1161
1162
                {
                    when {
                        beforeAgent true
1163
                        expression { params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1164
1165
1166
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
1167
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx942" -DCMAKE_CXX_FLAGS=" -O3 " """
1168
1169
1170
1171
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx942" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1172
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1173
1174
                    }
                    steps{
1175
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1176
                        cleanWs()
1177
1178
                    }
                }
Illia Silin's avatar
Illia Silin committed
1179
                stage("Build CK and run Tests on gfx90a")
1180
                {
1181
1182
                    when {
                        beforeAgent true
1183
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1184
                    }
1185
                    agent{ label rocmnode("gfx90a") }
1186
                    environment{
1187
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx90a" -DCMAKE_CXX_FLAGS=" -O3 " """
1188
1189
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
1190
                                           -DGPU_TARGETS="gfx90a" \
1191
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1192
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
Illia Silin's avatar
Illia Silin committed
1193
1194
1195
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1196
                        cleanWs()
Illia Silin's avatar
Illia Silin committed
1197
1198
                    }
                }
1199
1200
1201
1202
                stage("Build CK instances for different targets")
                {
                    when {
                        beforeAgent true
1203
                        expression { params.BUILD_INSTANCES_ONLY.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1204
1205
1206
1207
1208
1209
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        execute_args = """ cmake -D CMAKE_PREFIX_PATH=/opt/rocm \
                                           -D CMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -D CMAKE_BUILD_TYPE=Release \
1210
                                           -D GPU_ARCHS="gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102"  \
1211
                                           -D CMAKE_CXX_FLAGS=" -O3 " .. && make -j64 """
1212
                    }
1213
1214
1215
1216
1217
                    steps{
                        buildHipClangJobAndReboot(setup_cmd: "",  build_cmd: "", no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
Illia Silin's avatar
Illia Silin committed
1218
                stage("Build CK and run Tests on gfx1030")
Illia Silin's avatar
Illia Silin committed
1219
1220
1221
                {
                    when {
                        beforeAgent true
1222
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
Illia Silin's avatar
Illia Silin committed
1223
                    }
Illia Silin's avatar
Illia Silin committed
1224
                    agent{ label rocmnode("gfx1030") }
Illia Silin's avatar
Illia Silin committed
1225
                    environment{
1226
1227
1228
1229
1230
1231
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1030" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """ 
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1030" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1232
                    }
1233
1234
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1235
                        cleanWs()
1236
1237
                    }
                }
Illia Silin's avatar
Illia Silin committed
1238
                stage("Build CK and run Tests on gfx1101")
1239
1240
1241
                {
                    when {
                        beforeAgent true
1242
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1243
                    }
Illia Silin's avatar
Illia Silin committed
1244
                    agent{ label rocmnode("gfx1101") }
1245
                    environment{
1246
1247
1248
1249
1250
1251
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1101" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1101" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1252
                    }
1253
                    steps{
1254
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1255
                        cleanWs()
JD's avatar
JD committed
1256
1257
                    }
                }
Illia Silin's avatar
Illia Silin committed
1258
1259
1260
1261
                stage("Build CK and run Tests on gfx1201")
                {
                    when {
                        beforeAgent true
1262
                        expression { params.BUILD_GFX12.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
Illia Silin's avatar
Illia Silin committed
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
                    }
                    agent{ label rocmnode("gfx1201") }
                    environment{
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1201" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1201" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
                        cleanWs()
                    }
                }
1278
1279
            }
        }
1280
1281
1282
1283
        stage("Process Performance Test Results")
        {
            parallel
            {
1284
                stage("Process results"){
1285
1286
                    when {
                        beforeAgent true
1287
                        expression { params.RUN_PERFORMANCE_TESTS.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1288
                    }
1289
1290
                    agent { label 'mici' }
                    steps{
1291
                        process_results()
1292
                        cleanWs()
1293
1294
1295
1296
                    }
                }
            }
        }
JD's avatar
JD committed
1297
    }
1298
}