Jenkinsfile 58.1 KB
Newer Older
JD's avatar
JD committed
1
def rocmnode(name) {
2
    return '(rocmtest || miopen) && (' + name + ')'
JD's avatar
JD committed
3
4
5
6
7
8
9
}

def show_node_info() {
    sh """
        echo "NODE_NAME = \$NODE_NAME"
        lsb_release -sd
        uname -r
10
        cat /sys/module/amdgpu/version
JD's avatar
JD committed
11
12
13
14
        ls /opt/ -la
    """
}

15
16
17
18
19
20
21
22
23
24
25
26
27
28
def nthreads() {
    def nproc = sh(returnStdout: true, script: 'nproc')
    echo "Number of cores: ${nproc}"
    def n = nproc.toInteger()
    if (n > 32){
        n /= 2
    }
    if (n > 64){
        n = 64
    }
    echo "Number of threads used for building: ${n}"
    return n
}

29
def runShell(String command){
30
    def responseCode = sh returnStatus: true, script: "${command} > tmp.txt"
31
32
33
34
    def output = readFile(file: "tmp.txt")
    return (output != "")
}

35
def getBaseDockerImageName(){
36
    def img
37
38
39
40
    if (params.USE_CUSTOM_DOCKER != ""){
        img = "${params.USE_CUSTOM_DOCKER}"
    }
    else{
41
42
        def ROCM_numeric = "${params.ROCMVERSION}" as float
        if ( ROCM_numeric < 6.4 ){
43
            img = "${env.CK_DOCKERHUB}:ck_ub22.04_rocm${params.ROCMVERSION}"
44
45
            }
        else{
46
            img = "${env.CK_DOCKERHUB_PRIVATE}:ck_ub22.04_rocm${params.ROCMVERSION}"
47
            }
48
49
50
51
52
53
54
55
56
        }
    return img
}

def getDockerImageName(){
    def img
    def base_name = getBaseDockerImageName()
    if (params.USE_CUSTOM_DOCKER != ""){
        img = "${params.USE_CUSTOM_DOCKER}"
57
58
    }
    else{
59
       if (params.COMPILER_VERSION == "") {
60
           img = "${base_name}"
61
62
       }
       else{
63
          if (params.COMPILER_COMMIT == ""){
64
             img = "${base_name}_${params.COMPILER_VERSION}"
65
66
67
          }
          else{
             def commit = "${params.COMPILER_COMMIT}"[0..6]
68
             img = "${base_name}_${params.COMPILER_VERSION}_${commit}"
69
          }
70
       }
71
    }
72
73
74
    return img
}

75
def check_host() {
76
77
78
79
    if ("${env.CK_SCCACHE}" != "null"){
        def SCCACHE_SERVER="${env.CK_SCCACHE.split(':')[0]}"
        echo "sccache server: ${SCCACHE_SERVER}"
        sh '''ping -c 1 -p 6379 "${SCCACHE_SERVER}" | echo $? > tmp.txt'''
80
81
82
83
84
85
86
87
88
        def output = readFile(file: "tmp.txt")
        echo "tmp.txt contents: \$output"
        return (output != "0")
    }
    else{
        return 1
    }
}

89
90
def build_compiler(){
    def compiler
91
    compiler = "${params.BUILD_COMPILER}"
92
93
94
    return compiler
}

95
96
def getDockerImage(Map conf=[:]){
    env.DOCKER_BUILDKIT=1
97
    def prefixpath = conf.get("prefixpath", "/opt/rocm")
98
    def no_cache = conf.get("no_cache", false)
99
    def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' --build-arg DISABLE_CACHE='git rev-parse ${params.COMPILER_VERSION}' "
100
101
102
103
104
    if(no_cache)
    {
        dockerArgs = dockerArgs + " --no-cache "
    }
    echo "Docker Args: ${dockerArgs}"
105
106
107
108
109
110
111
112
113
    def image
    if ( params.BUILD_LEGACY_OS && conf.get("docker_name", "") != "" ){
        image = conf.get("docker_name", "")
        echo "Using legacy docker: ${image}"
    }
    else{
        image = getDockerImageName()
        echo "Using default docker: ${image}"
    }
114
115
116
117
118
119
    //Check if image exists 
    def retimage
    try 
    {
        echo "Pulling down image: ${image}"
        retimage = docker.image("${image}")
120
121
122
        withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
            retimage.pull()
        }
123
124
125
126
127
128
129
130
131
132
133
134
135
    }
    catch(Exception ex)
    {
        error "Unable to locate image: ${image}"
    }
    return [retimage, image]
}

def buildDocker(install_prefix){
    show_node_info()
    env.DOCKER_BUILDKIT=1
    checkout scm
    def image_name = getDockerImageName()
136
    def base_image_name = getBaseDockerImageName()
137
    echo "Building Docker for ${image_name}"
138
    def dockerArgs = "--build-arg PREFIX=${install_prefix} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
139
    if(params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
140
141
142
143
        dockerArgs = dockerArgs + " --no-cache --build-arg BASE_DOCKER='${base_image_name}' -f Dockerfile.compiler . "
    }
    else{
        dockerArgs = dockerArgs + " -f Dockerfile . "
144
    }
145
146
    echo "Build Args: ${dockerArgs}"
    try{
147
148
149
        if(params.BUILD_DOCKER){
            //force building the new docker if that parameter is true
            echo "Building image: ${image_name}"
150
            retimage = docker.build("${image_name}", dockerArgs)
151
152
153
            withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
                retimage.push()
            }
154
            sh 'docker images -q -f dangling=true | xargs --no-run-if-empty docker rmi'
155
156
157
158
        }
        else{
            echo "Checking for image: ${image_name}"
            sh "docker manifest inspect --insecure ${image_name}"
159
            echo "Image: ${image_name} found! Skipping building image"
160
        }
161
162
163
164
    }
    catch(Exception ex){
        echo "Unable to locate image: ${image_name}. Building image now"
        retimage = docker.build("${image_name}", dockerArgs + ' .')
165
166
167
        withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
            retimage.push()
        }
168
169
170
    }
}

JD's avatar
JD committed
171
172
def cmake_build(Map conf=[:]){

173
    def compiler = build_compiler()
JD's avatar
JD committed
174
175
    def config_targets = conf.get("config_targets","check")
    def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined " + conf.get("extradebugflags", "")
176
    def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","")
JD's avatar
JD committed
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
    def prefixpath = conf.get("prefixpath","/opt/rocm")
    def setup_args = conf.get("setup_args","")

    if (prefixpath != "/usr/local"){
        setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} "
    }

    def build_type_debug = (conf.get("build_type",'release') == 'debug')

    //cmake_env can overwrite default CXX variables.
    def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","")

    def package_build = (conf.get("package_build","") == "true")

    if (package_build == true) {
        config_targets = "package"
    }

    if(conf.get("build_install","") == "true")
    {
        config_targets = 'install ' + config_targets
198
        setup_args = ' -DBUILD_DEV=On -DCMAKE_INSTALL_PREFIX=../install' + setup_args
JD's avatar
JD committed
199
200
201
    } else{
        setup_args = ' -DBUILD_DEV=On' + setup_args
    }
202
203
204
    if (params.DL_KERNELS){
        setup_args = setup_args + " -DDL_KERNELS=ON "
    }
JD's avatar
JD committed
205
206
207
208
209
210
211
212

    if(build_type_debug){
        setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args
    }else{
        setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args
    }

    def pre_setup_cmd = """
213
            #!/bin/bash
JD's avatar
JD committed
214
215
216
217
218
219
220
221
            echo \$HSA_ENABLE_SDMA
            ulimit -c unlimited
            rm -rf build
            mkdir build
            rm -rf install
            mkdir install
            cd build
        """
222
    def invocation_tag=""
Illia Silin's avatar
Illia Silin committed
223
224
225
    if (setup_args.contains("gfx12")){
        invocation_tag="gfx12"
    }
226
227
228
229
230
231
232
233
234
235
236
237
    if (setup_args.contains("gfx11")){
        invocation_tag="gfx11"
    }
    if (setup_args.contains("gfx10")){
        invocation_tag="gfx10"
    }
    if (setup_args.contains("gfx90")){
        invocation_tag="gfx90"
    }
    if (setup_args.contains("gfx94")){
        invocation_tag="gfx94"
    }
238
239
    echo "invocation tag: ${invocation_tag}"
    def redis_pre_setup_cmd = pre_setup_cmd
240
    if(check_host() && params.USE_SCCACHE && "${env.CK_SCCACHE}" != "null" && "${invocation_tag}" != "") {
241
        redis_pre_setup_cmd = pre_setup_cmd + """
242
243
244
245
246
247
248
249
250
251
252
253
254
            #!/bin/bash
            export ROCM_PATH=/opt/rocm
            export SCCACHE_ENABLED=true
            export SCCACHE_LOG_LEVEL=debug
            export SCCACHE_IDLE_TIMEOUT=14400
            export COMPILERS_HASH_DIR=/tmp/.sccache
            export SCCACHE_BIN=/usr/local/.cargo/bin/sccache
            export SCCACHE_EXTRAFILES=/tmp/.sccache/rocm_compilers_hash_file
            export SCCACHE_REDIS="redis://${env.CK_SCCACHE}"
            echo "connect = ${env.CK_SCCACHE}" >> ../script/redis-cli.conf
            export SCCACHE_C_CUSTOM_CACHE_BUSTER="${invocation_tag}"
            echo \$SCCACHE_C_CUSTOM_CACHE_BUSTER
            stunnel ../script/redis-cli.conf
255
            ../script/sccache_wrapper.sh --enforce_redis
256
        """
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
        try {
            def cmd1 = conf.get("cmd1", """
                    ${redis_pre_setup_cmd}
                """)
            sh cmd1
            setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache " + setup_args
        }
        catch(Exception err){
            echo "could not connect to redis server: ${err.getMessage()}. will not use sccache."
            def cmd2 = conf.get("cmd2", """
                    ${pre_setup_cmd}
                """)
            sh cmd2
        }
    }
    else{
        def cmd3 = conf.get("cmd3",  """
                ${pre_setup_cmd}
            """)
        sh cmd3
277
    }
278

Chao Liu's avatar
Chao Liu committed
279
    // reduce parallelism when compiling, clang uses too much memory
280
    def nt = nthreads()
Paul Fultz II's avatar
Paul Fultz II committed
281
    def cmd
282
283
    def setup_cmd
    def build_cmd
JD's avatar
JD committed
284
    def execute_cmd = conf.get("execute_cmd", "")
Paul Fultz II's avatar
Paul Fultz II committed
285
    if(!setup_args.contains("NO_CK_BUILD")){
286
287
288
289
290
291
292
        if (setup_args.contains("gfx90a") && params.NINJA_BUILD_TRACE){
            echo "running ninja build trace"
            setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake -G Ninja ${setup_args}   .. ")
            build_cmd = conf.get("build_cmd", "${build_envs} ninja -j${nt} ${config_targets}")
        }
        else{
            setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args}   .. ")
293
            build_cmd = conf.get("build_cmd", "${build_envs} make -j${nt} ${config_targets}")
294
        }
Paul Fultz II's avatar
Paul Fultz II committed
295
        cmd = conf.get("cmd", """
JD's avatar
JD committed
296
297
298
299
            ${setup_cmd}
            ${build_cmd}
            ${execute_cmd}
        """)
Paul Fultz II's avatar
Paul Fultz II committed
300
301
302
303
304
305
    }
    else{
        cmd = conf.get("cmd", """
            ${execute_cmd}
        """)
    }
JD's avatar
JD committed
306
307

    echo cmd
308
309

    dir("build"){
310
        //build CK
311
        sh cmd
312
313
        //run tests except when NO_CK_BUILD or BUILD_LEGACY_OS are set
        if(!setup_args.contains("NO_CK_BUILD") && !params.BUILD_LEGACY_OS){
314
315
316
317
318
319
320
321
322
            if (setup_args.contains("gfx90a") && params.NINJA_BUILD_TRACE){
                sh "/ninjatracing/ninjatracing .ninja_log > ck_build_trace.json"
                archiveArtifacts "ck_build_trace.json"
                sh "ninja test"
            }
            else{
                sh "make check"
            }
        }
323
    }
JD's avatar
JD committed
324
325

    // Only archive from master or develop
326
    if (package_build == true && (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "amd-master")) {
JD's avatar
JD committed
327
328
        archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true
    }
329
330
331
332
333
334
335
336
337
    //check the node gpu architecture
    def arch_type = 0
    sh 'rocminfo | tee rocminfo.log'
    if ( runShell('grep -n "gfx90a" rocminfo.log') ){
        arch_type = 1
    }
    else if ( runShell('grep -n "gfx942" rocminfo.log') ) {
        arch_type = 2
    }
338
    if (params.RUN_CK_TILE_FMHA_TESTS){
339
        try{
340
341
342
343
344
345
346
347
348
349
350
351
            archiveArtifacts "perf_fmha_*.log"
            if (arch_type == 1){
                stash includes: "perf_fmha_**_gfx90a.log", name: "perf_fmha_log_gfx90a"
            }
            else if (arch_type == 2){
                stash includes: "perf_fmha_**_gfx942.log", name: "perf_fmha_log_gfx942"
            }
        }
        catch(Exception err){
            echo "could not locate the requested artifacts: ${err.getMessage()}. will skip the stashing."
        }
    }
352
353
354
355
356
357
358
    if (params.RUN_CK_TILE_FLEX_ATTENTION_TESTS){
        try{
            archiveArtifacts "perf_tile_flex_attn_*.log"
            if (arch_type == 1){
                stash includes: "perf_tile_flex_attn_**_fp16_gfx90a.log", name: "perf_tile_flex_attn_log_gfx90a"
            }
            else if (arch_type == 2){
359
                stash includes: "perf_tile_flex_attn_**_fp16_gfx942.log", name: "perf_tile_flex_attn_log_gfx942"
360
361
362
363
364
365
            }
        }
        catch(Exception err){
            echo "could not locate the requested artifacts: ${err.getMessage()}. will skip the stashing."
        }
    }
366
367
368
369
370
371
372
373
374
    if (params.RUN_CK_TILE_GEMM_TESTS){
        try{
            archiveArtifacts "perf_tile_gemm_*.log"
            if (arch_type == 1){
                stash includes: "perf_tile_gemm_**_fp16_gfx90a.log", name: "perf_tile_gemm_log_gfx90a"
            }
            else if (arch_type == 2){
                stash includes: "perf_tile_gemm_**_fp16_gfx942.log", name: "perf_tile_gemm_log_gfx942"
            }
375
376
377
378
379
        }
        catch(Exception err){
            echo "could not locate the requested artifacts: ${err.getMessage()}. will skip the stashing."
        }
    }
JD's avatar
JD committed
380
381
382
383
384
385
386
387
}

def buildHipClangJob(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

388
389
390
391
392
393
394
395
396
        def image
        if ( params.BUILD_LEGACY_OS  && conf.get("docker_name", "") != "" ){
            image = conf.get("docker_name", "")
            echo "Using legacy docker: ${image}"
        }
        else{
            image = getDockerImageName()
            echo "Using default docker: ${image}"
        }
JD's avatar
JD committed
397
398
399
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
400
        def dockerOpts="-u root --device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
JD's avatar
JD committed
401
        if (conf.get("enforce_xnack_on", false)) {
402
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
JD's avatar
JD committed
403
        }
404
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
405
        if (params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
406
407
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
408
409
410
411
        def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3')
        def render_id = sh(returnStdout: true, script: 'getent group render | cut -d: -f3')
        dockerOpts = dockerOpts + " --group-add=${video_id} --group-add=${render_id} "
        echo "Docker flags: ${dockerOpts}"
JD's avatar
JD committed
412
413
414
415

        def variant = env.STAGE_NAME

        def retimage
416
        (retimage, image) = getDockerImage(conf)
417

418
        gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
JD's avatar
JD committed
419
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
420
                timeout(time: 20, unit: 'HOURS')
JD's avatar
JD committed
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
                {
                    cmake_build(conf)
                }
            }
        }
        return retimage
}

def reboot(){
    build job: 'reboot-slaves', propagate: false , parameters: [string(name: 'server', value: "${env.NODE_NAME}"),]
}

def buildHipClangJobAndReboot(Map conf=[:]){
    try{
        buildHipClangJob(conf)
    }
    catch(e){
        echo "throwing error exception for the stage"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

449
450
451
452
def Build_CK(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
453
        env.DOCKER_BUILDKIT=1
454
455
        checkout scm

456
457
458
459
460
461
462
463
464
465
        def image
        if ( params.BUILD_LEGACY_OS  && conf.get("docker_name", "") != "" ){
            image = conf.get("docker_name", "")
            echo "Using legacy docker: ${image}"
        }
        else{
            image = getDockerImageName()
            echo "Using default docker: ${image}"
        }

466
467
468
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
469
        def dockerOpts="-u root --device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
470
471
472
        if (conf.get("enforce_xnack_on", false)) {
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
        }
473
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
474
        if (params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
475
476
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
477
478
479
        if(params.BUILD_LEGACY_OS){
            dockerOpts = dockerOpts + " --env LD_LIBRARY_PATH='/opt/Python-3.8.13/lib' "
        }
480
481
482
483
        def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3')
        def render_id = sh(returnStdout: true, script: 'getent group render | cut -d: -f3')
        dockerOpts = dockerOpts + " --group-add=${video_id} --group-add=${render_id} "
        echo "Docker flags: ${dockerOpts}"
484
485
486

        def variant = env.STAGE_NAME
        def retimage
487

488
        gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
489
490
491
            try {
                (retimage, image) = getDockerImage(conf)
                withDockerContainer(image: image, args: dockerOpts) {
Illia Silin's avatar
Illia Silin committed
492
                    timeout(time: 2, unit: 'MINUTES'){
493
494
                        sh 'rocminfo | tee rocminfo.log'
                        if ( !runShell('grep -n "gfx" rocminfo.log') ){
495
496
497
498
499
500
501
502
503
504
505
506
507
                            throw new Exception ("GPU not found")
                        }
                        else{
                            echo "GPU is OK"
                        }
                    }
                }
            }
            catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
                echo "The job was cancelled or aborted"
                throw e
            }
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
508
                timeout(time: 20, unit: 'HOURS')
509
                {
Illia Silin's avatar
Illia Silin committed
510
                    //check whether to run performance tests on this node
511
                    def arch_type = 0
512
                    sh 'rocminfo | tee rocminfo.log'
513
514
515
516
517
518
519
520
521
522
523
524
525
526
                    if ( runShell('grep -n "gfx90a" rocminfo.log') ){
                        arch_type = 1
                    }
                    else if ( runShell('grep -n "gfx942" rocminfo.log') ) {
                        arch_type = 2
                    }
                    else if ( runShell('grep -n "gfx1030" rocminfo.log') ) {
                        arch_type = 3
                    }
                    else if ( runShell('grep -n "gfx1101" rocminfo.log') ) {
                        arch_type = 4
                    }
                    else if ( runShell('grep -n "gfx1201" rocminfo.log') ) {
                        arch_type = 5
527
                    }
528
                    cmake_build(conf)
529
                    if ( !params.BUILD_LEGACY_OS && arch_type == 1 ){
530
531
532
533
534
535
                            echo "Run inductor codegen tests"
                            sh """
                                  pip install --verbose .
                                  pytest python/test/test_gen_instances.py
                            """
                    }
536
                    dir("build"){
537
538
539
                        if (params.RUN_FULL_QA && arch_type == 1 ){
                            // build deb packages for all gfx9 targets on gfx90a system and prepare to export
                            echo "Build ckProfiler package"
540
541
542
                            sh 'make -j package'
                            archiveArtifacts artifacts: 'composablekernel-ckprofiler_*.deb'
                            sh 'mv composablekernel-ckprofiler_*.deb ckprofiler_0.2.0_amd64.deb'
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
                            stash includes: "ckprofiler_0.2.0_amd64.deb", name: "ckprofiler_0.2.0_amd64.deb"
                        }
                    }
                    // run performance tests, stash the logs, results will be processed on the master node
					dir("script"){
                        if (params.RUN_PERFORMANCE_TESTS){
                        if (params.RUN_FULL_QA && arch_type == 1){
                            // run full tests on gfx90a
                            echo "Run full performance tests"
                            sh "./run_full_performance_tests.sh 0 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
                            archiveArtifacts "perf_batched_gemm.log"
                            archiveArtifacts "perf_grouped_gemm.log"
                            archiveArtifacts "perf_grouped_conv_fwd.log"
                            archiveArtifacts "perf_grouped_conv_bwd_data.log"
                            archiveArtifacts "perf_grouped_conv_bwd_weight.log"
                            archiveArtifacts "perf_gemm_bilinear.log"
                            archiveArtifacts "perf_reduction.log"
                            archiveArtifacts "perf_splitK_gemm.log"
                            archiveArtifacts "perf_onnx_gemm.log"
                            archiveArtifacts "perf_mixed_gemm.log"
                            stash includes: "perf_**.log", name: "perf_log"
                        }
                        else if ( arch_type == 1 ){
                            // run standard tests on gfx90a
                            echo "Run performance tests"
                            sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_onnx_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
                            stash includes: "perf_**.log", name: "perf_log"
                        }
                        // disable performance tests on gfx1030 for now.
                        //else if ( arch_type == 3){
                            // run basic tests on gfx1030
                        //    echo "Run gemm performance tests"
                        //    sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx10"
                        //    archiveArtifacts "perf_onnx_gemm_gfx10.log"
                        //    stash includes: "perf_onnx_gemm_gfx10.log", name: "perf_log_gfx10"
                        //}
                        else if ( arch_type == 4){
                            // run basic tests on gfx11
                            echo "Run gemm performance tests"
                            sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx11"
                            archiveArtifacts "perf_onnx_gemm_gfx11.log"
                            stash includes: "perf_onnx_gemm_gfx11.log", name: "perf_log_gfx11"
                        }
                        else if ( arch_type == 5 ){
                            // run basic tests on gfx12
                            echo "Run gemm performance tests"
                            sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx12"
                            archiveArtifacts "perf_onnx_gemm_gfx12.log"
                            stash includes: "perf_onnx_gemm_gfx12.log", name: "perf_log_gfx12"
                        }                        
600
                        }
601
                    }
602
603
                    if (params.hipTensor_test && arch_type == 1 ){
                        // build and test hipTensor on gfx90a node
604
605
606
                        sh """#!/bin/bash
                            rm -rf "${params.hipTensor_branch}".zip
                            rm -rf hipTensor-"${params.hipTensor_branch}"
607
                            wget https://github.com/ROCm/hipTensor/archive/refs/heads/"${params.hipTensor_branch}".zip
608
609
610
611
612
613
                            unzip -o "${params.hipTensor_branch}".zip
                        """
                        dir("hipTensor-${params.hipTensor_branch}"){
                            sh """#!/bin/bash
                                mkdir -p build
                                ls -ltr
614
                                CC=hipcc CXX=hipcc cmake -Bbuild . -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install"
615
                                cmake --build build -- -j
616
                                ctest --test-dir build
617
618
619
                            """
                        }
                    }
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
                }
            }
        }
        return retimage
}

def Build_CK_and_Reboot(Map conf=[:]){
    try{
        Build_CK(conf)
    }
    catch(e){
        echo "throwing error exception while building CK"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

642
643
644
def process_results(Map conf=[:]){
    env.HSA_ENABLE_SDMA=0
    checkout scm
645
    def image = getDockerImageName() 
646
647
648
    def prefixpath = "/opt/rocm"

    // Jenkins is complaining about the render group 
649
    def dockerOpts="--cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
650
    if (conf.get("enforce_xnack_on", false)) {
651
        dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
652
653
654
655
656
    }

    def variant = env.STAGE_NAME
    def retimage

657
    gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
658
        try {
659
            (retimage, image) = getDockerImage(conf)
660
661
662
663
664
665
666
667
        }
        catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
            echo "The job was cancelled or aborted"
            throw e
        }
    }

    withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
668
        timeout(time: 15, unit: 'MINUTES'){
669
670
            try{
                dir("script"){
671
                    if (params.RUN_CK_TILE_FMHA_TESTS){
672
                        try{
673
674
                            unstash "perf_fmha_log_gfx942"
                            unstash "perf_fmha_log_gfx90a"
675
676
677
678
679
                        }
                        catch(Exception err){
                            echo "could not locate the FMHA performance logs: ${err.getMessage()}."
                        }
                    }
680
681
682
683
684
685
686
687
688
                    if (params.RUN_CK_TILE_FLEX_ATTENTION_TESTS){
                        try{
                            unstash "perf_tile_flex_attn_log_gfx90a"
                            unstash "perf_tile_flex_attn_log_gfx942"
                        }
                        catch(Exception err){
                            echo "could not locate the FMHA performance logs: ${err.getMessage()}."
                        }
                    }
689
690
691
692
693
694
695
696
697
                    if (params.RUN_CK_TILE_GEMM_TESTS){
                        try{
                            unstash "perf_tile_gemm_log_gfx942"
                            unstash "perf_tile_gemm_log_gfx90a"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM performance logs: ${err.getMessage()}."
                        }
                    }
698
699
                    if (params.RUN_FULL_QA){
                        // unstash perf files to master
700
701
                        unstash "ckprofiler_0.2.0_amd64.deb"
                        sh "sshpass -p ${env.ck_deb_pw} scp -o StrictHostKeyChecking=no ckprofiler_0.2.0_amd64.deb ${env.ck_deb_user}@${env.ck_deb_ip}:/var/www/html/composable_kernel/"
702
703
704
705
706
707
708
709
                        unstash "perf_log"
                        try{
                            unstash "perf_log_gfx11"
                            unstash "perf_log_gfx12"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM gfx11/gfx12 performance logs: ${err.getMessage()}."
                        }
710
                        sh "./process_qa_data.sh"
711
712
713
                    }
                    else{
                        // unstash perf files to master
714
715
716
717
718
719
720
721
                        unstash "perf_log"
                        try{
                            unstash "perf_log_gfx11"
                            unstash "perf_log_gfx12"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM gfx11/gfx12 performance logs: ${err.getMessage()}."
                        }
722
                        sh "./process_perf_data.sh"
723
724
725
726
                    }
                }
            }
            catch(e){
727
                echo "Throwing error exception while processing performance test results"
728
729
730
                echo 'Exception occurred: ' + e.toString()
                throw e
            }
731
732
733
            finally{
                echo "Finished processing performance test results"
            }
734
735
736
737
        }
    }
}

738
//launch develop branch daily at 23:00 UT in FULL_QA mode and at 19:00 UT with latest staging compiler version
739
CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;ROCMVERSION=6.3;RUN_CK_TILE_FMHA_TESTS=true;RUN_CK_TILE_FLEX_ATTENTION_TESTS=true;RUN_CK_TILE_GEMM_TESTS=true
740
                                              0 21 * * * % ROCMVERSION=6.3;hipTensor_test=true;RUN_CODEGEN_TESTS=true
741
742
                                              0 19 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-staging;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true
                                              0 17 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-mainline;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true
743
744
                                              0 15 * * * % BUILD_INSTANCES_ONLY=true;RUN_PERFORMANCE_TESTS=false;USE_SCCACHE=false
                                              0 13 * * * % BUILD_LEGACY_OS=true''' : ""
745

JD's avatar
JD committed
746
747
pipeline {
    agent none
748
749
750
    triggers {
        parameterizedCron(CRON_SETTINGS)
    }
JD's avatar
JD committed
751
752
753
    options {
        parallelsAlwaysFailFast()
    }
754
    parameters {
755
756
        booleanParam(
            name: "BUILD_DOCKER",
757
            defaultValue: false,
758
            description: "Force building docker image (default: false), set to true if docker image needs to be updated.")
759
760
761
        string(
            name: 'USE_CUSTOM_DOCKER',
            defaultValue: '',
Paul Fultz II's avatar
Paul Fultz II committed
762
            description: 'If you want to use a custom docker image, please specify it here (default: leave blank).')
763
764
        string(
            name: 'ROCMVERSION', 
765
766
            defaultValue: '6.3',
            description: 'Specify which ROCM version to use: 6.3 (default).')
767
768
        string(
            name: 'COMPILER_VERSION', 
769
            defaultValue: '', 
770
            description: 'Specify which version of compiler to use: release, amd-staging, amd-mainline, or leave blank (default).')
771
772
        string(
            name: 'COMPILER_COMMIT', 
773
            defaultValue: '', 
774
            description: 'Specify which commit of compiler branch to use: leave blank to use the latest commit (default), or use some specific commit of llvm-project branch.')
775
776
        string(
            name: 'BUILD_COMPILER', 
777
778
            defaultValue: '/opt/rocm/llvm/bin/clang++', 
            description: 'Build CK with /opt/rocm/bin/hipcc, /llvm-project/build/bin/clang++, or with /opt/rocm/llvm/bin/clang++ (default).')
779
780
781
782
        booleanParam(
            name: "RUN_FULL_QA",
            defaultValue: false,
            description: "Select whether to run small set of performance tests (default) or full QA")
783
784
785
786
        booleanParam(
            name: "DL_KERNELS",
            defaultValue: false,
            description: "Select whether to build DL kernels (default: OFF)")
787
788
        booleanParam(
            name: "hipTensor_test",
789
790
            defaultValue: false,
            description: "Use the CK build to verify hipTensor build and tests (default: OFF)")
791
792
        string(
            name: 'hipTensor_branch',
793
794
            defaultValue: 'mainline',
            description: 'Specify which branch of hipTensor to use (default: mainline)')
795
796
797
798
        booleanParam(
            name: "USE_SCCACHE",
            defaultValue: true,
            description: "Use the sccache for building CK (default: ON)")
Illia Silin's avatar
Illia Silin committed
799
800
801
802
        booleanParam(
            name: "RUN_CPPCHECK",
            defaultValue: false,
            description: "Run the cppcheck static analysis (default: OFF)")
803
804
        booleanParam(
            name: "RUN_PERFORMANCE_TESTS",
805
806
            defaultValue: true,
            description: "Run the performance tests (default: ON)")
807
808
809
810
        booleanParam(
            name: "RUN_GROUPED_CONV_LARGE_CASES_TESTS",
            defaultValue: false,
            description: "Run the grouped conv large cases tests (default: OFF)")
811
812
813
814
        booleanParam(
            name: "RUN_CODEGEN_TESTS",
            defaultValue: false,
            description: "Run codegen tests (default: OFF)")
815
        booleanParam(
816
            name: "RUN_CK_TILE_FMHA_TESTS",
817
            defaultValue: false,
818
            description: "Run the ck_tile FMHA tests (default: OFF)")
819
        booleanParam(
Aviral Goel's avatar
Aviral Goel committed
820
            name: "RUN_CK_TILE_FLEX_ATTENTION_TESTS",
Aviral Goel's avatar
Aviral Goel committed
821
            defaultValue: true,
Aviral Goel's avatar
Aviral Goel committed
822
            description: "Run the ck_tile FLEX ATTENTION tests (default: ON)")
823
824
        booleanParam(
            name: "RUN_CK_TILE_GEMM_TESTS",
825
            defaultValue: true,
826
            description: "Run the ck_tile GEMM tests (default: ON)")
827
828
829
830
        booleanParam(
            name: "BUILD_INSTANCES_ONLY",
            defaultValue: false,
            description: "Test building instances for various architectures simultaneously (default: OFF)")
Illia Silin's avatar
Illia Silin committed
831
832
        booleanParam(
            name: "BUILD_GFX12",
833
834
            defaultValue: true,
            description: "Build CK and run tests on gfx12 (default: ON)")
835
836
837
838
        booleanParam(
            name: "NINJA_BUILD_TRACE",
            defaultValue: false,
            description: "Generate a ninja build trace (default: OFF)")
839
840
841
842
        booleanParam(
            name: "BUILD_LEGACY_OS",
            defaultValue: false,
            description: "Try building CK with legacy OS dockers: RHEL8 and SLES15 (default: OFF)")
843
844
845
846
847
848
849
850
    }
    environment{
        dbuser = "${dbuser}"
        dbpassword = "${dbpassword}"
        dbsship = "${dbsship}"
        dbsshport = "${dbsshport}"
        dbsshuser = "${dbsshuser}"
        dbsshpassword = "${dbsshpassword}"
851
        ck_git_creds = "${ck_git_creds}"
852
853
        gerrit_cred="${gerrit_cred}"
        DOCKER_BUILDKIT = "1"
854
    }
JD's avatar
JD committed
855
    stages{
856
857
858
859
860
861
        stage("Build Docker"){
            parallel{
                stage('Docker /opt/rocm'){
                    agent{ label rocmnode("nogpu") }
                    steps{
                        buildDocker('/opt/rocm')
862
                        cleanWs()
863
864
865
866
                    }
                }
            }
        }
JD's avatar
JD committed
867
868
        stage("Static checks") {
            parallel{
Illia Silin's avatar
Illia Silin committed
869
870
871
872
873
874
875
                stage('Clang Format and Cppcheck') {
                    when {
                        beforeAgent true
                        expression { params.RUN_CPPCHECK.toBoolean() }
                    }
                    agent{ label rocmnode("nogpu") }
                    environment{
876
                        setup_args = "NO_CK_BUILD"
Illia Silin's avatar
Illia Silin committed
877
878
879
880
881
882
883
884
885
                        execute_cmd = "find .. -not -path \'*.git*\' -iname \'*.h\' \
                                -o -not -path \'*.git*\' -iname \'*.hpp\' \
                                -o -not -path \'*.git*\' -iname \'*.cpp\' \
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\' && \
886
887
888
889
890
                                /cppcheck/build/bin/cppcheck ../* -v -j \$(nproc) -I ../include -I ../profiler/include -I ../library/include \
                                -D CK_ENABLE_FP64 -D CK_ENABLE_FP32 -D CK_ENABLE_FP16 -D CK_ENABLE_FP8 -D CK_ENABLE_BF16 -D CK_ENABLE_BF8 -D CK_ENABLE_INT8 -D DL_KERNELS \
                                -D __gfx908__ -D __gfx90a__ -D __gfx940__ -D __gfx941__ -D __gfx942__ -D __gfx1030__ -D __gfx1100__ -D __gfx1101__ -D __gfx1102__ \
                                -U __gfx803__ -U __gfx900__ -U __gfx906__ -U CK_EXPERIMENTAL_BIT_INT_EXTENSION_INT4 \
                                --file-filter=*.cpp --force --enable=all --output-file=ck_cppcheck.log"
Illia Silin's avatar
Illia Silin committed
891
892
                    }
                    steps{
893
                        buildHipClangJobAndReboot(setup_args:setup_args, setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
Illia Silin's avatar
Illia Silin committed
894
895
896
897
                        archiveArtifacts "build/ck_cppcheck.log"
                        cleanWs()
                    }
                }
JD's avatar
JD committed
898
                stage('Clang Format') {
Illia Silin's avatar
Illia Silin committed
899
900
901
902
                    when {
                        beforeAgent true
                        expression { !params.RUN_CPPCHECK.toBoolean() }
                    }
JD's avatar
JD committed
903
904
                    agent{ label rocmnode("nogpu") }
                    environment{
905
                        setup_args = "NO_CK_BUILD"
Illia Silin's avatar
Illia Silin committed
906
907
908
                        execute_cmd = "find .. -not -path \'*.git*\' -iname \'*.h\' \
                                -o -not -path \'*.git*\' -iname \'*.hpp\' \
                                -o -not -path \'*.git*\' -iname \'*.cpp\' \
JD's avatar
JD committed
909
910
911
912
913
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
914
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\'"
JD's avatar
JD committed
915
916
                    }
                    steps{
917
                        buildHipClangJobAndReboot(setup_args:setup_args, setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
918
                        cleanWs()
JD's avatar
JD committed
919
920
921
922
                    }
                }
            }
        }
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
        stage("Run Grouped Conv Large Case Tests")
        {
            parallel
            {
                stage("Run Grouped Conv Large Case Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_GROUPED_CONV_LARGE_CASES_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a")}
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
                                           make -j64 test_grouped_convnd_fwd_large_cases_xdl && \
                                           ./bin/test_grouped_convnd_fwd_large_cases_xdl"""
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
        }
        stage("Run Codegen Tests")
        {
            parallel
            {
                stage("Run Codegen Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CODEGEN_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a")}
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ CXX=/opt/rocm/llvm/bin/clang++ cmake ../codegen && \
                                           make -j64 check"""
                    }
963
964
965
966
967
968
969
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
        }
970
        stage("Run CK_TILE_FMHA Tests")
971
972
973
        {
            parallel
            {
974
                stage("Run CK_TILE_FMHA Tests on gfx90a")
975
976
977
                {
                    when {
                        beforeAgent true
978
                        expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() }
979
980
981
982
983
984
985
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
                                           make -j64 tile_example_fmha_fwd tile_example_fmha_bwd && \
                                           cd ../ &&
986
                                           example/ck_tile/01_fmha/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
987
                    }
988
989
990
991
992
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
993
                stage("Run CK_TILE_FMHA Tests on gfx942")
994
995
996
                {
                    when {
                        beforeAgent true
997
                        expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() }
998
999
1000
1001
1002
1003
1004
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
                                           make -j64 tile_example_fmha_fwd tile_example_fmha_bwd && \
                                           cd ../ &&
1005
                                           example/ck_tile/01_fmha/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
1006
                    }
1007
1008
1009
1010
1011
1012
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
1013
        }
Aviral Goel's avatar
Aviral Goel committed
1014
        stage("Run RUN_CK_TILE_FLEX_ATTENTION_TESTS Test")
1015
        {
Aviral Goel's avatar
Aviral Goel committed
1016

1017
            parallel
1018
            {   
Aviral Goel's avatar
Aviral Goel committed
1019
                stage("Run RUN_CK_TILE_FLEX_ATTENTION_TESTS Tests on gfx90a")
1020
1021
1022
                {
                    when {
                        beforeAgent true
1023
                        expression { params.RUN_CK_TILE_FLEX_ATTENTION_TESTS.toBoolean() } 
1024
1025
1026
1027
1028
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
Aviral Goel's avatar
Aviral Goel committed
1029
                                           make -j64 tile_example_flexattn_fwd && \
1030
1031
1032
1033
1034
1035
1036
1037
                                           cd ../ &&
                                           example/ck_tile/18_flexattn/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
                stage("Run RUN_CK_TILE_FLEX_ATTENTION_TESTS Tests on gfx942")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_FLEX_ATTENTION_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
Aviral Goel's avatar
Aviral Goel committed
1048
                                           make -j64 tile_example_flexattn_fwd && \
1049
                                           cd ../ &&
Aviral Goel's avatar
Aviral Goel committed
1050
                                           example/ck_tile/18_flexattn/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
1051
1052
1053
1054
1055
1056
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
1057
1058
            }
        }
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
        stage("Run CK_TILE_GEMM Tests")
        {
            parallel
            {
                stage("Run CK_TILE_GEMM Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_GEMM_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
1073
                                           make -j64 tile_example_gemm_basic tile_example_gemm_universal && \
1074
1075
                                           cd ../ &&
                                           example/ck_tile/03_gemm/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
1076
                    }
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
                stage("Run CK_TILE_GEMM Tests on gfx942")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_GEMM_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
1092
                                           make -j64 tile_example_gemm_basic tile_example_gemm_universal && \
1093
1094
                                           cd ../ &&
                                           example/ck_tile/03_gemm/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
1095
                    }
1096
1097
1098
1099
1100
1101
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
Paul Fultz II's avatar
Paul Fultz II committed
1102
        }
1103

1104
		stage("Build CK and run Tests")
1105
1106
1107
        {
            parallel
            {
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
                stage("Build CK with RHEL8")
                {
                    when {
                        beforeAgent true
                        expression { params.BUILD_LEGACY_OS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        def docker_name = "${env.CK_DOCKERHUB_PRIVATE}:ck_rhel8_rocm6.3"
                        setup_args = """ -DGPU_TARGETS="gfx942" \
                                         -DCMAKE_CXX_FLAGS=" -O3 " \
                                         -DCK_USE_ALTERNATIVE_PYTHON=/opt/Python-3.8.13/bin/python3.8 """
                        execute_args = " "
1121
                    }
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: " ", no_reboot:true, build_type: 'Release', docker_name: docker_name)
                        cleanWs()
                    }
                }
                stage("Build CK with SLES15")
                {
                    when {
                        beforeAgent true
                        expression { params.BUILD_LEGACY_OS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        def docker_name = "${env.CK_DOCKERHUB_PRIVATE}:ck_sles15_rocm6.3"
                        setup_args = """ -DGPU_TARGETS="gfx942" \
                                         -DCMAKE_CXX_FLAGS=" -O3 " \
                                         -DCK_USE_ALTERNATIVE_PYTHON=/opt/Python-3.8.13/bin/python3.8 """
                        execute_args = " "
1140
                    }
1141
1142
1143
1144
1145
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: " ", no_reboot:true, build_type: 'Release', docker_name: docker_name)
                        cleanWs()
                    }
                }
1146
                stage("Build CK for all gfx9 targets")
1147
1148
1149
                {
                    when {
                        beforeAgent true
1150
                        expression { params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1151
                    }
1152
                    agent{ label rocmnode("gfx90a") }
1153
                    environment{
1154
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install \
1155
                                         -DGPU_TARGETS="gfx908;gfx90a;gfx942" \
1156
                                         -DCMAKE_CXX_FLAGS=" -O3 " """
1157
1158
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
1159
                                           -DGPU_TARGETS="gfx908;gfx90a;gfx942" \
1160
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1161
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1162
1163
                    }
                    steps{
1164
1165
1166
1167
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
                        cleanWs()
                    }
                }
Illia Silin's avatar
Illia Silin committed
1168
                stage("Build CK and run Tests on gfx942")
1169
1170
1171
                {
                    when {
                        beforeAgent true
1172
                        expression { params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1173
1174
1175
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
1176
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx942" -DCMAKE_CXX_FLAGS=" -O3 " """
1177
1178
1179
1180
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx942" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1181
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1182
1183
                    }
                    steps{
1184
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1185
                        cleanWs()
1186
1187
                    }
                }
Illia Silin's avatar
Illia Silin committed
1188
                stage("Build CK and run Tests on gfx90a")
1189
                {
1190
1191
                    when {
                        beforeAgent true
1192
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1193
                    }
1194
                    agent{ label rocmnode("gfx90a") }
1195
                    environment{
1196
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx90a" -DCMAKE_CXX_FLAGS=" -O3 " """
1197
1198
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
1199
                                           -DGPU_TARGETS="gfx90a" \
1200
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1201
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
Illia Silin's avatar
Illia Silin committed
1202
1203
1204
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1205
                        cleanWs()
Illia Silin's avatar
Illia Silin committed
1206
1207
                    }
                }
1208
1209
1210
1211
                stage("Build CK instances for different targets")
                {
                    when {
                        beforeAgent true
1212
                        expression { params.BUILD_INSTANCES_ONLY.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1213
1214
1215
1216
1217
1218
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        execute_args = """ cmake -D CMAKE_PREFIX_PATH=/opt/rocm \
                                           -D CMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -D CMAKE_BUILD_TYPE=Release \
1219
                                           -D GPU_ARCHS="gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102"  \
1220
                                           -D CMAKE_CXX_FLAGS=" -O3 " .. && make -j64 """
1221
                    }
1222
1223
1224
1225
1226
                    steps{
                        buildHipClangJobAndReboot(setup_cmd: "",  build_cmd: "", no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
Illia Silin's avatar
Illia Silin committed
1227
                stage("Build CK and run Tests on gfx1030")
Illia Silin's avatar
Illia Silin committed
1228
1229
1230
                {
                    when {
                        beforeAgent true
1231
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
Illia Silin's avatar
Illia Silin committed
1232
                    }
Illia Silin's avatar
Illia Silin committed
1233
                    agent{ label rocmnode("gfx1030") }
Illia Silin's avatar
Illia Silin committed
1234
                    environment{
1235
1236
1237
1238
1239
1240
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1030" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """ 
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1030" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1241
                    }
1242
1243
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1244
                        cleanWs()
1245
1246
                    }
                }
Illia Silin's avatar
Illia Silin committed
1247
                stage("Build CK and run Tests on gfx1101")
1248
1249
1250
                {
                    when {
                        beforeAgent true
1251
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1252
                    }
Illia Silin's avatar
Illia Silin committed
1253
                    agent{ label rocmnode("gfx1101") }
1254
                    environment{
1255
1256
1257
1258
1259
1260
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1101" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1101" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1261
                    }
1262
                    steps{
1263
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1264
                        cleanWs()
JD's avatar
JD committed
1265
1266
                    }
                }
Illia Silin's avatar
Illia Silin committed
1267
1268
1269
1270
                stage("Build CK and run Tests on gfx1201")
                {
                    when {
                        beforeAgent true
1271
                        expression { params.BUILD_GFX12.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
Illia Silin's avatar
Illia Silin committed
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
                    }
                    agent{ label rocmnode("gfx1201") }
                    environment{
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1201" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1201" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
                        cleanWs()
                    }
                }
1287
1288
            }
        }
1289
1290
1291
1292
        stage("Process Performance Test Results")
        {
            parallel
            {
1293
                stage("Process results"){
1294
1295
                    when {
                        beforeAgent true
1296
                        expression { params.RUN_PERFORMANCE_TESTS.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1297
                    }
1298
1299
                    agent { label 'mici' }
                    steps{
1300
                        process_results()
1301
                        cleanWs()
1302
1303
1304
1305
                    }
                }
            }
        }
JD's avatar
JD committed
1306
    }
1307
}