Jenkinsfile 57.1 KB
Newer Older
JD's avatar
JD committed
1
def rocmnode(name) {
2
    return '(rocmtest || miopen) && (' + name + ')'
JD's avatar
JD committed
3
4
5
6
7
8
9
}

def show_node_info() {
    sh """
        echo "NODE_NAME = \$NODE_NAME"
        lsb_release -sd
        uname -r
10
        cat /sys/module/amdgpu/version
JD's avatar
JD committed
11
12
13
14
        ls /opt/ -la
    """
}

15
16
17
18
19
20
21
22
23
24
25
26
27
28
def nthreads() {
    def nproc = sh(returnStdout: true, script: 'nproc')
    echo "Number of cores: ${nproc}"
    def n = nproc.toInteger()
    if (n > 32){
        n /= 2
    }
    if (n > 64){
        n = 64
    }
    echo "Number of threads used for building: ${n}"
    return n
}

29
def runShell(String command){
30
    def responseCode = sh returnStatus: true, script: "${command} > tmp.txt"
31
32
33
34
    def output = readFile(file: "tmp.txt")
    return (output != "")
}

35
def getBaseDockerImageName(){
36
    def img
37
38
39
40
    if (params.USE_CUSTOM_DOCKER != ""){
        img = "${params.USE_CUSTOM_DOCKER}"
    }
    else{
41
42
        def ROCM_numeric = "${params.ROCMVERSION}" as float
        if ( ROCM_numeric < 6.4 ){
43
            img = "${env.CK_DOCKERHUB}:ck_ub22.04_rocm${params.ROCMVERSION}"
44
45
            }
        else{
46
            img = "${env.CK_DOCKERHUB_PRIVATE}:ck_ub22.04_rocm${params.ROCMVERSION}"
47
            }
48
49
50
51
52
53
54
55
56
        }
    return img
}

def getDockerImageName(){
    def img
    def base_name = getBaseDockerImageName()
    if (params.USE_CUSTOM_DOCKER != ""){
        img = "${params.USE_CUSTOM_DOCKER}"
57
58
    }
    else{
59
       if (params.COMPILER_VERSION == "") {
60
           img = "${base_name}"
61
62
       }
       else{
63
          if (params.COMPILER_COMMIT == ""){
64
             img = "${base_name}_${params.COMPILER_VERSION}"
65
66
67
          }
          else{
             def commit = "${params.COMPILER_COMMIT}"[0..6]
68
             img = "${base_name}_${params.COMPILER_VERSION}_${commit}"
69
          }
70
       }
71
    }
72
73
74
    return img
}

75
def check_host() {
76
77
78
79
    if ("${env.CK_SCCACHE}" != "null"){
        def SCCACHE_SERVER="${env.CK_SCCACHE.split(':')[0]}"
        echo "sccache server: ${SCCACHE_SERVER}"
        sh '''ping -c 1 -p 6379 "${SCCACHE_SERVER}" | echo $? > tmp.txt'''
80
81
82
83
84
85
86
87
88
        def output = readFile(file: "tmp.txt")
        echo "tmp.txt contents: \$output"
        return (output != "0")
    }
    else{
        return 1
    }
}

89
90
def build_compiler(){
    def compiler
91
    compiler = "${params.BUILD_COMPILER}"
92
93
94
    return compiler
}

95
96
def getDockerImage(Map conf=[:]){
    env.DOCKER_BUILDKIT=1
97
    def prefixpath = conf.get("prefixpath", "/opt/rocm")
98
    def no_cache = conf.get("no_cache", false)
99
    def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' --build-arg DISABLE_CACHE='git rev-parse ${params.COMPILER_VERSION}' "
100
101
102
103
104
    if(no_cache)
    {
        dockerArgs = dockerArgs + " --no-cache "
    }
    echo "Docker Args: ${dockerArgs}"
105
106
107
108
109
110
111
112
113
    def image
    if ( params.BUILD_LEGACY_OS && conf.get("docker_name", "") != "" ){
        image = conf.get("docker_name", "")
        echo "Using legacy docker: ${image}"
    }
    else{
        image = getDockerImageName()
        echo "Using default docker: ${image}"
    }
114
115
116
117
118
119
    //Check if image exists 
    def retimage
    try 
    {
        echo "Pulling down image: ${image}"
        retimage = docker.image("${image}")
120
121
122
        withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
            retimage.pull()
        }
123
124
125
126
127
128
129
130
131
132
133
134
135
    }
    catch(Exception ex)
    {
        error "Unable to locate image: ${image}"
    }
    return [retimage, image]
}

def buildDocker(install_prefix){
    show_node_info()
    env.DOCKER_BUILDKIT=1
    checkout scm
    def image_name = getDockerImageName()
136
    def base_image_name = getBaseDockerImageName()
137
    echo "Building Docker for ${image_name}"
138
    def dockerArgs = "--build-arg PREFIX=${install_prefix} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
139
    if(params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
140
141
142
143
        dockerArgs = dockerArgs + " --no-cache --build-arg BASE_DOCKER='${base_image_name}' -f Dockerfile.compiler . "
    }
    else{
        dockerArgs = dockerArgs + " -f Dockerfile . "
144
    }
145
146
    echo "Build Args: ${dockerArgs}"
    try{
147
148
149
        if(params.BUILD_DOCKER){
            //force building the new docker if that parameter is true
            echo "Building image: ${image_name}"
150
            retimage = docker.build("${image_name}", dockerArgs)
151
152
153
            withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
                retimage.push()
            }
154
            sh 'docker images -q -f dangling=true | xargs --no-run-if-empty docker rmi'
155
156
157
158
        }
        else{
            echo "Checking for image: ${image_name}"
            sh "docker manifest inspect --insecure ${image_name}"
159
            echo "Image: ${image_name} found! Skipping building image"
160
        }
161
162
163
164
    }
    catch(Exception ex){
        echo "Unable to locate image: ${image_name}. Building image now"
        retimage = docker.build("${image_name}", dockerArgs + ' .')
165
166
167
        withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
            retimage.push()
        }
168
169
170
    }
}

JD's avatar
JD committed
171
172
def cmake_build(Map conf=[:]){

173
    def compiler = build_compiler()
JD's avatar
JD committed
174
175
    def config_targets = conf.get("config_targets","check")
    def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined " + conf.get("extradebugflags", "")
176
    def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","")
JD's avatar
JD committed
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
    def prefixpath = conf.get("prefixpath","/opt/rocm")
    def setup_args = conf.get("setup_args","")

    if (prefixpath != "/usr/local"){
        setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} "
    }

    def build_type_debug = (conf.get("build_type",'release') == 'debug')

    //cmake_env can overwrite default CXX variables.
    def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","")

    def package_build = (conf.get("package_build","") == "true")

    if (package_build == true) {
        config_targets = "package"
    }

    if(conf.get("build_install","") == "true")
    {
        config_targets = 'install ' + config_targets
198
        setup_args = ' -DBUILD_DEV=On -DCMAKE_INSTALL_PREFIX=../install' + setup_args
JD's avatar
JD committed
199
200
201
    } else{
        setup_args = ' -DBUILD_DEV=On' + setup_args
    }
202
203
204
    if (params.DL_KERNELS){
        setup_args = setup_args + " -DDL_KERNELS=ON "
    }
JD's avatar
JD committed
205
206
207
208
209
210
211
212

    if(build_type_debug){
        setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args
    }else{
        setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args
    }

    def pre_setup_cmd = """
213
            #!/bin/bash
JD's avatar
JD committed
214
215
216
217
218
219
220
221
            echo \$HSA_ENABLE_SDMA
            ulimit -c unlimited
            rm -rf build
            mkdir build
            rm -rf install
            mkdir install
            cd build
        """
222
    def invocation_tag=""
Illia Silin's avatar
Illia Silin committed
223
224
225
    if (setup_args.contains("gfx12")){
        invocation_tag="gfx12"
    }
226
227
228
229
230
231
232
233
234
235
236
237
    if (setup_args.contains("gfx11")){
        invocation_tag="gfx11"
    }
    if (setup_args.contains("gfx10")){
        invocation_tag="gfx10"
    }
    if (setup_args.contains("gfx90")){
        invocation_tag="gfx90"
    }
    if (setup_args.contains("gfx94")){
        invocation_tag="gfx94"
    }
238
239
    echo "invocation tag: ${invocation_tag}"
    def redis_pre_setup_cmd = pre_setup_cmd
240
    if(check_host() && params.USE_SCCACHE && "${env.CK_SCCACHE}" != "null" && "${invocation_tag}" != "") {
241
        redis_pre_setup_cmd = pre_setup_cmd + """
242
243
244
245
246
247
248
249
250
251
252
253
254
            #!/bin/bash
            export ROCM_PATH=/opt/rocm
            export SCCACHE_ENABLED=true
            export SCCACHE_LOG_LEVEL=debug
            export SCCACHE_IDLE_TIMEOUT=14400
            export COMPILERS_HASH_DIR=/tmp/.sccache
            export SCCACHE_BIN=/usr/local/.cargo/bin/sccache
            export SCCACHE_EXTRAFILES=/tmp/.sccache/rocm_compilers_hash_file
            export SCCACHE_REDIS="redis://${env.CK_SCCACHE}"
            echo "connect = ${env.CK_SCCACHE}" >> ../script/redis-cli.conf
            export SCCACHE_C_CUSTOM_CACHE_BUSTER="${invocation_tag}"
            echo \$SCCACHE_C_CUSTOM_CACHE_BUSTER
            stunnel ../script/redis-cli.conf
255
            ../script/sccache_wrapper.sh --enforce_redis
256
        """
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
        try {
            def cmd1 = conf.get("cmd1", """
                    ${redis_pre_setup_cmd}
                """)
            sh cmd1
            setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache " + setup_args
        }
        catch(Exception err){
            echo "could not connect to redis server: ${err.getMessage()}. will not use sccache."
            def cmd2 = conf.get("cmd2", """
                    ${pre_setup_cmd}
                """)
            sh cmd2
        }
    }
    else{
        def cmd3 = conf.get("cmd3",  """
                ${pre_setup_cmd}
            """)
        sh cmd3
277
    }
278

Chao Liu's avatar
Chao Liu committed
279
    // reduce parallelism when compiling, clang uses too much memory
280
    def nt = nthreads()
Paul Fultz II's avatar
Paul Fultz II committed
281
    def cmd
282
283
    def setup_cmd
    def build_cmd
JD's avatar
JD committed
284
    def execute_cmd = conf.get("execute_cmd", "")
Paul Fultz II's avatar
Paul Fultz II committed
285
    if(!setup_args.contains("NO_CK_BUILD")){
286
287
288
289
290
291
292
        if (setup_args.contains("gfx90a") && params.NINJA_BUILD_TRACE){
            echo "running ninja build trace"
            setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake -G Ninja ${setup_args}   .. ")
            build_cmd = conf.get("build_cmd", "${build_envs} ninja -j${nt} ${config_targets}")
        }
        else{
            setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args}   .. ")
293
            build_cmd = conf.get("build_cmd", "${build_envs} make -j${nt} ${config_targets}")
294
        }
Paul Fultz II's avatar
Paul Fultz II committed
295
        cmd = conf.get("cmd", """
JD's avatar
JD committed
296
297
298
299
            ${setup_cmd}
            ${build_cmd}
            ${execute_cmd}
        """)
Paul Fultz II's avatar
Paul Fultz II committed
300
301
302
303
304
305
    }
    else{
        cmd = conf.get("cmd", """
            ${execute_cmd}
        """)
    }
JD's avatar
JD committed
306
307

    echo cmd
308
309

    dir("build"){
310
        //build CK
311
        sh cmd
312
313
        //run tests except when NO_CK_BUILD or BUILD_LEGACY_OS are set
        if(!setup_args.contains("NO_CK_BUILD") && !params.BUILD_LEGACY_OS){
314
315
316
317
318
319
320
321
322
            if (setup_args.contains("gfx90a") && params.NINJA_BUILD_TRACE){
                sh "/ninjatracing/ninjatracing .ninja_log > ck_build_trace.json"
                archiveArtifacts "ck_build_trace.json"
                sh "ninja test"
            }
            else{
                sh "make check"
            }
        }
323
    }
JD's avatar
JD committed
324
325

    // Only archive from master or develop
326
    if (package_build == true && (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "amd-master")) {
JD's avatar
JD committed
327
328
        archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true
    }
329
330
331
332
333
334
335
336
337
    //check the node gpu architecture
    def arch_type = 0
    sh 'rocminfo | tee rocminfo.log'
    if ( runShell('grep -n "gfx90a" rocminfo.log') ){
        arch_type = 1
    }
    else if ( runShell('grep -n "gfx942" rocminfo.log') ) {
        arch_type = 2
    }
338
    if (params.RUN_CK_TILE_FMHA_TESTS){
339
        try{
340
341
342
343
344
345
346
347
348
349
350
351
            archiveArtifacts "perf_fmha_*.log"
            if (arch_type == 1){
                stash includes: "perf_fmha_**_gfx90a.log", name: "perf_fmha_log_gfx90a"
            }
            else if (arch_type == 2){
                stash includes: "perf_fmha_**_gfx942.log", name: "perf_fmha_log_gfx942"
            }
        }
        catch(Exception err){
            echo "could not locate the requested artifacts: ${err.getMessage()}. will skip the stashing."
        }
    }
352
    // TODO: Do we do same for flex attention?
353
354
355
356
357
358
359
360
361
    if (params.RUN_CK_TILE_GEMM_TESTS){
        try{
            archiveArtifacts "perf_tile_gemm_*.log"
            if (arch_type == 1){
                stash includes: "perf_tile_gemm_**_fp16_gfx90a.log", name: "perf_tile_gemm_log_gfx90a"
            }
            else if (arch_type == 2){
                stash includes: "perf_tile_gemm_**_fp16_gfx942.log", name: "perf_tile_gemm_log_gfx942"
            }
362
363
364
365
366
        }
        catch(Exception err){
            echo "could not locate the requested artifacts: ${err.getMessage()}. will skip the stashing."
        }
    }
JD's avatar
JD committed
367
368
369
370
371
372
373
374
}

def buildHipClangJob(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

375
376
377
378
379
380
381
382
383
        def image
        if ( params.BUILD_LEGACY_OS  && conf.get("docker_name", "") != "" ){
            image = conf.get("docker_name", "")
            echo "Using legacy docker: ${image}"
        }
        else{
            image = getDockerImageName()
            echo "Using default docker: ${image}"
        }
JD's avatar
JD committed
384
385
386
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
387
        def dockerOpts="-u root --device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
JD's avatar
JD committed
388
        if (conf.get("enforce_xnack_on", false)) {
389
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
JD's avatar
JD committed
390
        }
391
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
392
        if (params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
393
394
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
395
396
397
398
        def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3')
        def render_id = sh(returnStdout: true, script: 'getent group render | cut -d: -f3')
        dockerOpts = dockerOpts + " --group-add=${video_id} --group-add=${render_id} "
        echo "Docker flags: ${dockerOpts}"
JD's avatar
JD committed
399
400
401
402

        def variant = env.STAGE_NAME

        def retimage
403
        (retimage, image) = getDockerImage(conf)
404

405
        gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
JD's avatar
JD committed
406
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
407
                timeout(time: 20, unit: 'HOURS')
JD's avatar
JD committed
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
                {
                    cmake_build(conf)
                }
            }
        }
        return retimage
}

def reboot(){
    build job: 'reboot-slaves', propagate: false , parameters: [string(name: 'server', value: "${env.NODE_NAME}"),]
}

def buildHipClangJobAndReboot(Map conf=[:]){
    try{
        buildHipClangJob(conf)
    }
    catch(e){
        echo "throwing error exception for the stage"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

436
437
438
439
def Build_CK(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
440
        env.DOCKER_BUILDKIT=1
441
442
        checkout scm

443
444
445
446
447
448
449
450
451
452
        def image
        if ( params.BUILD_LEGACY_OS  && conf.get("docker_name", "") != "" ){
            image = conf.get("docker_name", "")
            echo "Using legacy docker: ${image}"
        }
        else{
            image = getDockerImageName()
            echo "Using default docker: ${image}"
        }

453
454
455
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
456
        def dockerOpts="-u root --device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
457
458
459
        if (conf.get("enforce_xnack_on", false)) {
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
        }
460
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
461
        if (params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
462
463
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
464
465
466
        if(params.BUILD_LEGACY_OS){
            dockerOpts = dockerOpts + " --env LD_LIBRARY_PATH='/opt/Python-3.8.13/lib' "
        }
467
468
469
470
        def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3')
        def render_id = sh(returnStdout: true, script: 'getent group render | cut -d: -f3')
        dockerOpts = dockerOpts + " --group-add=${video_id} --group-add=${render_id} "
        echo "Docker flags: ${dockerOpts}"
471
472
473

        def variant = env.STAGE_NAME
        def retimage
474

475
        gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
476
477
478
            try {
                (retimage, image) = getDockerImage(conf)
                withDockerContainer(image: image, args: dockerOpts) {
Illia Silin's avatar
Illia Silin committed
479
                    timeout(time: 2, unit: 'MINUTES'){
480
481
                        sh 'rocminfo | tee rocminfo.log'
                        if ( !runShell('grep -n "gfx" rocminfo.log') ){
482
483
484
485
486
487
488
489
490
491
492
493
494
                            throw new Exception ("GPU not found")
                        }
                        else{
                            echo "GPU is OK"
                        }
                    }
                }
            }
            catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
                echo "The job was cancelled or aborted"
                throw e
            }
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
495
                timeout(time: 20, unit: 'HOURS')
496
                {
Illia Silin's avatar
Illia Silin committed
497
                    //check whether to run performance tests on this node
498
                    def arch_type = 0
499
                    sh 'rocminfo | tee rocminfo.log'
500
501
502
503
504
505
506
507
508
509
510
511
512
513
                    if ( runShell('grep -n "gfx90a" rocminfo.log') ){
                        arch_type = 1
                    }
                    else if ( runShell('grep -n "gfx942" rocminfo.log') ) {
                        arch_type = 2
                    }
                    else if ( runShell('grep -n "gfx1030" rocminfo.log') ) {
                        arch_type = 3
                    }
                    else if ( runShell('grep -n "gfx1101" rocminfo.log') ) {
                        arch_type = 4
                    }
                    else if ( runShell('grep -n "gfx1201" rocminfo.log') ) {
                        arch_type = 5
514
                    }
515
                    cmake_build(conf)
516
                    if ( !params.BUILD_LEGACY_OS && arch_type == 1 ){
517
518
519
520
521
522
                            echo "Run inductor codegen tests"
                            sh """
                                  pip install --verbose .
                                  pytest python/test/test_gen_instances.py
                            """
                    }
523
                    dir("build"){
524
525
526
                        if (params.RUN_FULL_QA && arch_type == 1 ){
                            // build deb packages for all gfx9 targets on gfx90a system and prepare to export
                            echo "Build ckProfiler package"
527
528
529
                            sh 'make -j package'
                            archiveArtifacts artifacts: 'composablekernel-ckprofiler_*.deb'
                            sh 'mv composablekernel-ckprofiler_*.deb ckprofiler_0.2.0_amd64.deb'
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
                            stash includes: "ckprofiler_0.2.0_amd64.deb", name: "ckprofiler_0.2.0_amd64.deb"
                        }
                    }
                    // run performance tests, stash the logs, results will be processed on the master node
					dir("script"){
                        if (params.RUN_PERFORMANCE_TESTS){
                        if (params.RUN_FULL_QA && arch_type == 1){
                            // run full tests on gfx90a
                            echo "Run full performance tests"
                            sh "./run_full_performance_tests.sh 0 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
                            archiveArtifacts "perf_batched_gemm.log"
                            archiveArtifacts "perf_grouped_gemm.log"
                            archiveArtifacts "perf_grouped_conv_fwd.log"
                            archiveArtifacts "perf_grouped_conv_bwd_data.log"
                            archiveArtifacts "perf_grouped_conv_bwd_weight.log"
                            archiveArtifacts "perf_gemm_bilinear.log"
                            archiveArtifacts "perf_reduction.log"
                            archiveArtifacts "perf_splitK_gemm.log"
                            archiveArtifacts "perf_onnx_gemm.log"
                            archiveArtifacts "perf_mixed_gemm.log"
                            stash includes: "perf_**.log", name: "perf_log"
                        }
                        else if ( arch_type == 1 ){
                            // run standard tests on gfx90a
                            echo "Run performance tests"
                            sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_onnx_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
                            stash includes: "perf_**.log", name: "perf_log"
                        }
                        // disable performance tests on gfx1030 for now.
                        //else if ( arch_type == 3){
                            // run basic tests on gfx1030
                        //    echo "Run gemm performance tests"
                        //    sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx10"
                        //    archiveArtifacts "perf_onnx_gemm_gfx10.log"
                        //    stash includes: "perf_onnx_gemm_gfx10.log", name: "perf_log_gfx10"
                        //}
                        else if ( arch_type == 4){
                            // run basic tests on gfx11
                            echo "Run gemm performance tests"
                            sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx11"
                            archiveArtifacts "perf_onnx_gemm_gfx11.log"
                            stash includes: "perf_onnx_gemm_gfx11.log", name: "perf_log_gfx11"
                        }
                        else if ( arch_type == 5 ){
                            // run basic tests on gfx12
                            echo "Run gemm performance tests"
                            sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx12"
                            archiveArtifacts "perf_onnx_gemm_gfx12.log"
                            stash includes: "perf_onnx_gemm_gfx12.log", name: "perf_log_gfx12"
                        }                        
587
                        }
588
                    }
589
590
                    if (params.hipTensor_test && arch_type == 1 ){
                        // build and test hipTensor on gfx90a node
591
592
593
                        sh """#!/bin/bash
                            rm -rf "${params.hipTensor_branch}".zip
                            rm -rf hipTensor-"${params.hipTensor_branch}"
594
                            wget https://github.com/ROCm/hipTensor/archive/refs/heads/"${params.hipTensor_branch}".zip
595
596
597
598
599
600
                            unzip -o "${params.hipTensor_branch}".zip
                        """
                        dir("hipTensor-${params.hipTensor_branch}"){
                            sh """#!/bin/bash
                                mkdir -p build
                                ls -ltr
601
                                CC=hipcc CXX=hipcc cmake -Bbuild . -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install"
602
                                cmake --build build -- -j
603
                                ctest --test-dir build
604
605
606
                            """
                        }
                    }
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
                }
            }
        }
        return retimage
}

def Build_CK_and_Reboot(Map conf=[:]){
    try{
        Build_CK(conf)
    }
    catch(e){
        echo "throwing error exception while building CK"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

629
630
631
def process_results(Map conf=[:]){
    env.HSA_ENABLE_SDMA=0
    checkout scm
632
    def image = getDockerImageName() 
633
634
635
    def prefixpath = "/opt/rocm"

    // Jenkins is complaining about the render group 
636
    def dockerOpts="--cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
637
    if (conf.get("enforce_xnack_on", false)) {
638
        dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
639
640
641
642
643
    }

    def variant = env.STAGE_NAME
    def retimage

644
    gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
645
        try {
646
            (retimage, image) = getDockerImage(conf)
647
648
649
650
651
652
653
654
        }
        catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
            echo "The job was cancelled or aborted"
            throw e
        }
    }

    withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
655
        timeout(time: 15, unit: 'MINUTES'){
656
657
            try{
                dir("script"){
658
                    if (params.RUN_CK_TILE_FMHA_TESTS){
659
                        try{
660
661
                            unstash "perf_fmha_log_gfx942"
                            unstash "perf_fmha_log_gfx90a"
662
663
664
665
666
                        }
                        catch(Exception err){
                            echo "could not locate the FMHA performance logs: ${err.getMessage()}."
                        }
                    }
667
668
669
670
671
672
673
674
675
                    if (params.RUN_CK_TILE_GEMM_TESTS){
                        try{
                            unstash "perf_tile_gemm_log_gfx942"
                            unstash "perf_tile_gemm_log_gfx90a"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM performance logs: ${err.getMessage()}."
                        }
                    }
676
677
                    if (params.RUN_FULL_QA){
                        // unstash perf files to master
678
679
                        unstash "ckprofiler_0.2.0_amd64.deb"
                        sh "sshpass -p ${env.ck_deb_pw} scp -o StrictHostKeyChecking=no ckprofiler_0.2.0_amd64.deb ${env.ck_deb_user}@${env.ck_deb_ip}:/var/www/html/composable_kernel/"
680
681
682
683
684
685
686
687
                        unstash "perf_log"
                        try{
                            unstash "perf_log_gfx11"
                            unstash "perf_log_gfx12"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM gfx11/gfx12 performance logs: ${err.getMessage()}."
                        }
688
                        sh "./process_qa_data.sh"
689
690
691
                    }
                    else{
                        // unstash perf files to master
692
693
694
695
696
697
698
699
                        unstash "perf_log"
                        try{
                            unstash "perf_log_gfx11"
                            unstash "perf_log_gfx12"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM gfx11/gfx12 performance logs: ${err.getMessage()}."
                        }
700
                        sh "./process_perf_data.sh"
701
702
703
704
                    }
                }
            }
            catch(e){
705
                echo "Throwing error exception while processing performance test results"
706
707
708
                echo 'Exception occurred: ' + e.toString()
                throw e
            }
709
710
711
            finally{
                echo "Finished processing performance test results"
            }
712
713
714
715
        }
    }
}

716
//launch develop branch daily at 23:00 UT in FULL_QA mode and at 19:00 UT with latest staging compiler version
717
CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;ROCMVERSION=6.3;RUN_CK_TILE_FMHA_TESTS=true;RUN_CK_TILE_FLEX_ATTENTION_TESTS=true;RUN_CK_TILE_GEMM_TESTS=true
718
                                              0 21 * * * % ROCMVERSION=6.3;hipTensor_test=true;RUN_CODEGEN_TESTS=true
719
720
                                              0 19 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-staging;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true
                                              0 17 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-mainline;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true
721
722
                                              0 15 * * * % BUILD_INSTANCES_ONLY=true;RUN_PERFORMANCE_TESTS=false;USE_SCCACHE=false
                                              0 13 * * * % BUILD_LEGACY_OS=true''' : ""
723

JD's avatar
JD committed
724
725
pipeline {
    agent none
726
727
728
    triggers {
        parameterizedCron(CRON_SETTINGS)
    }
JD's avatar
JD committed
729
730
731
    options {
        parallelsAlwaysFailFast()
    }
732
    parameters {
733
734
        booleanParam(
            name: "BUILD_DOCKER",
735
            defaultValue: false,
736
            description: "Force building docker image (default: false), set to true if docker image needs to be updated.")
737
738
739
        string(
            name: 'USE_CUSTOM_DOCKER',
            defaultValue: '',
Paul Fultz II's avatar
Paul Fultz II committed
740
            description: 'If you want to use a custom docker image, please specify it here (default: leave blank).')
741
742
        string(
            name: 'ROCMVERSION', 
743
744
            defaultValue: '6.3',
            description: 'Specify which ROCM version to use: 6.3 (default).')
745
746
        string(
            name: 'COMPILER_VERSION', 
747
            defaultValue: '', 
748
            description: 'Specify which version of compiler to use: release, amd-staging, amd-mainline, or leave blank (default).')
749
750
        string(
            name: 'COMPILER_COMMIT', 
751
            defaultValue: '', 
752
            description: 'Specify which commit of compiler branch to use: leave blank to use the latest commit (default), or use some specific commit of llvm-project branch.')
753
754
        string(
            name: 'BUILD_COMPILER', 
755
756
            defaultValue: '/opt/rocm/llvm/bin/clang++', 
            description: 'Build CK with /opt/rocm/bin/hipcc, /llvm-project/build/bin/clang++, or with /opt/rocm/llvm/bin/clang++ (default).')
757
758
759
760
        booleanParam(
            name: "RUN_FULL_QA",
            defaultValue: false,
            description: "Select whether to run small set of performance tests (default) or full QA")
761
762
763
764
        booleanParam(
            name: "DL_KERNELS",
            defaultValue: false,
            description: "Select whether to build DL kernels (default: OFF)")
765
766
        booleanParam(
            name: "hipTensor_test",
767
768
            defaultValue: false,
            description: "Use the CK build to verify hipTensor build and tests (default: OFF)")
769
770
        string(
            name: 'hipTensor_branch',
771
772
            defaultValue: 'mainline',
            description: 'Specify which branch of hipTensor to use (default: mainline)')
773
774
775
776
        booleanParam(
            name: "USE_SCCACHE",
            defaultValue: true,
            description: "Use the sccache for building CK (default: ON)")
Illia Silin's avatar
Illia Silin committed
777
778
779
780
        booleanParam(
            name: "RUN_CPPCHECK",
            defaultValue: false,
            description: "Run the cppcheck static analysis (default: OFF)")
781
782
        booleanParam(
            name: "RUN_PERFORMANCE_TESTS",
783
784
            defaultValue: true,
            description: "Run the performance tests (default: ON)")
785
786
787
788
        booleanParam(
            name: "RUN_GROUPED_CONV_LARGE_CASES_TESTS",
            defaultValue: false,
            description: "Run the grouped conv large cases tests (default: OFF)")
789
790
791
792
        booleanParam(
            name: "RUN_CODEGEN_TESTS",
            defaultValue: false,
            description: "Run codegen tests (default: OFF)")
793
        booleanParam(
794
            name: "RUN_CK_TILE_FMHA_TESTS",
795
            defaultValue: false,
796
            description: "Run the ck_tile FMHA tests (default: OFF)")
797
        booleanParam(
Aviral Goel's avatar
Aviral Goel committed
798
            name: "RUN_CK_TILE_FLEX_ATTENTION_TESTS",
Aviral Goel's avatar
Aviral Goel committed
799
            defaultValue: true,
Aviral Goel's avatar
Aviral Goel committed
800
            description: "Run the ck_tile FLEX ATTENTION tests (default: ON)")
801
802
        booleanParam(
            name: "RUN_CK_TILE_GEMM_TESTS",
803
            defaultValue: true,
804
            description: "Run the ck_tile GEMM tests (default: ON)")
805
806
807
808
        booleanParam(
            name: "BUILD_INSTANCES_ONLY",
            defaultValue: false,
            description: "Test building instances for various architectures simultaneously (default: OFF)")
Illia Silin's avatar
Illia Silin committed
809
810
        booleanParam(
            name: "BUILD_GFX12",
811
812
            defaultValue: true,
            description: "Build CK and run tests on gfx12 (default: ON)")
813
814
815
816
        booleanParam(
            name: "NINJA_BUILD_TRACE",
            defaultValue: false,
            description: "Generate a ninja build trace (default: OFF)")
817
818
819
820
        booleanParam(
            name: "BUILD_LEGACY_OS",
            defaultValue: false,
            description: "Try building CK with legacy OS dockers: RHEL8 and SLES15 (default: OFF)")
821
822
823
824
825
826
827
828
    }
    environment{
        dbuser = "${dbuser}"
        dbpassword = "${dbpassword}"
        dbsship = "${dbsship}"
        dbsshport = "${dbsshport}"
        dbsshuser = "${dbsshuser}"
        dbsshpassword = "${dbsshpassword}"
829
        ck_git_creds = "${ck_git_creds}"
830
831
        gerrit_cred="${gerrit_cred}"
        DOCKER_BUILDKIT = "1"
832
    }
JD's avatar
JD committed
833
    stages{
834
835
836
837
838
839
        stage("Build Docker"){
            parallel{
                stage('Docker /opt/rocm'){
                    agent{ label rocmnode("nogpu") }
                    steps{
                        buildDocker('/opt/rocm')
840
                        cleanWs()
841
842
843
844
                    }
                }
            }
        }
JD's avatar
JD committed
845
846
        stage("Static checks") {
            parallel{
Illia Silin's avatar
Illia Silin committed
847
848
849
850
851
852
853
                stage('Clang Format and Cppcheck') {
                    when {
                        beforeAgent true
                        expression { params.RUN_CPPCHECK.toBoolean() }
                    }
                    agent{ label rocmnode("nogpu") }
                    environment{
854
                        setup_args = "NO_CK_BUILD"
Illia Silin's avatar
Illia Silin committed
855
856
857
858
859
860
861
862
863
                        execute_cmd = "find .. -not -path \'*.git*\' -iname \'*.h\' \
                                -o -not -path \'*.git*\' -iname \'*.hpp\' \
                                -o -not -path \'*.git*\' -iname \'*.cpp\' \
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\' && \
864
865
866
867
868
                                /cppcheck/build/bin/cppcheck ../* -v -j \$(nproc) -I ../include -I ../profiler/include -I ../library/include \
                                -D CK_ENABLE_FP64 -D CK_ENABLE_FP32 -D CK_ENABLE_FP16 -D CK_ENABLE_FP8 -D CK_ENABLE_BF16 -D CK_ENABLE_BF8 -D CK_ENABLE_INT8 -D DL_KERNELS \
                                -D __gfx908__ -D __gfx90a__ -D __gfx940__ -D __gfx941__ -D __gfx942__ -D __gfx1030__ -D __gfx1100__ -D __gfx1101__ -D __gfx1102__ \
                                -U __gfx803__ -U __gfx900__ -U __gfx906__ -U CK_EXPERIMENTAL_BIT_INT_EXTENSION_INT4 \
                                --file-filter=*.cpp --force --enable=all --output-file=ck_cppcheck.log"
Illia Silin's avatar
Illia Silin committed
869
870
                    }
                    steps{
871
                        buildHipClangJobAndReboot(setup_args:setup_args, setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
Illia Silin's avatar
Illia Silin committed
872
873
874
875
                        archiveArtifacts "build/ck_cppcheck.log"
                        cleanWs()
                    }
                }
JD's avatar
JD committed
876
                stage('Clang Format') {
Illia Silin's avatar
Illia Silin committed
877
878
879
880
                    when {
                        beforeAgent true
                        expression { !params.RUN_CPPCHECK.toBoolean() }
                    }
JD's avatar
JD committed
881
882
                    agent{ label rocmnode("nogpu") }
                    environment{
883
                        setup_args = "NO_CK_BUILD"
Illia Silin's avatar
Illia Silin committed
884
885
886
                        execute_cmd = "find .. -not -path \'*.git*\' -iname \'*.h\' \
                                -o -not -path \'*.git*\' -iname \'*.hpp\' \
                                -o -not -path \'*.git*\' -iname \'*.cpp\' \
JD's avatar
JD committed
887
888
889
890
891
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
892
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\'"
JD's avatar
JD committed
893
894
                    }
                    steps{
895
                        buildHipClangJobAndReboot(setup_args:setup_args, setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
896
                        cleanWs()
JD's avatar
JD committed
897
898
899
900
                    }
                }
            }
        }
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
        stage("Run Grouped Conv Large Case Tests")
        {
            parallel
            {
                stage("Run Grouped Conv Large Case Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_GROUPED_CONV_LARGE_CASES_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a")}
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
                                           make -j64 test_grouped_convnd_fwd_large_cases_xdl && \
                                           ./bin/test_grouped_convnd_fwd_large_cases_xdl"""
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
        }
        stage("Run Codegen Tests")
        {
            parallel
            {
                stage("Run Codegen Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CODEGEN_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a")}
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ CXX=/opt/rocm/llvm/bin/clang++ cmake ../codegen && \
                                           make -j64 check"""
                    }
941
942
943
944
945
946
947
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
        }
948
        stage("Run CK_TILE_FMHA Tests")
949
950
951
        {
            parallel
            {
952
                stage("Run CK_TILE_FMHA Tests on gfx90a")
953
954
955
                {
                    when {
                        beforeAgent true
956
                        expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() }
957
958
959
960
961
962
963
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
                                           make -j64 tile_example_fmha_fwd tile_example_fmha_bwd && \
                                           cd ../ &&
964
                                           example/ck_tile/01_fmha/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
965
                    }
966
967
968
969
970
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
971
                stage("Run CK_TILE_FMHA Tests on gfx942")
972
973
974
                {
                    when {
                        beforeAgent true
975
                        expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() }
976
977
978
979
980
981
982
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
                                           make -j64 tile_example_fmha_fwd tile_example_fmha_bwd && \
                                           cd ../ &&
983
                                           example/ck_tile/01_fmha/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
984
                    }
985
986
987
988
989
990
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
991
        }
Aviral Goel's avatar
Aviral Goel committed
992
        stage("Run RUN_CK_TILE_FLEX_ATTENTION_TESTS Test")
993
        {
Aviral Goel's avatar
Aviral Goel committed
994

995
            parallel
996
            {   
Aviral Goel's avatar
Aviral Goel committed
997
                stage("Run RUN_CK_TILE_FLEX_ATTENTION_TESTS Tests on gfx90a")
998
999
1000
                {
                    when {
                        beforeAgent true
Aviral Goel's avatar
Aviral Goel committed
1001
                        expression { params.RUN_CK_TILE_FLEX_ATTENTION_TESTS.toBoolean() } // false 
1002
1003
1004
1005
1006
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
Aviral Goel's avatar
Aviral Goel committed
1007
                                           make -j64 tile_example_flexattn_fwd && \
1008
1009
1010
1011
1012
1013
1014
1015
                                           cd ../ &&
                                           example/ck_tile/18_flexattn/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
                stage("Run RUN_CK_TILE_FLEX_ATTENTION_TESTS Tests on gfx942")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_FLEX_ATTENTION_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
Aviral Goel's avatar
Aviral Goel committed
1026
                                           make -j64 tile_example_flexattn_fwd && \
1027
                                           cd ../ &&
Aviral Goel's avatar
Aviral Goel committed
1028
                                           example/ck_tile/18_flexattn/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
1029
1030
1031
1032
1033
1034
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
1035
1036
            }
        }
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
        stage("Run CK_TILE_GEMM Tests")
        {
            parallel
            {
                stage("Run CK_TILE_GEMM Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_GEMM_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
1051
                                           make -j64 tile_example_gemm_basic tile_example_gemm_universal && \
1052
1053
                                           cd ../ &&
                                           example/ck_tile/03_gemm/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
1054
                    }
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
                stage("Run CK_TILE_GEMM Tests on gfx942")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_GEMM_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
1070
                                           make -j64 tile_example_gemm_basic tile_example_gemm_universal && \
1071
1072
                                           cd ../ &&
                                           example/ck_tile/03_gemm/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
1073
                    }
1074
1075
1076
1077
1078
1079
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
Paul Fultz II's avatar
Paul Fultz II committed
1080
        }
1081

1082
		stage("Build CK and run Tests")
1083
1084
1085
        {
            parallel
            {
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
                stage("Build CK with RHEL8")
                {
                    when {
                        beforeAgent true
                        expression { params.BUILD_LEGACY_OS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        def docker_name = "${env.CK_DOCKERHUB_PRIVATE}:ck_rhel8_rocm6.3"
                        setup_args = """ -DGPU_TARGETS="gfx942" \
                                         -DCMAKE_CXX_FLAGS=" -O3 " \
                                         -DCK_USE_ALTERNATIVE_PYTHON=/opt/Python-3.8.13/bin/python3.8 """
                        execute_args = " "
1099
                    }
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: " ", no_reboot:true, build_type: 'Release', docker_name: docker_name)
                        cleanWs()
                    }
                }
                stage("Build CK with SLES15")
                {
                    when {
                        beforeAgent true
                        expression { params.BUILD_LEGACY_OS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        def docker_name = "${env.CK_DOCKERHUB_PRIVATE}:ck_sles15_rocm6.3"
                        setup_args = """ -DGPU_TARGETS="gfx942" \
                                         -DCMAKE_CXX_FLAGS=" -O3 " \
                                         -DCK_USE_ALTERNATIVE_PYTHON=/opt/Python-3.8.13/bin/python3.8 """
                        execute_args = " "
1118
                    }
1119
1120
1121
1122
1123
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: " ", no_reboot:true, build_type: 'Release', docker_name: docker_name)
                        cleanWs()
                    }
                }
1124
                stage("Build CK for all gfx9 targets")
1125
1126
1127
                {
                    when {
                        beforeAgent true
1128
                        expression { params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1129
                    }
1130
                    agent{ label rocmnode("gfx90a") }
1131
                    environment{
1132
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install \
1133
                                         -DGPU_TARGETS="gfx908;gfx90a;gfx942" \
1134
                                         -DCMAKE_CXX_FLAGS=" -O3 " """
1135
1136
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
1137
                                           -DGPU_TARGETS="gfx908;gfx90a;gfx942" \
1138
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1139
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1140
1141
                    }
                    steps{
1142
1143
1144
1145
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
                        cleanWs()
                    }
                }
Illia Silin's avatar
Illia Silin committed
1146
                stage("Build CK and run Tests on gfx942")
1147
1148
1149
                {
                    when {
                        beforeAgent true
1150
                        expression { params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1151
1152
1153
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
1154
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx942" -DCMAKE_CXX_FLAGS=" -O3 " """
1155
1156
1157
1158
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx942" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1159
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1160
1161
                    }
                    steps{
1162
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1163
                        cleanWs()
1164
1165
                    }
                }
Illia Silin's avatar
Illia Silin committed
1166
                stage("Build CK and run Tests on gfx90a")
1167
                {
1168
1169
                    when {
                        beforeAgent true
1170
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1171
                    }
1172
                    agent{ label rocmnode("gfx90a") }
1173
                    environment{
1174
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx90a" -DCMAKE_CXX_FLAGS=" -O3 " """
1175
1176
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
1177
                                           -DGPU_TARGETS="gfx90a" \
1178
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1179
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
Illia Silin's avatar
Illia Silin committed
1180
1181
1182
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1183
                        cleanWs()
Illia Silin's avatar
Illia Silin committed
1184
1185
                    }
                }
1186
1187
1188
1189
                stage("Build CK instances for different targets")
                {
                    when {
                        beforeAgent true
1190
                        expression { params.BUILD_INSTANCES_ONLY.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1191
1192
1193
1194
1195
1196
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        execute_args = """ cmake -D CMAKE_PREFIX_PATH=/opt/rocm \
                                           -D CMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -D CMAKE_BUILD_TYPE=Release \
1197
                                           -D GPU_ARCHS="gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102"  \
1198
                                           -D CMAKE_CXX_FLAGS=" -O3 " .. && make -j64 """
1199
                    }
1200
1201
1202
1203
1204
                    steps{
                        buildHipClangJobAndReboot(setup_cmd: "",  build_cmd: "", no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
Illia Silin's avatar
Illia Silin committed
1205
                stage("Build CK and run Tests on gfx1030")
Illia Silin's avatar
Illia Silin committed
1206
1207
1208
                {
                    when {
                        beforeAgent true
1209
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
Illia Silin's avatar
Illia Silin committed
1210
                    }
Illia Silin's avatar
Illia Silin committed
1211
                    agent{ label rocmnode("gfx1030") }
Illia Silin's avatar
Illia Silin committed
1212
                    environment{
1213
1214
1215
1216
1217
1218
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1030" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """ 
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1030" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1219
                    }
1220
1221
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1222
                        cleanWs()
1223
1224
                    }
                }
Illia Silin's avatar
Illia Silin committed
1225
                stage("Build CK and run Tests on gfx1101")
1226
1227
1228
                {
                    when {
                        beforeAgent true
1229
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1230
                    }
Illia Silin's avatar
Illia Silin committed
1231
                    agent{ label rocmnode("gfx1101") }
1232
                    environment{
1233
1234
1235
1236
1237
1238
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1101" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1101" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1239
                    }
1240
                    steps{
1241
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1242
                        cleanWs()
JD's avatar
JD committed
1243
1244
                    }
                }
Illia Silin's avatar
Illia Silin committed
1245
1246
1247
1248
                stage("Build CK and run Tests on gfx1201")
                {
                    when {
                        beforeAgent true
1249
                        expression { params.BUILD_GFX12.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
Illia Silin's avatar
Illia Silin committed
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
                    }
                    agent{ label rocmnode("gfx1201") }
                    environment{
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1201" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1201" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
                        cleanWs()
                    }
                }
1265
1266
            }
        }
1267
1268
1269
1270
        stage("Process Performance Test Results")
        {
            parallel
            {
1271
                stage("Process results"){
1272
1273
                    when {
                        beforeAgent true
1274
                        expression { params.RUN_PERFORMANCE_TESTS.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1275
                    }
1276
1277
                    agent { label 'mici' }
                    steps{
1278
                        process_results()
1279
                        cleanWs()
1280
1281
1282
1283
                    }
                }
            }
        }
JD's avatar
JD committed
1284
    }
1285
}