Jenkinsfile 54.2 KB
Newer Older
JD's avatar
JD committed
1
def rocmnode(name) {
2
    return '(rocmtest || miopen) && (' + name + ')'
JD's avatar
JD committed
3
4
5
6
7
8
9
}

def show_node_info() {
    sh """
        echo "NODE_NAME = \$NODE_NAME"
        lsb_release -sd
        uname -r
10
        cat /sys/module/amdgpu/version
JD's avatar
JD committed
11
12
13
14
        ls /opt/ -la
    """
}

15
16
17
18
19
20
21
22
23
24
25
26
27
28
def nthreads() {
    def nproc = sh(returnStdout: true, script: 'nproc')
    echo "Number of cores: ${nproc}"
    def n = nproc.toInteger()
    if (n > 32){
        n /= 2
    }
    if (n > 64){
        n = 64
    }
    echo "Number of threads used for building: ${n}"
    return n
}

29
def runShell(String command){
30
    def responseCode = sh returnStatus: true, script: "${command} > tmp.txt"
31
32
33
34
    def output = readFile(file: "tmp.txt")
    return (output != "")
}

35
def getBaseDockerImageName(){
36
    def img
37
38
39
40
    if (params.USE_CUSTOM_DOCKER != ""){
        img = "${params.USE_CUSTOM_DOCKER}"
    }
    else{
41
42
        def ROCM_numeric = "${params.ROCMVERSION}" as float
        if ( ROCM_numeric < 6.4 ){
43
            img = "${env.CK_DOCKERHUB}:ck_ub22.04_rocm${params.ROCMVERSION}"
44
45
            }
        else{
46
            img = "${env.CK_DOCKERHUB_PRIVATE}:ck_ub22.04_rocm${params.ROCMVERSION}"
47
            }
48
49
50
51
52
53
54
55
56
        }
    return img
}

def getDockerImageName(){
    def img
    def base_name = getBaseDockerImageName()
    if (params.USE_CUSTOM_DOCKER != ""){
        img = "${params.USE_CUSTOM_DOCKER}"
57
58
    }
    else{
59
       if (params.COMPILER_VERSION == "") {
60
           img = "${base_name}"
61
62
       }
       else{
63
          if (params.COMPILER_COMMIT == ""){
64
             img = "${base_name}_${params.COMPILER_VERSION}"
65
66
67
          }
          else{
             def commit = "${params.COMPILER_COMMIT}"[0..6]
68
             img = "${base_name}_${params.COMPILER_VERSION}_${commit}"
69
          }
70
       }
71
    }
72
73
74
    return img
}

75
def check_host() {
76
77
78
79
    if ("${env.CK_SCCACHE}" != "null"){
        def SCCACHE_SERVER="${env.CK_SCCACHE.split(':')[0]}"
        echo "sccache server: ${SCCACHE_SERVER}"
        sh '''ping -c 1 -p 6379 "${SCCACHE_SERVER}" | echo $? > tmp.txt'''
80
81
82
83
84
85
86
87
88
        def output = readFile(file: "tmp.txt")
        echo "tmp.txt contents: \$output"
        return (output != "0")
    }
    else{
        return 1
    }
}

89
90
def build_compiler(){
    def compiler
91
    compiler = "${params.BUILD_COMPILER}"
92
93
94
    return compiler
}

95
96
def getDockerImage(Map conf=[:]){
    env.DOCKER_BUILDKIT=1
97
    def prefixpath = conf.get("prefixpath", "/opt/rocm")
98
    def no_cache = conf.get("no_cache", false)
99
    def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' --build-arg DISABLE_CACHE='git rev-parse ${params.COMPILER_VERSION}' "
100
101
102
103
104
    if(no_cache)
    {
        dockerArgs = dockerArgs + " --no-cache "
    }
    echo "Docker Args: ${dockerArgs}"
105
106
107
108
109
110
111
112
113
    def image
    if ( params.BUILD_LEGACY_OS && conf.get("docker_name", "") != "" ){
        image = conf.get("docker_name", "")
        echo "Using legacy docker: ${image}"
    }
    else{
        image = getDockerImageName()
        echo "Using default docker: ${image}"
    }
114
115
116
117
118
119
    //Check if image exists 
    def retimage
    try 
    {
        echo "Pulling down image: ${image}"
        retimage = docker.image("${image}")
120
121
122
        withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
            retimage.pull()
        }
123
124
125
126
127
128
129
130
131
132
133
134
135
    }
    catch(Exception ex)
    {
        error "Unable to locate image: ${image}"
    }
    return [retimage, image]
}

def buildDocker(install_prefix){
    show_node_info()
    env.DOCKER_BUILDKIT=1
    checkout scm
    def image_name = getDockerImageName()
136
    def base_image_name = getBaseDockerImageName()
137
    echo "Building Docker for ${image_name}"
138
    def dockerArgs = "--build-arg PREFIX=${install_prefix} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
139
    if(params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
140
141
142
143
        dockerArgs = dockerArgs + " --no-cache --build-arg BASE_DOCKER='${base_image_name}' -f Dockerfile.compiler . "
    }
    else{
        dockerArgs = dockerArgs + " -f Dockerfile . "
144
    }
145
146
    echo "Build Args: ${dockerArgs}"
    try{
147
148
149
        if(params.BUILD_DOCKER){
            //force building the new docker if that parameter is true
            echo "Building image: ${image_name}"
150
            retimage = docker.build("${image_name}", dockerArgs)
151
152
153
            withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
                retimage.push()
            }
154
            sh 'docker images -q -f dangling=true | xargs --no-run-if-empty docker rmi'
155
156
157
158
        }
        else{
            echo "Checking for image: ${image_name}"
            sh "docker manifest inspect --insecure ${image_name}"
159
            echo "Image: ${image_name} found! Skipping building image"
160
        }
161
162
163
164
    }
    catch(Exception ex){
        echo "Unable to locate image: ${image_name}. Building image now"
        retimage = docker.build("${image_name}", dockerArgs + ' .')
165
166
167
        withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) {
            retimage.push()
        }
168
169
170
    }
}

JD's avatar
JD committed
171
172
def cmake_build(Map conf=[:]){

173
    def compiler = build_compiler()
JD's avatar
JD committed
174
175
    def config_targets = conf.get("config_targets","check")
    def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined " + conf.get("extradebugflags", "")
176
    def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","")
JD's avatar
JD committed
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
    def prefixpath = conf.get("prefixpath","/opt/rocm")
    def setup_args = conf.get("setup_args","")

    if (prefixpath != "/usr/local"){
        setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} "
    }

    def build_type_debug = (conf.get("build_type",'release') == 'debug')

    //cmake_env can overwrite default CXX variables.
    def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","")

    def package_build = (conf.get("package_build","") == "true")

    if (package_build == true) {
        config_targets = "package"
    }

    if(conf.get("build_install","") == "true")
    {
        config_targets = 'install ' + config_targets
198
        setup_args = ' -DBUILD_DEV=On -DCMAKE_INSTALL_PREFIX=../install' + setup_args
JD's avatar
JD committed
199
200
201
    } else{
        setup_args = ' -DBUILD_DEV=On' + setup_args
    }
202
203
204
    if (params.DL_KERNELS){
        setup_args = setup_args + " -DDL_KERNELS=ON "
    }
JD's avatar
JD committed
205
206
207
208
209
210
211
212

    if(build_type_debug){
        setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args
    }else{
        setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args
    }

    def pre_setup_cmd = """
213
            #!/bin/bash
JD's avatar
JD committed
214
215
216
217
218
219
220
221
            echo \$HSA_ENABLE_SDMA
            ulimit -c unlimited
            rm -rf build
            mkdir build
            rm -rf install
            mkdir install
            cd build
        """
222
    def invocation_tag=""
Illia Silin's avatar
Illia Silin committed
223
224
225
    if (setup_args.contains("gfx12")){
        invocation_tag="gfx12"
    }
226
227
228
229
230
231
232
233
234
235
236
237
    if (setup_args.contains("gfx11")){
        invocation_tag="gfx11"
    }
    if (setup_args.contains("gfx10")){
        invocation_tag="gfx10"
    }
    if (setup_args.contains("gfx90")){
        invocation_tag="gfx90"
    }
    if (setup_args.contains("gfx94")){
        invocation_tag="gfx94"
    }
238
239
    echo "invocation tag: ${invocation_tag}"
    def redis_pre_setup_cmd = pre_setup_cmd
240
    if(check_host() && params.USE_SCCACHE && "${env.CK_SCCACHE}" != "null" && "${invocation_tag}" != "") {
241
        redis_pre_setup_cmd = pre_setup_cmd + """
242
243
244
245
246
247
248
249
250
251
252
253
254
            #!/bin/bash
            export ROCM_PATH=/opt/rocm
            export SCCACHE_ENABLED=true
            export SCCACHE_LOG_LEVEL=debug
            export SCCACHE_IDLE_TIMEOUT=14400
            export COMPILERS_HASH_DIR=/tmp/.sccache
            export SCCACHE_BIN=/usr/local/.cargo/bin/sccache
            export SCCACHE_EXTRAFILES=/tmp/.sccache/rocm_compilers_hash_file
            export SCCACHE_REDIS="redis://${env.CK_SCCACHE}"
            echo "connect = ${env.CK_SCCACHE}" >> ../script/redis-cli.conf
            export SCCACHE_C_CUSTOM_CACHE_BUSTER="${invocation_tag}"
            echo \$SCCACHE_C_CUSTOM_CACHE_BUSTER
            stunnel ../script/redis-cli.conf
255
            ../script/sccache_wrapper.sh --enforce_redis
256
        """
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
        try {
            def cmd1 = conf.get("cmd1", """
                    ${redis_pre_setup_cmd}
                """)
            sh cmd1
            setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache " + setup_args
        }
        catch(Exception err){
            echo "could not connect to redis server: ${err.getMessage()}. will not use sccache."
            def cmd2 = conf.get("cmd2", """
                    ${pre_setup_cmd}
                """)
            sh cmd2
        }
    }
    else{
        def cmd3 = conf.get("cmd3",  """
                ${pre_setup_cmd}
            """)
        sh cmd3
277
    }
278

Chao Liu's avatar
Chao Liu committed
279
    // reduce parallelism when compiling, clang uses too much memory
280
    def nt = nthreads()
Paul Fultz II's avatar
Paul Fultz II committed
281
    def cmd
282
283
    def setup_cmd
    def build_cmd
JD's avatar
JD committed
284
    def execute_cmd = conf.get("execute_cmd", "")
Paul Fultz II's avatar
Paul Fultz II committed
285
    if(!setup_args.contains("NO_CK_BUILD")){
286
287
288
289
290
291
292
        if (setup_args.contains("gfx90a") && params.NINJA_BUILD_TRACE){
            echo "running ninja build trace"
            setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake -G Ninja ${setup_args}   .. ")
            build_cmd = conf.get("build_cmd", "${build_envs} ninja -j${nt} ${config_targets}")
        }
        else{
            setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args}   .. ")
293
            build_cmd = conf.get("build_cmd", "${build_envs} make -j${nt} ${config_targets}")
294
        }
Paul Fultz II's avatar
Paul Fultz II committed
295
        cmd = conf.get("cmd", """
JD's avatar
JD committed
296
297
298
299
            ${setup_cmd}
            ${build_cmd}
            ${execute_cmd}
        """)
Paul Fultz II's avatar
Paul Fultz II committed
300
301
302
303
304
305
    }
    else{
        cmd = conf.get("cmd", """
            ${execute_cmd}
        """)
    }
JD's avatar
JD committed
306
307

    echo cmd
308
309

    dir("build"){
310
        //build CK
311
        sh cmd
312
313
        //run tests except when NO_CK_BUILD or BUILD_LEGACY_OS are set
        if(!setup_args.contains("NO_CK_BUILD") && !params.BUILD_LEGACY_OS){
314
315
316
317
318
319
320
321
322
            if (setup_args.contains("gfx90a") && params.NINJA_BUILD_TRACE){
                sh "/ninjatracing/ninjatracing .ninja_log > ck_build_trace.json"
                archiveArtifacts "ck_build_trace.json"
                sh "ninja test"
            }
            else{
                sh "make check"
            }
        }
323
    }
JD's avatar
JD committed
324
325

    // Only archive from master or develop
326
    if (package_build == true && (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "amd-master")) {
JD's avatar
JD committed
327
328
        archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true
    }
329
330
331
332
333
334
335
336
337
    //check the node gpu architecture
    def arch_type = 0
    sh 'rocminfo | tee rocminfo.log'
    if ( runShell('grep -n "gfx90a" rocminfo.log') ){
        arch_type = 1
    }
    else if ( runShell('grep -n "gfx942" rocminfo.log') ) {
        arch_type = 2
    }
338
    if (params.RUN_CK_TILE_FMHA_TESTS){
339
        try{
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
            archiveArtifacts "perf_fmha_*.log"
            if (arch_type == 1){
                stash includes: "perf_fmha_**_gfx90a.log", name: "perf_fmha_log_gfx90a"
            }
            else if (arch_type == 2){
                stash includes: "perf_fmha_**_gfx942.log", name: "perf_fmha_log_gfx942"
            }
        }
        catch(Exception err){
            echo "could not locate the requested artifacts: ${err.getMessage()}. will skip the stashing."
        }
    }
    if (params.RUN_CK_TILE_GEMM_TESTS){
        try{
            archiveArtifacts "perf_tile_gemm_*.log"
            if (arch_type == 1){
                stash includes: "perf_tile_gemm_**_fp16_gfx90a.log", name: "perf_tile_gemm_log_gfx90a"
            }
            else if (arch_type == 2){
                stash includes: "perf_tile_gemm_**_fp16_gfx942.log", name: "perf_tile_gemm_log_gfx942"
            }
361
362
363
364
365
        }
        catch(Exception err){
            echo "could not locate the requested artifacts: ${err.getMessage()}. will skip the stashing."
        }
    }
JD's avatar
JD committed
366
367
368
369
370
371
372
373
}

def buildHipClangJob(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

374
375
376
377
378
379
380
381
382
        def image
        if ( params.BUILD_LEGACY_OS  && conf.get("docker_name", "") != "" ){
            image = conf.get("docker_name", "")
            echo "Using legacy docker: ${image}"
        }
        else{
            image = getDockerImageName()
            echo "Using default docker: ${image}"
        }
JD's avatar
JD committed
383
384
385
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
386
        def dockerOpts="-u root --device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
JD's avatar
JD committed
387
        if (conf.get("enforce_xnack_on", false)) {
388
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
JD's avatar
JD committed
389
        }
390
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
391
        if (params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
392
393
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
394
395
396
397
        def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3')
        def render_id = sh(returnStdout: true, script: 'getent group render | cut -d: -f3')
        dockerOpts = dockerOpts + " --group-add=${video_id} --group-add=${render_id} "
        echo "Docker flags: ${dockerOpts}"
JD's avatar
JD committed
398
399
400
401

        def variant = env.STAGE_NAME

        def retimage
402
        (retimage, image) = getDockerImage(conf)
403

404
        gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
JD's avatar
JD committed
405
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
406
                timeout(time: 20, unit: 'HOURS')
JD's avatar
JD committed
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
                {
                    cmake_build(conf)
                }
            }
        }
        return retimage
}

def reboot(){
    build job: 'reboot-slaves', propagate: false , parameters: [string(name: 'server', value: "${env.NODE_NAME}"),]
}

def buildHipClangJobAndReboot(Map conf=[:]){
    try{
        buildHipClangJob(conf)
    }
    catch(e){
        echo "throwing error exception for the stage"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

435
436
437
438
def Build_CK(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
439
        env.DOCKER_BUILDKIT=1
440
441
        checkout scm

442
443
444
445
446
447
448
449
450
451
        def image
        if ( params.BUILD_LEGACY_OS  && conf.get("docker_name", "") != "" ){
            image = conf.get("docker_name", "")
            echo "Using legacy docker: ${image}"
        }
        else{
            image = getDockerImageName()
            echo "Using default docker: ${image}"
        }

452
453
454
        def prefixpath = conf.get("prefixpath", "/opt/rocm")

        // Jenkins is complaining about the render group 
455
        def dockerOpts="-u root --device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
456
457
458
        if (conf.get("enforce_xnack_on", false)) {
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
        }
459
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
460
        if (params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){
461
462
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
463
464
465
        if(params.BUILD_LEGACY_OS){
            dockerOpts = dockerOpts + " --env LD_LIBRARY_PATH='/opt/Python-3.8.13/lib' "
        }
466
467
468
469
        def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3')
        def render_id = sh(returnStdout: true, script: 'getent group render | cut -d: -f3')
        dockerOpts = dockerOpts + " --group-add=${video_id} --group-add=${render_id} "
        echo "Docker flags: ${dockerOpts}"
470
471
472

        def variant = env.STAGE_NAME
        def retimage
473

474
        gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
475
476
477
            try {
                (retimage, image) = getDockerImage(conf)
                withDockerContainer(image: image, args: dockerOpts) {
Illia Silin's avatar
Illia Silin committed
478
                    timeout(time: 2, unit: 'MINUTES'){
479
480
                        sh 'rocminfo | tee rocminfo.log'
                        if ( !runShell('grep -n "gfx" rocminfo.log') ){
481
482
483
484
485
486
487
488
489
490
491
492
493
                            throw new Exception ("GPU not found")
                        }
                        else{
                            echo "GPU is OK"
                        }
                    }
                }
            }
            catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
                echo "The job was cancelled or aborted"
                throw e
            }
            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
494
                timeout(time: 20, unit: 'HOURS')
495
                {
Illia Silin's avatar
Illia Silin committed
496
                    //check whether to run performance tests on this node
497
                    def arch_type = 0
498
                    sh 'rocminfo | tee rocminfo.log'
499
500
501
502
503
504
505
506
507
508
509
510
511
512
                    if ( runShell('grep -n "gfx90a" rocminfo.log') ){
                        arch_type = 1
                    }
                    else if ( runShell('grep -n "gfx942" rocminfo.log') ) {
                        arch_type = 2
                    }
                    else if ( runShell('grep -n "gfx1030" rocminfo.log') ) {
                        arch_type = 3
                    }
                    else if ( runShell('grep -n "gfx1101" rocminfo.log') ) {
                        arch_type = 4
                    }
                    else if ( runShell('grep -n "gfx1201" rocminfo.log') ) {
                        arch_type = 5
513
                    }
514
515
                    cmake_build(conf)
                    dir("build"){
516
517
518
                        if (params.RUN_FULL_QA && arch_type == 1 ){
                            // build deb packages for all gfx9 targets on gfx90a system and prepare to export
                            echo "Build ckProfiler package"
519
520
521
                            sh 'make -j package'
                            archiveArtifacts artifacts: 'composablekernel-ckprofiler_*.deb'
                            sh 'mv composablekernel-ckprofiler_*.deb ckprofiler_0.2.0_amd64.deb'
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
                            stash includes: "ckprofiler_0.2.0_amd64.deb", name: "ckprofiler_0.2.0_amd64.deb"
                        }
                    }
                    // run performance tests, stash the logs, results will be processed on the master node
					dir("script"){
                        if (params.RUN_PERFORMANCE_TESTS){
                        if (params.RUN_FULL_QA && arch_type == 1){
                            // run full tests on gfx90a
                            echo "Run full performance tests"
                            sh "./run_full_performance_tests.sh 0 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
                            archiveArtifacts "perf_batched_gemm.log"
                            archiveArtifacts "perf_grouped_gemm.log"
                            archiveArtifacts "perf_grouped_conv_fwd.log"
                            archiveArtifacts "perf_grouped_conv_bwd_data.log"
                            archiveArtifacts "perf_grouped_conv_bwd_weight.log"
                            archiveArtifacts "perf_gemm_bilinear.log"
                            archiveArtifacts "perf_reduction.log"
                            archiveArtifacts "perf_splitK_gemm.log"
                            archiveArtifacts "perf_onnx_gemm.log"
                            archiveArtifacts "perf_mixed_gemm.log"
                            stash includes: "perf_**.log", name: "perf_log"
                        }
                        else if ( arch_type == 1 ){
                            // run standard tests on gfx90a
                            echo "Run performance tests"
                            sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
                            archiveArtifacts "perf_gemm.log"
                            archiveArtifacts "perf_onnx_gemm.log"
                            archiveArtifacts "perf_resnet50_N256.log"
                            archiveArtifacts "perf_resnet50_N4.log"
                            stash includes: "perf_**.log", name: "perf_log"
                        }
                        // disable performance tests on gfx1030 for now.
                        //else if ( arch_type == 3){
                            // run basic tests on gfx1030
                        //    echo "Run gemm performance tests"
                        //    sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx10"
                        //    archiveArtifacts "perf_onnx_gemm_gfx10.log"
                        //    stash includes: "perf_onnx_gemm_gfx10.log", name: "perf_log_gfx10"
                        //}
                        else if ( arch_type == 4){
                            // run basic tests on gfx11
                            echo "Run gemm performance tests"
                            sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx11"
                            archiveArtifacts "perf_onnx_gemm_gfx11.log"
                            stash includes: "perf_onnx_gemm_gfx11.log", name: "perf_log_gfx11"
                        }
                        else if ( arch_type == 5 ){
                            // run basic tests on gfx12
                            echo "Run gemm performance tests"
                            sh "./run_gemm_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx12"
                            archiveArtifacts "perf_onnx_gemm_gfx12.log"
                            stash includes: "perf_onnx_gemm_gfx12.log", name: "perf_log_gfx12"
                        }                        
579
                        }
580
                    }
581
582
                    if (params.hipTensor_test && arch_type == 1 ){
                        // build and test hipTensor on gfx90a node
583
584
585
                        sh """#!/bin/bash
                            rm -rf "${params.hipTensor_branch}".zip
                            rm -rf hipTensor-"${params.hipTensor_branch}"
586
                            wget https://github.com/ROCm/hipTensor/archive/refs/heads/"${params.hipTensor_branch}".zip
587
588
589
590
591
592
                            unzip -o "${params.hipTensor_branch}".zip
                        """
                        dir("hipTensor-${params.hipTensor_branch}"){
                            sh """#!/bin/bash
                                mkdir -p build
                                ls -ltr
593
                                CC=hipcc CXX=hipcc cmake -Bbuild . -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install"
594
                                cmake --build build -- -j
595
                                ctest --test-dir build
596
597
598
                            """
                        }
                    }
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
                }
            }
        }
        return retimage
}

def Build_CK_and_Reboot(Map conf=[:]){
    try{
        Build_CK(conf)
    }
    catch(e){
        echo "throwing error exception while building CK"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

621
622
623
def process_results(Map conf=[:]){
    env.HSA_ENABLE_SDMA=0
    checkout scm
624
    def image = getDockerImageName() 
625
626
627
    def prefixpath = "/opt/rocm"

    // Jenkins is complaining about the render group 
628
    def dockerOpts="--cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
629
    if (conf.get("enforce_xnack_on", false)) {
630
        dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
631
632
633
634
635
    }

    def variant = env.STAGE_NAME
    def retimage

636
    gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') {
637
        try {
638
            (retimage, image) = getDockerImage(conf)
639
640
641
642
643
644
645
646
        }
        catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
            echo "The job was cancelled or aborted"
            throw e
        }
    }

    withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
Illia Silin's avatar
Illia Silin committed
647
        timeout(time: 15, unit: 'MINUTES'){
648
649
            try{
                dir("script"){
650
                    if (params.RUN_CK_TILE_FMHA_TESTS){
651
                        try{
652
653
                            unstash "perf_fmha_log_gfx942"
                            unstash "perf_fmha_log_gfx90a"
654
655
656
657
658
                        }
                        catch(Exception err){
                            echo "could not locate the FMHA performance logs: ${err.getMessage()}."
                        }
                    }
659
660
661
662
663
664
665
666
667
                    if (params.RUN_CK_TILE_GEMM_TESTS){
                        try{
                            unstash "perf_tile_gemm_log_gfx942"
                            unstash "perf_tile_gemm_log_gfx90a"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM performance logs: ${err.getMessage()}."
                        }
                    }
668
669
                    if (params.RUN_FULL_QA){
                        // unstash perf files to master
670
671
                        unstash "ckprofiler_0.2.0_amd64.deb"
                        sh "sshpass -p ${env.ck_deb_pw} scp -o StrictHostKeyChecking=no ckprofiler_0.2.0_amd64.deb ${env.ck_deb_user}@${env.ck_deb_ip}:/var/www/html/composable_kernel/"
672
673
674
675
676
677
678
679
                        unstash "perf_log"
                        try{
                            unstash "perf_log_gfx11"
                            unstash "perf_log_gfx12"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM gfx11/gfx12 performance logs: ${err.getMessage()}."
                        }
680
                        sh "./process_qa_data.sh"
681
682
683
                    }
                    else{
                        // unstash perf files to master
684
685
686
687
688
689
690
691
                        unstash "perf_log"
                        try{
                            unstash "perf_log_gfx11"
                            unstash "perf_log_gfx12"
                        }
                        catch(Exception err){
                            echo "could not locate the GEMM gfx11/gfx12 performance logs: ${err.getMessage()}."
                        }
692
                        sh "./process_perf_data.sh"
693
694
695
696
                    }
                }
            }
            catch(e){
697
                echo "Throwing error exception while processing performance test results"
698
699
700
                echo 'Exception occurred: ' + e.toString()
                throw e
            }
701
702
703
            finally{
                echo "Finished processing performance test results"
            }
704
705
706
707
        }
    }
}

708
//launch develop branch daily at 23:00 UT in FULL_QA mode and at 19:00 UT with latest staging compiler version
709
710
CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;ROCMVERSION=6.3;RUN_CK_TILE_FMHA_TESTS=true;RUN_CK_TILE_GEMM_TESTS=true
                                              0 21 * * * % ROCMVERSION=6.3;hipTensor_test=true;RUN_CODEGEN_TESTS=true
711
712
                                              0 19 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-staging;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true
                                              0 17 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-mainline;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true
713
714
                                              0 15 * * * % BUILD_INSTANCES_ONLY=true;RUN_PERFORMANCE_TESTS=false;USE_SCCACHE=false
                                              0 13 * * * % BUILD_LEGACY_OS=true''' : ""
715

JD's avatar
JD committed
716
717
pipeline {
    agent none
718
719
720
    triggers {
        parameterizedCron(CRON_SETTINGS)
    }
JD's avatar
JD committed
721
722
723
    options {
        parallelsAlwaysFailFast()
    }
724
    parameters {
725
726
        booleanParam(
            name: "BUILD_DOCKER",
727
            defaultValue: false,
728
            description: "Force building docker image (default: false), set to true if docker image needs to be updated.")
729
730
731
        string(
            name: 'USE_CUSTOM_DOCKER',
            defaultValue: '',
Paul Fultz II's avatar
Paul Fultz II committed
732
            description: 'If you want to use a custom docker image, please specify it here (default: leave blank).')
733
734
        string(
            name: 'ROCMVERSION', 
735
736
            defaultValue: '6.3',
            description: 'Specify which ROCM version to use: 6.3 (default).')
737
738
        string(
            name: 'COMPILER_VERSION', 
739
            defaultValue: '', 
740
            description: 'Specify which version of compiler to use: release, amd-staging, amd-mainline, or leave blank (default).')
741
742
        string(
            name: 'COMPILER_COMMIT', 
743
            defaultValue: '', 
744
            description: 'Specify which commit of compiler branch to use: leave blank to use the latest commit (default), or use some specific commit of llvm-project branch.')
745
746
        string(
            name: 'BUILD_COMPILER', 
747
748
            defaultValue: '/opt/rocm/llvm/bin/clang++', 
            description: 'Build CK with /opt/rocm/bin/hipcc, /llvm-project/build/bin/clang++, or with /opt/rocm/llvm/bin/clang++ (default).')
749
750
751
752
        booleanParam(
            name: "RUN_FULL_QA",
            defaultValue: false,
            description: "Select whether to run small set of performance tests (default) or full QA")
753
754
755
756
        booleanParam(
            name: "DL_KERNELS",
            defaultValue: false,
            description: "Select whether to build DL kernels (default: OFF)")
757
758
        booleanParam(
            name: "hipTensor_test",
759
760
            defaultValue: false,
            description: "Use the CK build to verify hipTensor build and tests (default: OFF)")
761
762
        string(
            name: 'hipTensor_branch',
763
764
            defaultValue: 'mainline',
            description: 'Specify which branch of hipTensor to use (default: mainline)')
765
766
767
768
        booleanParam(
            name: "USE_SCCACHE",
            defaultValue: true,
            description: "Use the sccache for building CK (default: ON)")
Illia Silin's avatar
Illia Silin committed
769
770
771
772
        booleanParam(
            name: "RUN_CPPCHECK",
            defaultValue: false,
            description: "Run the cppcheck static analysis (default: OFF)")
773
774
        booleanParam(
            name: "RUN_PERFORMANCE_TESTS",
775
776
            defaultValue: true,
            description: "Run the performance tests (default: ON)")
777
778
779
780
        booleanParam(
            name: "RUN_GROUPED_CONV_LARGE_CASES_TESTS",
            defaultValue: false,
            description: "Run the grouped conv large cases tests (default: OFF)")
781
782
783
784
        booleanParam(
            name: "RUN_CODEGEN_TESTS",
            defaultValue: false,
            description: "Run codegen tests (default: OFF)")
785
        booleanParam(
786
            name: "RUN_CK_TILE_FMHA_TESTS",
787
            defaultValue: false,
788
789
790
791
792
            description: "Run the ck_tile FMHA tests (default: OFF)")
        booleanParam(
            name: "RUN_CK_TILE_GEMM_TESTS",
            defaultValue: false,
            description: "Run the ck_tile GEMM tests (default: OFF)")
793
794
795
796
        booleanParam(
            name: "BUILD_INSTANCES_ONLY",
            defaultValue: false,
            description: "Test building instances for various architectures simultaneously (default: OFF)")
Illia Silin's avatar
Illia Silin committed
797
798
        booleanParam(
            name: "BUILD_GFX12",
799
800
            defaultValue: true,
            description: "Build CK and run tests on gfx12 (default: ON)")
801
802
803
804
        booleanParam(
            name: "NINJA_BUILD_TRACE",
            defaultValue: false,
            description: "Generate a ninja build trace (default: OFF)")
805
806
807
808
        booleanParam(
            name: "BUILD_LEGACY_OS",
            defaultValue: false,
            description: "Try building CK with legacy OS dockers: RHEL8 and SLES15 (default: OFF)")
809
810
811
812
813
814
815
816
    }
    environment{
        dbuser = "${dbuser}"
        dbpassword = "${dbpassword}"
        dbsship = "${dbsship}"
        dbsshport = "${dbsshport}"
        dbsshuser = "${dbsshuser}"
        dbsshpassword = "${dbsshpassword}"
817
        ck_git_creds = "${ck_git_creds}"
818
819
        gerrit_cred="${gerrit_cred}"
        DOCKER_BUILDKIT = "1"
820
    }
JD's avatar
JD committed
821
    stages{
822
823
824
825
826
827
        stage("Build Docker"){
            parallel{
                stage('Docker /opt/rocm'){
                    agent{ label rocmnode("nogpu") }
                    steps{
                        buildDocker('/opt/rocm')
828
                        cleanWs()
829
830
831
832
                    }
                }
            }
        }
JD's avatar
JD committed
833
834
        stage("Static checks") {
            parallel{
Illia Silin's avatar
Illia Silin committed
835
836
837
838
839
840
841
                stage('Clang Format and Cppcheck') {
                    when {
                        beforeAgent true
                        expression { params.RUN_CPPCHECK.toBoolean() }
                    }
                    agent{ label rocmnode("nogpu") }
                    environment{
842
                        setup_args = "NO_CK_BUILD"
Illia Silin's avatar
Illia Silin committed
843
844
845
846
847
848
849
850
851
                        execute_cmd = "find .. -not -path \'*.git*\' -iname \'*.h\' \
                                -o -not -path \'*.git*\' -iname \'*.hpp\' \
                                -o -not -path \'*.git*\' -iname \'*.cpp\' \
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\' && \
852
853
854
855
856
                                /cppcheck/build/bin/cppcheck ../* -v -j \$(nproc) -I ../include -I ../profiler/include -I ../library/include \
                                -D CK_ENABLE_FP64 -D CK_ENABLE_FP32 -D CK_ENABLE_FP16 -D CK_ENABLE_FP8 -D CK_ENABLE_BF16 -D CK_ENABLE_BF8 -D CK_ENABLE_INT8 -D DL_KERNELS \
                                -D __gfx908__ -D __gfx90a__ -D __gfx940__ -D __gfx941__ -D __gfx942__ -D __gfx1030__ -D __gfx1100__ -D __gfx1101__ -D __gfx1102__ \
                                -U __gfx803__ -U __gfx900__ -U __gfx906__ -U CK_EXPERIMENTAL_BIT_INT_EXTENSION_INT4 \
                                --file-filter=*.cpp --force --enable=all --output-file=ck_cppcheck.log"
Illia Silin's avatar
Illia Silin committed
857
858
                    }
                    steps{
859
                        buildHipClangJobAndReboot(setup_args:setup_args, setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
Illia Silin's avatar
Illia Silin committed
860
861
862
863
                        archiveArtifacts "build/ck_cppcheck.log"
                        cleanWs()
                    }
                }
JD's avatar
JD committed
864
                stage('Clang Format') {
Illia Silin's avatar
Illia Silin committed
865
866
867
868
                    when {
                        beforeAgent true
                        expression { !params.RUN_CPPCHECK.toBoolean() }
                    }
JD's avatar
JD committed
869
870
                    agent{ label rocmnode("nogpu") }
                    environment{
871
                        setup_args = "NO_CK_BUILD"
Illia Silin's avatar
Illia Silin committed
872
873
874
                        execute_cmd = "find .. -not -path \'*.git*\' -iname \'*.h\' \
                                -o -not -path \'*.git*\' -iname \'*.hpp\' \
                                -o -not -path \'*.git*\' -iname \'*.cpp\' \
JD's avatar
JD committed
875
876
877
878
879
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
880
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\'"
JD's avatar
JD committed
881
882
                    }
                    steps{
883
                        buildHipClangJobAndReboot(setup_args:setup_args, setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
884
                        cleanWs()
JD's avatar
JD committed
885
886
887
888
                    }
                }
            }
        }
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
        stage("Run Grouped Conv Large Case Tests")
        {
            parallel
            {
                stage("Run Grouped Conv Large Case Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_GROUPED_CONV_LARGE_CASES_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a")}
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
                                           make -j64 test_grouped_convnd_fwd_large_cases_xdl && \
                                           ./bin/test_grouped_convnd_fwd_large_cases_xdl"""
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
        }
        stage("Run Codegen Tests")
        {
            parallel
            {
                stage("Run Codegen Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CODEGEN_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a")}
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ CXX=/opt/rocm/llvm/bin/clang++ cmake ../codegen && \
                                           make -j64 check"""
                    }
929
930
931
932
933
934
935
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
        }
936
        stage("Run CK_TILE_FMHA Tests")
937
938
939
        {
            parallel
            {
940
                stage("Run CK_TILE_FMHA Tests on gfx90a")
941
942
943
                {
                    when {
                        beforeAgent true
944
                        expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() }
945
946
947
948
949
950
951
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
                                           make -j64 tile_example_fmha_fwd tile_example_fmha_bwd && \
                                           cd ../ &&
952
                                           example/ck_tile/01_fmha/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
953
                    }
954
955
956
957
958
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
959
                stage("Run CK_TILE_FMHA Tests on gfx942")
960
961
962
                {
                    when {
                        beforeAgent true
963
                        expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() }
964
965
966
967
968
969
970
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
                                           make -j64 tile_example_fmha_fwd tile_example_fmha_bwd && \
                                           cd ../ &&
971
                                           example/ck_tile/01_fmha/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
972
                    }
973
974
975
976
977
978
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
        }
        stage("Run CK_TILE_GEMM Tests")
        {
            parallel
            {
                stage("Run CK_TILE_GEMM Tests on gfx90a")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_GEMM_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx90a && \
994
                                           make -j64 tile_example_gemm_basic tile_example_gemm_universal && \
995
996
                                           cd ../ &&
                                           example/ck_tile/03_gemm/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx90a """
997
                    }
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
                stage("Run CK_TILE_GEMM Tests on gfx942")
                {
                    when {
                        beforeAgent true
                        expression { params.RUN_CK_TILE_GEMM_TESTS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
                        setup_args = "NO_CK_BUILD"
                        execute_args = """ ../script/cmake-ck-dev.sh  ../ gfx942 && \
1013
                                           make -j64 tile_example_gemm_basic tile_example_gemm_universal && \
1014
1015
                                           cd ../ &&
                                           example/ck_tile/03_gemm/script/run_full_test.sh "CI_${params.COMPILER_VERSION}" "${env.BRANCH_NAME}" "${NODE_NAME}" gfx942 """
1016
                    }
1017
1018
1019
1020
1021
1022
                    steps{
                        buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
            }
Paul Fultz II's avatar
Paul Fultz II committed
1023
        }
1024

1025
		stage("Build CK and run Tests")
1026
1027
1028
        {
            parallel
            {
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
                stage("Build CK with RHEL8")
                {
                    when {
                        beforeAgent true
                        expression { params.BUILD_LEGACY_OS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        def docker_name = "${env.CK_DOCKERHUB_PRIVATE}:ck_rhel8_rocm6.3"
                        setup_args = """ -DGPU_TARGETS="gfx942" \
                                         -DCMAKE_CXX_FLAGS=" -O3 " \
                                         -DCK_USE_ALTERNATIVE_PYTHON=/opt/Python-3.8.13/bin/python3.8 """
                        execute_args = " "
1042
                    }
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: " ", no_reboot:true, build_type: 'Release', docker_name: docker_name)
                        cleanWs()
                    }
                }
                stage("Build CK with SLES15")
                {
                    when {
                        beforeAgent true
                        expression { params.BUILD_LEGACY_OS.toBoolean() }
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        def docker_name = "${env.CK_DOCKERHUB_PRIVATE}:ck_sles15_rocm6.3"
                        setup_args = """ -DGPU_TARGETS="gfx942" \
                                         -DCMAKE_CXX_FLAGS=" -O3 " \
                                         -DCK_USE_ALTERNATIVE_PYTHON=/opt/Python-3.8.13/bin/python3.8 """
                        execute_args = " "
1061
                    }
1062
1063
1064
1065
1066
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: " ", no_reboot:true, build_type: 'Release', docker_name: docker_name)
                        cleanWs()
                    }
                }
1067
                stage("Build CK for all gfx9 targets")
1068
1069
1070
                {
                    when {
                        beforeAgent true
1071
                        expression { params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1072
                    }
1073
                    agent{ label rocmnode("gfx90a") }
1074
                    environment{
1075
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install \
1076
                                         -DGPU_TARGETS="gfx908;gfx90a;gfx942" \
1077
                                         -DCMAKE_CXX_FLAGS=" -O3 " """
1078
1079
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
1080
                                           -DGPU_TARGETS="gfx908;gfx90a;gfx942" \
1081
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1082
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1083
1084
                    }
                    steps{
1085
1086
1087
1088
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
                        cleanWs()
                    }
                }
Illia Silin's avatar
Illia Silin committed
1089
                stage("Build CK and run Tests on gfx942")
1090
1091
1092
                {
                    when {
                        beforeAgent true
1093
                        expression { params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1094
1095
1096
                    }
                    agent{ label rocmnode("gfx942") }
                    environment{
1097
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx942" -DCMAKE_CXX_FLAGS=" -O3 " """
1098
1099
1100
1101
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx942" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1102
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1103
1104
                    }
                    steps{
1105
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1106
                        cleanWs()
1107
1108
                    }
                }
Illia Silin's avatar
Illia Silin committed
1109
                stage("Build CK and run Tests on gfx90a")
1110
                {
1111
1112
                    when {
                        beforeAgent true
1113
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1114
                    }
1115
                    agent{ label rocmnode("gfx90a") }
1116
                    environment{
1117
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx90a" -DCMAKE_CXX_FLAGS=" -O3 " """
1118
1119
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
1120
                                           -DGPU_TARGETS="gfx90a" \
1121
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
1122
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
Illia Silin's avatar
Illia Silin committed
1123
1124
1125
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1126
                        cleanWs()
Illia Silin's avatar
Illia Silin committed
1127
1128
                    }
                }
1129
1130
1131
1132
                stage("Build CK instances for different targets")
                {
                    when {
                        beforeAgent true
1133
                        expression { params.BUILD_INSTANCES_ONLY.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1134
1135
1136
1137
1138
1139
                    }
                    agent{ label rocmnode("gfx90a") }
                    environment{
                        execute_args = """ cmake -D CMAKE_PREFIX_PATH=/opt/rocm \
                                           -D CMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -D CMAKE_BUILD_TYPE=Release \
1140
                                           -D GPU_ARCHS="gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102"  \
1141
                                           -D CMAKE_CXX_FLAGS=" -O3 " .. && make -j64 """
1142
                    }
1143
1144
1145
1146
1147
                    steps{
                        buildHipClangJobAndReboot(setup_cmd: "",  build_cmd: "", no_reboot:true, build_type: 'Release', execute_cmd: execute_args)
                        cleanWs()
                    }
                }
Illia Silin's avatar
Illia Silin committed
1148
                stage("Build CK and run Tests on gfx1030")
Illia Silin's avatar
Illia Silin committed
1149
1150
1151
                {
                    when {
                        beforeAgent true
1152
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
Illia Silin's avatar
Illia Silin committed
1153
                    }
Illia Silin's avatar
Illia Silin committed
1154
                    agent{ label rocmnode("gfx1030") }
Illia Silin's avatar
Illia Silin committed
1155
                    environment{
1156
1157
1158
1159
1160
1161
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1030" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """ 
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1030" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1162
                    }
1163
1164
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1165
                        cleanWs()
1166
1167
                    }
                }
Illia Silin's avatar
Illia Silin committed
1168
                stage("Build CK and run Tests on gfx1101")
1169
1170
1171
                {
                    when {
                        beforeAgent true
1172
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1173
                    }
Illia Silin's avatar
Illia Silin committed
1174
                    agent{ label rocmnode("gfx1101") }
1175
                    environment{
1176
1177
1178
1179
1180
1181
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1101" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1101" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
1182
                    }
1183
                    steps{
1184
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
1185
                        cleanWs()
JD's avatar
JD committed
1186
1187
                    }
                }
Illia Silin's avatar
Illia Silin committed
1188
1189
1190
1191
                stage("Build CK and run Tests on gfx1201")
                {
                    when {
                        beforeAgent true
1192
                        expression { params.BUILD_GFX12.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
Illia Silin's avatar
Illia Silin committed
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
                    }
                    agent{ label rocmnode("gfx1201") }
                    environment{
                        setup_args = """ -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx1201" -DDL_KERNELS=ON -DCMAKE_CXX_FLAGS=" -O3 " """
                        execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && \
                                           cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" \
                                           -DGPU_TARGETS="gfx1201" \
                                           -DCMAKE_CXX_COMPILER="${build_compiler()}" \
                                           -DCMAKE_CXX_FLAGS=" -O3 " .. && make -j """
                    }
                    steps{
                        Build_CK_and_Reboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
                        cleanWs()
                    }
                }
1208
1209
            }
        }
1210
1211
1212
1213
        stage("Process Performance Test Results")
        {
            parallel
            {
1214
                stage("Process results"){
1215
1216
                    when {
                        beforeAgent true
1217
                        expression { params.RUN_PERFORMANCE_TESTS.toBoolean() && !params.BUILD_LEGACY_OS.toBoolean() }
1218
                    }
1219
1220
                    agent { label 'mici' }
                    steps{
1221
                        process_results()
1222
                        cleanWs()
1223
1224
1225
1226
                    }
                }
            }
        }
JD's avatar
JD committed
1227
    }
1228
}