Jenkinsfile 30.1 KB
Newer Older
JD's avatar
JD committed
1
2
3
4
5
6
7
8
9
10
11
12
13
def rocmnode(name) {
    return 'rocmtest && miopen && ' + name
}

def show_node_info() {
    sh """
        echo "NODE_NAME = \$NODE_NAME"
        lsb_release -sd
        uname -r
        ls /opt/ -la
    """
}

14
def runShell(String command){
15
    def responseCode = sh returnStatus: true, script: "${command} > tmp.txt"
16
    def output = readFile(file: "tmp.txt")
17
    echo "tmp.txt contents: $output"
18
19
20
    return (output != "")
}

21
def getDockerImageName(){
22
    def img = "${env.CK_IMAGE_URL}:ck_ub20.04_rocm5.2.3_${params.COMPILER_VERSION}"
23
24
25
    return img
}

26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
def build_compiler(){
    def compiler
    if (params.BUILD_COMPILER == "hipcc"){
        compiler = '/opt/rocm/bin/hipcc'
    }
    else{
        if (params.COMPILER_VERSION == "release"){
            compiler = "/opt/rocm/llvm/bin/clang++"
        }
        else{
            compiler = "/llvm-project/build/bin/clang++"
        }        
    }
    return compiler
}

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
def getDockerImage(Map conf=[:]){
    env.DOCKER_BUILDKIT=1
    def prefixpath = conf.get("prefixpath", "/opt/rocm") // prefix:/opt/rocm
    def gpu_arch = conf.get("gpu_arch", "gfx908") // prebuilt dockers should have all the architectures enabled so one image can be used for all stages
    def no_cache = conf.get("no_cache", false)
    def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' "
    if(env.CCACHE_HOST)
    {
        def check_host = sh(script:"""(printf "PING\r\n";) | nc -N ${env.CCACHE_HOST} 6379 """, returnStdout: true).trim()
        if(check_host == "+PONG")
        {
            echo "FOUND CCACHE SERVER: ${CCACHE_HOST}"
        }
        else 
        {
            echo "CCACHE SERVER: ${CCACHE_HOST} NOT FOUND, got ${check_host} response"
        }
        dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CCACHE_HOST}' --build-arg COMPILER_LAUNCHER='ccache' "
        env.CCACHE_DIR = """/tmp/ccache_store"""
        env.CCACHE_SECONDARY_STORAGE="""redis://${env.CCACHE_HOST}"""
    }
    if(no_cache)
    {
        dockerArgs = dockerArgs + " --no-cache "
    }
    echo "Docker Args: ${dockerArgs}"
    def image = getDockerImageName()
    //Check if image exists 
    def retimage
    try 
    {
        echo "Pulling down image: ${image}"
        retimage = docker.image("${image}")
        retimage.pull()
    }
    catch(Exception ex)
    {
        error "Unable to locate image: ${image}"
    }
    return [retimage, image]
}

def buildDocker(install_prefix){
    show_node_info()
    env.DOCKER_BUILDKIT=1
    checkout scm
    def image_name = getDockerImageName()
    echo "Building Docker for ${image_name}"
    def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${install_prefix} --build-arg compiler_version='${params.COMPILER_VERSION}' "
    if(env.CCACHE_HOST)
    {
        def check_host = sh(script:"""(printf "PING\\r\\n";) | nc  -N ${env.CCACHE_HOST} 6379 """, returnStdout: true).trim()
        if(check_host == "+PONG")
        {
            echo "FOUND CCACHE SERVER: ${CCACHE_HOST}"
        }
        else 
        {
            echo "CCACHE SERVER: ${CCACHE_HOST} NOT FOUND, got ${check_host} response"
        }
        dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CCACHE_HOST}' --build-arg COMPILER_LAUNCHER='ccache' "
        env.CCACHE_DIR = """/tmp/ccache_store"""
        env.CCACHE_SECONDARY_STORAGE="""redis://${env.CCACHE_HOST}"""
    }

    echo "Build Args: ${dockerArgs}"
    try{
        echo "Checking for image: ${image_name}"
        sh "docker manifest inspect --insecure ${image_name}"
        echo "Image: ${image_name} found!! Skipping building image"
    }
    catch(Exception ex){
        echo "Unable to locate image: ${image_name}. Building image now"
        retimage = docker.build("${image_name}", dockerArgs + ' .')
        retimage.push()
    }
}

JD's avatar
JD committed
120
121
def cmake_build(Map conf=[:]){

122
    def compiler = build_compiler()
JD's avatar
JD committed
123
124
    def config_targets = conf.get("config_targets","check")
    def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined " + conf.get("extradebugflags", "")
125
    def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","")
JD's avatar
JD committed
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
    def prefixpath = conf.get("prefixpath","/opt/rocm")
    def setup_args = conf.get("setup_args","")

    if (prefixpath != "/usr/local"){
        setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} "
    }

    def build_type_debug = (conf.get("build_type",'release') == 'debug')

    //cmake_env can overwrite default CXX variables.
    def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","")

    def package_build = (conf.get("package_build","") == "true")

    if (package_build == true) {
        config_targets = "package"
    }

    if(conf.get("build_install","") == "true")
    {
        config_targets = 'install ' + config_targets
        setup_args = ' -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=../install' + setup_args
    } else{
        setup_args = ' -DBUILD_DEV=On' + setup_args
    }

    if(build_type_debug){
        setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args
    }else{
        setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args
    }

    def pre_setup_cmd = """
            echo \$HSA_ENABLE_SDMA
            ulimit -c unlimited
            rm -rf build
            mkdir build
            rm -rf install
            mkdir install
            cd build
        """
    def setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args}   .. ")
Chao Liu's avatar
Chao Liu committed
168
    // reduce parallelism when compiling, clang uses too much memory
169
    def build_cmd = conf.get("build_cmd", "${build_envs} dumb-init make  -j\$(( \$(nproc) / 2 )) ${config_targets}")
JD's avatar
JD committed
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
    def execute_cmd = conf.get("execute_cmd", "")

    def cmd = conf.get("cmd", """
            ${pre_setup_cmd}
            ${setup_cmd}
            ${build_cmd}
            ${execute_cmd}
        """)

    echo cmd
    sh cmd

    // Only archive from master or develop
    if (package_build == true && (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "master")) {
        archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true
    }
}

def buildHipClangJob(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

194
        def image = "composable_kernels_${params.COMPILER_VERSION}"
JD's avatar
JD committed
195
196
197
198
199
200
201
        def prefixpath = conf.get("prefixpath", "/opt/rocm")
        def gpu_arch = conf.get("gpu_arch", "gfx908")

        // Jenkins is complaining about the render group 
        // def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
        def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
        if (conf.get("enforce_xnack_on", false)) {
202
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 --env GPU_ARCH='${gpu_arch}' "
JD's avatar
JD committed
203
        }
204
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' "
205
        if (params.COMPILER_VERSION != "release"){
206
207
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
JD's avatar
JD committed
208
209
210
211

        def variant = env.STAGE_NAME

        def retimage
212
213

        gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') {
214
            try {
215
216
                //retimage = docker.build("${image}", dockerArgs + '.')
                (retimage, image) = getDockerImage(conf)
217
218
                withDockerContainer(image: image, args: dockerOpts) {
                    timeout(time: 5, unit: 'MINUTES'){
219
220
                        sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo | tee clinfo.log'
                        if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
221
                            throw new Exception ("GPU not found")
222
223
224
225
                        }
                        else{
                            echo "GPU is OK"
                        }
JD's avatar
JD committed
226
227
228
                    }
                }
            }
229
230
231
232
233
234
235
236
            catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
                echo "The job was cancelled or aborted"
                throw e
            }
            catch(Exception ex) {
                retimage = docker.build("${image}", dockerArgs + " --no-cache .")
                withDockerContainer(image: image, args: dockerOpts) {
                    timeout(time: 5, unit: 'MINUTES'){
237
238
                        sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo |tee clinfo.log'
                        if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
239
                            throw new Exception ("GPU not found")
240
241
242
243
                        }
                        else{
                            echo "GPU is OK"
                        }
244
                    }
245
246
                }
            }
JD's avatar
JD committed
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277

            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
                timeout(time: 5, unit: 'HOURS')
                {
                    cmake_build(conf)
                }
            }
        }
        return retimage
}

def reboot(){
    build job: 'reboot-slaves', propagate: false , parameters: [string(name: 'server', value: "${env.NODE_NAME}"),]
}

def buildHipClangJobAndReboot(Map conf=[:]){
    try{
        buildHipClangJob(conf)
    }
    catch(e){
        echo "throwing error exception for the stage"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

278
279
280
281
282
283
def runCKProfiler(Map conf=[:]){
        show_node_info()

        env.HSA_ENABLE_SDMA=0
        checkout scm

284
285

        def image = "composable_kernels_${params.COMPILER_VERSION}"
286
287
288
289
290
291
292
        def prefixpath = conf.get("prefixpath", "/opt/rocm")
        def gpu_arch = conf.get("gpu_arch", "gfx908")

        // Jenkins is complaining about the render group 
        // def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
        def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
        if (conf.get("enforce_xnack_on", false)) {
293
            dockerOpts = dockerOpts + " --env HSA_XNACK=1 --env GPU_ARCH='${gpu_arch}' "
294
        }
295
        def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' "
296
        if (params.COMPILER_VERSION != "release"){
297
298
            dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
        }
299
300
301

        def variant = env.STAGE_NAME
        def retimage
302
303

        gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') {
304
            try {
305
306
                //retimage = docker.build("${image}", dockerArgs + '.')
                (retimage, image) = getDockerImage(conf)
307
308
                withDockerContainer(image: image, args: dockerOpts) {
                    timeout(time: 5, unit: 'MINUTES'){
309
310
                        sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo | tee clinfo.log'
                        if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
311
                            throw new Exception ("GPU not found")
312
313
314
315
                        }
                        else{
                            echo "GPU is OK"
                        }
316
317
318
                    }
                }
            }
319
320
321
322
323
324
325
326
            catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
                echo "The job was cancelled or aborted"
                throw e
            }
            catch(Exception ex) {
                retimage = docker.build("${image}", dockerArgs + " --no-cache .")
                withDockerContainer(image: image, args: dockerOpts) {
                    timeout(time: 5, unit: 'MINUTES'){
327
328
                        sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo | tee clinfo.log'
                        if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
329
                            throw new Exception ("GPU not found")
330
331
332
333
                        }
                        else{
                            echo "GPU is OK"
                        }
334
                    }
335
336
                }
            }
337
338

            withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
339
                timeout(time: 24, unit: 'HOURS')
340
341
342
                {
                    cmake_build(conf)
					dir("script"){
343
344
                        if (params.RUN_FULL_QA){
                            def qa_log = "qa_${gpu_arch}.log"
345
                            sh "./run_full_performance_tests.sh 1 QA_${params.COMPILER_VERSION} ${gpu_arch} ${env.BRANCH_NAME} ${NODE_NAME}"
346
347
348
                            archiveArtifacts "perf_gemm_${gpu_arch}.log"
                            archiveArtifacts "perf_resnet50_N256_${gpu_arch}.log"
                            archiveArtifacts "perf_resnet50_N4_${gpu_arch}.log"
349
                            archiveArtifacts "perf_batched_gemm_${gpu_arch}.log"
350
                            archiveArtifacts "perf_grouped_gemm_${gpu_arch}.log"
351
                            archiveArtifacts "perf_conv_fwd_${gpu_arch}.log"
352
                            archiveArtifacts "perf_conv_bwd_data_${gpu_arch}.log"
353
                            archiveArtifacts "perf_gemm_bilinear_${gpu_arch}.log"
354
                            archiveArtifacts "perf_reduction_${gpu_arch}.log"
355
356
                            archiveArtifacts "perf_splitK_gemm_${gpu_arch}.log"
                            archiveArtifacts "perf_onnx_gemm_${gpu_arch}.log"
357
358
359
360
                           // stash perf files to master
                            stash name: "perf_gemm_${gpu_arch}.log"
                            stash name: "perf_resnet50_N256_${gpu_arch}.log"
                            stash name: "perf_resnet50_N4_${gpu_arch}.log"
361
                            stash name: "perf_batched_gemm_${gpu_arch}.log"
362
                            stash name: "perf_grouped_gemm_${gpu_arch}.log"
363
                            stash name: "perf_conv_fwd_${gpu_arch}.log"
364
                            stash name: "perf_conv_bwd_data_${gpu_arch}.log"
365
                            stash name: "perf_gemm_bilinear_${gpu_arch}.log"
366
                            stash name: "perf_reduction_${gpu_arch}.log"
367
368
                            stash name: "perf_splitK_gemm_${gpu_arch}.log"
                            stash name: "perf_onnx_gemm_${gpu_arch}.log"
369
                            //we will process results on the master node
370
371
                        }
                        else{
372
                            sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${gpu_arch} ${env.BRANCH_NAME} ${NODE_NAME}"
373
374
375
376
377
378
379
380
                            archiveArtifacts "perf_gemm_${gpu_arch}.log"
                            archiveArtifacts "perf_resnet50_N256_${gpu_arch}.log"
                            archiveArtifacts "perf_resnet50_N4_${gpu_arch}.log"
                            // stash perf files to master
                            stash name: "perf_gemm_${gpu_arch}.log"
                            stash name: "perf_resnet50_N256_${gpu_arch}.log"
                            stash name: "perf_resnet50_N4_${gpu_arch}.log"
                            //we will process the results on the master node
381
                        }
382

383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
					}
                }
            }
        }
        return retimage
}

def runPerfTest(Map conf=[:]){
    try{
        runCKProfiler(conf)
    }
    catch(e){
        echo "throwing error exception in performance tests"
        echo 'Exception occurred: ' + e.toString()
        throw e
    }
    finally{
        if (!conf.get("no_reboot", false)) {
            reboot()
        }
    }
}

406
407
408
def process_results(Map conf=[:]){
    env.HSA_ENABLE_SDMA=0
    checkout scm
409
    def image = "composable_kernels_${params.COMPILER_VERSION}"
410
411
412
413
414
415
    def prefixpath = "/opt/rocm"
    def gpu_arch = conf.get("gpu_arch", "gfx908")

    // Jenkins is complaining about the render group 
    def dockerOpts="--cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
    if (conf.get("enforce_xnack_on", false)) {
416
        dockerOpts = dockerOpts + " --env HSA_XNACK=1 --env GPU_ARCH='${gpu_arch}' "
417
    }
418
    def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='release' "
419
420
421
422
423
424

    def variant = env.STAGE_NAME
    def retimage

    gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') {
        try {
425
426
            //retimage = docker.build("${image}", dockerArgs + '.')
            (retimage, image) = getDockerImage(conf)
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
        }
        catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
            echo "The job was cancelled or aborted"
            throw e
        }
    }

    withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
        timeout(time: 1, unit: 'HOURS'){
            try{
                dir("script"){
                    if (params.RUN_FULL_QA){
                        // unstash perf files to master
                        unstash "perf_gemm_${gpu_arch}.log"
                        unstash "perf_resnet50_N256_${gpu_arch}.log"
                        unstash "perf_resnet50_N4_${gpu_arch}.log"
443
                        unstash "perf_batched_gemm_${gpu_arch}.log"
444
                        unstash "perf_grouped_gemm_${gpu_arch}.log"
445
                        unstash "perf_conv_fwd_${gpu_arch}.log"
446
                        unstash "perf_conv_bwd_data_${gpu_arch}.log"
447
                        unstash "perf_gemm_bilinear_${gpu_arch}.log"
448
                        unstash "perf_reduction_${gpu_arch}.log"
449
450
                        unstash "perf_splitK_gemm_${gpu_arch}.log"
                        unstash "perf_onnx_gemm_${gpu_arch}.log"
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
                        sh "./process_qa_data.sh ${gpu_arch}"
                    }
                    else{
                        // unstash perf files to master
                        unstash "perf_gemm_${gpu_arch}.log"
                        unstash "perf_resnet50_N256_${gpu_arch}.log"
                        unstash "perf_resnet50_N4_${gpu_arch}.log"
                        sh "./process_perf_data.sh ${gpu_arch}"
                    }
                }
            }
            catch(e){
                echo "throwing error exception while processing performance test results"
                echo 'Exception occurred: ' + e.toString()
                throw e
            }
        }
    }
}

//launch develop branch daily at 23:00 in FULL_QA mode
472
CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true''' : ""
473

JD's avatar
JD committed
474
475
pipeline {
    agent none
476
477
478
    triggers {
        parameterizedCron(CRON_SETTINGS)
    }
JD's avatar
JD committed
479
480
481
    options {
        parallelsAlwaysFailFast()
    }
482
    parameters {
483
484
485
486
        booleanParam(
            name: "BUILD_DOCKER",
            defaultValue: true,
            description: "Force building docker image (default: true)")
487
488
489
        string(
            name: 'COMPILER_VERSION', 
            defaultValue: 'ck-9110', 
490
            description: 'Specify which version of compiler to use: ck-9110 (default), release, or amd-stg-open.')
491
492
493
494
        string(
            name: 'BUILD_COMPILER', 
            defaultValue: 'hipcc', 
            description: 'Specify whether to build CK with hipcc (default) or with clang.')
495
496
497
498
        booleanParam(
            name: "RUN_FULL_QA",
            defaultValue: false,
            description: "Select whether to run small set of performance tests (default) or full QA")
499
500
501
502
        booleanParam(
            name: "TEST_NODE_PERFORMANCE",
            defaultValue: false,
            description: "Test the node GPU performance (default: false)")
503
504
505
506
507
508
509
510
    }
    environment{
        dbuser = "${dbuser}"
        dbpassword = "${dbpassword}"
        dbsship = "${dbsship}"
        dbsshport = "${dbsshport}"
        dbsshuser = "${dbsshuser}"
        dbsshpassword = "${dbsshpassword}"
511
        status_wrapper_creds = "${status_wrapper_creds}"
512
513
        gerrit_cred="${gerrit_cred}"
        DOCKER_BUILDKIT = "1"
514
    }
JD's avatar
JD committed
515
    stages{
516
517
518
519
520
521
522
523
524
525
526
527
528
        stage("Build Docker"){
            when {
                expression { params.BUILD_DOCKER.toBoolean() }
            }
            parallel{
                stage('Docker /opt/rocm'){
                    agent{ label rocmnode("nogpu") }
                    steps{
                        buildDocker('/opt/rocm')
                    }
                }
            }
        }
JD's avatar
JD committed
529
        stage("Static checks") {
530
531
532
533
            when {
                beforeAgent true
                expression { !params.TEST_NODE_PERFORMANCE.toBoolean() }
            }
JD's avatar
JD committed
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
            parallel{
                // enable after we move from hipcc to hip-clang
                // stage('Tidy') {
                //     agent{ label rocmnode("nogpu") }
                //     environment{
                //         // setup_cmd = "CXX='/opt/rocm/bin/hipcc' cmake -DBUILD_DEV=On .. "
                //         build_cmd = "make -j\$(nproc) -k analyze"
                //     }
                //     steps{
                //         buildHipClangJobAndReboot(build_cmd: build_cmd, no_reboot:true, prefixpath: '/opt/rocm', build_type: 'debug')
                //     }
                // }
                stage('Clang Format') {
                    agent{ label rocmnode("nogpu") }
                    environment{
JD's avatar
JD committed
549
                        execute_cmd = "find .. -iname \'*.h\' \
JD's avatar
JD committed
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
                                -o -iname \'*.hpp\' \
                                -o -iname \'*.cpp\' \
                                -o -iname \'*.h.in\' \
                                -o -iname \'*.hpp.in\' \
                                -o -iname \'*.cpp.in\' \
                                -o -iname \'*.cl\' \
                                | grep -v 'build/' \
                                | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-10 -style=file {} | diff - {}\'"
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, no_reboot:true)
                    }
                }
            }
        }
565
		stage("Tests")
566
        {
567
568
569
570
            when {
                beforeAgent true
                expression { !params.TEST_NODE_PERFORMANCE.toBoolean() }
            }
571
572
573
574
575
576
            parallel
            {
                stage("Run Tests: gfx908")
                {
                    agent{ label rocmnode("gfx908")}
                    environment{
577
578
                        //setup_args = """ -D CMAKE_CXX_FLAGS=" --offload-arch=gfx908 -O3 " -DBUILD_DEV=On """
                        setup_args = "${params.COMPILER_VERSION == "release" ? """ -D CMAKE_CXX_FLAGS=" --offload-arch=gfx908 -O3 " -DBUILD_DEV=On """ : """ -D CMAKE_CXX_FLAGS=" --offload-arch=gfx908 -O3 -Xclang -mlink-builtin-bitcode -Xclang /opt/rocm/amdgcn/bitcode/oclc_abi_version_400.bc" -DBUILD_DEV=On """}"
579
580
                    }
                    steps{
581
                        buildHipClangJobAndReboot(setup_args:setup_args, config_targets: "check", no_reboot:true, build_type: 'Release', gpu_arch: "gfx908")
582
583
                    }
                }
JD's avatar
JD committed
584
585
                stage("Run Tests: gfx90a")
                {
586
587
588
589
                    when {
                        beforeAgent true
                        expression { params.RUN_FULL_QA.toBoolean() }
                    }
590
                    options { retry(2) }
JD's avatar
JD committed
591
592
                    agent{ label rocmnode("gfx90a")}
                    environment{
593
594
                        //setup_args = """ -D CMAKE_CXX_FLAGS="--offload-arch=gfx90a -O3 " -DBUILD_DEV=On """
                        setup_args = "${params.COMPILER_VERSION == "release" ? """ -D CMAKE_CXX_FLAGS=" --offload-arch=gfx90a -O3 " -DBUILD_DEV=On """ : """ -D CMAKE_CXX_FLAGS=" --offload-arch=gfx90a -O3 -Xclang -mlink-builtin-bitcode -Xclang /opt/rocm/amdgcn/bitcode/oclc_abi_version_400.bc" -DBUILD_DEV=On """}"
JD's avatar
JD committed
595
596
                    }
                    steps{
597
                        buildHipClangJobAndReboot(setup_args:setup_args, config_targets: "check", no_reboot:true, build_type: 'Release', gpu_arch: "gfx90a")
JD's avatar
JD committed
598
599
                    }
                }
600
601
            }
        }
Chao Liu's avatar
Chao Liu committed
602
603
        stage("Client App")
        {
604
605
606
607
            when {
                beforeAgent true
                expression { !params.TEST_NODE_PERFORMANCE.toBoolean() }
            }
Chao Liu's avatar
Chao Liu committed
608
609
610
611
612
613
            parallel
            {
                stage("Run Client App")
                {
                    agent{ label rocmnode("gfx908")}
                    environment{
614
615
616
617
618
                        //setup_args = """ -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=../install -D CMAKE_CXX_FLAGS="--offload-arch=gfx908 -O3 " """
                        setup_args = "${params.COMPILER_VERSION == "release" ? """ -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=../install -D CMAKE_CXX_FLAGS="--offload-arch=gfx908 -O3 " """ : """ -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=../install -D CMAKE_CXX_FLAGS="--offload-arch=gfx908 -O3 -Xclang -mlink-builtin-bitcode -Xclang /opt/rocm/amdgcn/bitcode/oclc_abi_version_400.bc" """ }"
                        //execute_args = """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -D CMAKE_CXX_FLAGS=" --offload-arch=gfx908 -O3" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """ 
                        execute_args = "${params.COMPILER_VERSION == "release" ? """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -D CMAKE_CXX_FLAGS=" --offload-arch=gfx908 -O3" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """ : """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -D CMAKE_CXX_FLAGS=" --offload-arch=gfx908 -O3 -Xclang -mlink-builtin-bitcode -Xclang /opt/rocm/amdgcn/bitcode/oclc_abi_version_400.bc" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """ }"

Chao Liu's avatar
Chao Liu committed
619
620
621
622
623
624
625
                    }
                    steps{
                        buildHipClangJobAndReboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
                    }
                }
            }
        }
626
627
628
629
630
631
        stage("Performance Tests")
        {
            parallel
            {
                stage("Run ckProfiler: gfx908")
                {
632
633
                    when {
                        beforeAgent true
634
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.TEST_NODE_PERFORMANCE.toBoolean() }
635
                    }
636
                    options { retry(2) }
637
638
                    agent{ label rocmnode("gfx908")}
                    environment{
639
640
                        //setup_args = """ -D CMAKE_CXX_FLAGS="--offload-arch=gfx908 -O3 " -DBUILD_DEV=On """
                        setup_args = "${params.COMPILER_VERSION == "release" ? """ -D CMAKE_CXX_FLAGS=" --offload-arch=gfx908 -O3 " -DBUILD_DEV=On """ : """ -D CMAKE_CXX_FLAGS=" --offload-arch=gfx908 -O3 -Xclang -mlink-builtin-bitcode -Xclang /opt/rocm/amdgcn/bitcode/oclc_abi_version_400.bc" -DBUILD_DEV=On """}"
641
                   }
642
                    steps{
643
644
645
646
647
                        runPerfTest(setup_args:setup_args, config_targets: "ckProfiler", no_reboot:true, build_type: 'Release', gpu_arch: "gfx908")
                    }
                }
                stage("Run ckProfiler: gfx90a")
                {
648
649
                    when {
                        beforeAgent true
650
                        expression { params.RUN_FULL_QA.toBoolean() || params.TEST_NODE_PERFORMANCE.toBoolean() }
651
                    }
652
                    options { retry(2) }
653
654
                    agent{ label rocmnode("gfx90a")}
                    environment{
655
656
657
                        //setup_args = """ -D CMAKE_CXX_FLAGS="--offload-arch=gfx90a -O3 " -DBUILD_DEV=On """
                        setup_args = "${params.COMPILER_VERSION == "release" ? """ -D CMAKE_CXX_FLAGS=" --offload-arch=gfx90a -O3 " -DBUILD_DEV=On """ : """ -D CMAKE_CXX_FLAGS=" --offload-arch=gfx90a -O3 -Xclang -mlink-builtin-bitcode -Xclang /opt/rocm/amdgcn/bitcode/oclc_abi_version_400.bc" -DBUILD_DEV=On """}"
                    }
658
659
                    steps{
                        runPerfTest(setup_args:setup_args, config_targets: "ckProfiler", no_reboot:true, build_type: 'Release', gpu_arch: "gfx90a")
660
661
662
663
                    }
                }
            }
        }
664
665
666
667
668
        stage("Process Performance Test Results")
        {
            parallel
            {
                stage("Process results for gfx908"){
669
670
                    when {
                        beforeAgent true
671
                        expression { !params.RUN_FULL_QA.toBoolean() && !params.TEST_NODE_PERFORMANCE.toBoolean() }
672
                    }
673
674
675
676
677
678
                    agent { label 'mici' }
                    steps{
                        process_results(gpu_arch: "gfx908")
                    }
                }
                stage("Process results for gfx90a"){
679
680
                    when {
                        beforeAgent true
681
                        expression { params.RUN_FULL_QA.toBoolean() || params.TEST_NODE_PERFORMANCE.toBoolean() }
682
                    }
683
684
685
686
687
688
689
690
                    agent { label 'mici' }
                    steps{
                        process_results(gpu_arch: "gfx90a")
                    }
                }
            }
        }

691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
        /* enable after the cmake file supports packaging
        stage("Packages") {
            when {
                expression { params.BUILD_PACKAGES && params.TARGET_NOGPU && params.DATATYPE_NA }
            }
            parallel {
                stage("Package /opt/rocm") {
                    agent{ label rocmnode("nogpu") }
                    steps{
                        buildHipClangJobAndReboot( package_build: "true", prefixpath: '/opt/rocm', gpu_arch: "gfx906;gfx908;gfx90a")
                    }
                }
            }
        }
        */
JD's avatar
JD committed
706
    }
707
}