Jenkinsfile 9.72 KB
Newer Older
1
2
3
4
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils

DOCKER_IMAGE = 'rocm/migraphx-ci-ubuntu'

Chris Austen's avatar
Chris Austen committed
5
def getgputargets() {
6
    targets="gfx906;gfx908;gfx90a;gfx1030;gfx1100;gfx1101;gfx1102"
Chris Austen's avatar
Chris Austen committed
7
8
    return targets
}
Paul's avatar
Paul committed
9

10
11
12
13
14
15
16
17
// def rocmtestnode(variant, name, body, args, pre) {
def rocmtestnode(Map conf) {
    def variant = conf.get("variant")
    def name = conf.get("node")
    def body = conf.get("body")
    def docker_args = conf.get("docker_args", "")
    def docker_build_args = conf.get("docker_build_args", "")
    def pre = conf.get("pre", {})
Paul Fultz II's avatar
Paul Fultz II committed
18
    def ccache = "/var/jenkins/.cache/ccache"
Paul's avatar
Paul committed
19
    def image = 'migraphxlib'
Paul Fultz II's avatar
Paul Fultz II committed
20
21
    env.CCACHE_COMPRESSLEVEL = 7
    env.CCACHE_DIR = ccache
Paul Fultz II's avatar
Paul Fultz II committed
22
23
24
25
    def cmake_build = { bconf ->
        def compiler = bconf.get("compiler", "/opt/rocm/llvm/bin/clang++")
        def flags = bconf.get("flags", "")
        def gpu_debug = bconf.get("gpu_debug", "0")
Paul's avatar
Paul committed
26
        def cmd = """
Paul's avatar
Paul committed
27
            ulimit -c unlimited
Paul's avatar
Paul committed
28
29
            echo "leak:dnnl::impl::malloc" > suppressions.txt
            export LSAN_OPTIONS="suppressions=\$(pwd)/suppressions.txt"
Paul Fultz II's avatar
Paul Fultz II committed
30
31
32
33
            export MIGRAPHX_GPU_DEBUG=${gpu_debug}
            export CXX=${compiler}
            export CXXFLAGS='-Werror'
            env
Paul's avatar
Paul committed
34
35
36
            rm -rf build
            mkdir build
            cd build
37
            cmake -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DBUILD_DEV=On -DCMAKE_EXECUTE_PROCESS_COMMAND_ECHO=STDOUT ${flags} ..
38
39
40
41
42
43
            git diff
            git diff-index --quiet HEAD || (echo "Git repo is not clean after running cmake." && exit 1)
            make -j\$(nproc) generate VERBOSE=1
            git diff
            git diff-index --quiet HEAD || (echo "Generated files are different. Please run make generate and commit the changes." && exit 1)
            make -j\$(nproc) all doc package check VERBOSE=1
Chris Austen's avatar
Chris Austen committed
44
            md5sum ./*.deb
Paul's avatar
Paul committed
45
46
47
        """
        echo cmd
        sh cmd
Umang Yadav's avatar
Umang Yadav committed
48
49
50
        // Only archive from master or develop
        if (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "master") {
            archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true
Paul's avatar
Paul committed
51
        }
Paul's avatar
Paul committed
52
53
    }
    node(name) {
54
        withEnv(['HSA_ENABLE_SDMA=0']) {
Paul's avatar
Paul committed
55
56
57
            stage("checkout ${variant}") {
                checkout scm
            }
Paul's avatar
Paul committed
58
            gitStatusWrapper(credentialsId: "${env.status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'AMDMIGraphX') {
59
60
61
62
63
64
65
66
                withCredentials([usernamePassword(credentialsId: 'docker_test_cred', passwordVariable: 'DOCKERHUB_PASS', usernameVariable: 'DOCKERHUB_USER')]) {
                    sh "echo $DOCKERHUB_PASS | docker login --username $DOCKERHUB_USER --password-stdin"
                    pre()
                    sh "docker pull ${DOCKER_IMAGE}:${env.IMAGE_TAG}"
                    withDockerContainer(image: "${DOCKER_IMAGE}:${env.IMAGE_TAG}", args: "--device=/dev/kfd --device=/dev/dri --group-add video --cap-add SYS_PTRACE -v=/var/jenkins/:/var/jenkins ${docker_args}") {
                        timeout(time: 2, unit: 'HOURS') {
                            body(cmake_build)
                        }
Paul's avatar
Paul committed
67
                    }
Paul's avatar
Paul committed
68
                }
Paul's avatar
Paul committed
69
70
71
72
73
74
            }
        }
    }
}
def rocmtest(m) {
    def builders = [:]
Paul Fultz II's avatar
Paul Fultz II committed
75
    m.each { e ->
Paul's avatar
Paul committed
76
77
78
79
80
81
82
83
84
        def label = e.key;
        def action = e.value;
        builders[label] = {
            action(label)
        }
    }
    parallel builders
}

85
def rocmnodename(name) {
Paul's avatar
Paul committed
86
87
88
89
90
91
    def rocmtest_name = "(rocmtest || migraphx)"
    def node_name = "${rocmtest_name}"
    if(name == "fiji") {
        node_name = "${rocmtest_name} && fiji";
    } else if(name == "vega") {
        node_name = "${rocmtest_name} && vega";
92
93
    } else if(name == "navi21") {
        node_name = "${rocmtest_name} && navi21";
94
    } else if(name == "mi100+") {
95
        node_name = "${rocmtest_name} && (gfx908 || gfx90a) && !vm";
96
    } else if(name == "cdna") {
97
        node_name = "${rocmtest_name} && (gfx908 || gfx90a || vega20) && !vm";
Paul's avatar
Paul committed
98
    } else if(name == "nogpu") {
99
        node_name = "${rocmtest_name} && nogpu";
Paul's avatar
Paul committed
100
    }
101
    return node_name
Paul's avatar
Paul committed
102
103
}

Paul Fultz II's avatar
Paul Fultz II committed
104
def rocmnode(name, body) {
105
106
107
    return { label ->
        rocmtestnode(variant: label, node: rocmnodename(name), body: body)
    }
Paul Fultz II's avatar
Paul Fultz II committed
108
109
}

110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
properties([
    parameters([
        booleanParam(name: 'FORCE_DOCKER_IMAGE_BUILD', defaultValue: false)
    ])
])

node("(rocmtest || migraphx)") {
    Boolean imageExists = false
    withCredentials([usernamePassword(credentialsId: 'docker_test_cred', passwordVariable: 'DOCKERHUB_PASS', usernameVariable: 'DOCKERHUB_USER')]) {
        sh "echo $DOCKERHUB_PASS | docker login --username $DOCKERHUB_USER --password-stdin"
        stage('Check image') {
            checkout scm
            def calculateImageTagScript = """
                shopt -s globstar
                sha256sum **/Dockerfile **/*requirements.txt **/install_prereqs.sh **/rbuild.ini | sha256sum | cut -d " " -f 1
            """
            env.IMAGE_TAG = sh(script: "bash -c '${calculateImageTagScript}'", returnStdout: true).trim()
            imageExists = sh(script: "docker manifest inspect ${DOCKER_IMAGE}:${IMAGE_TAG}", returnStatus: true) == 0
        }
        stage('Build image') {
            if(!imageExists || params.FORCE_DOCKER_IMAGE_BUILD) {
                def builtImage

                try {
                    sh "docker pull ${DOCKER_IMAGE}:latest"
                    builtImage = docker.build("${DOCKER_IMAGE}:${IMAGE_TAG}", "--cache-from ${DOCKER_IMAGE}:latest .")
                } catch(Exception ex) {
                    builtImage = docker.build("${DOCKER_IMAGE}:${IMAGE_TAG}", " --no-cache .")
                }
                builtImage.push("${IMAGE_TAG}")
                builtImage.push("latest")
            } else {
                echo "Image already exists, skip building available"
                // Skip stage so it remains in the visualization
                Utils.markStageSkippedForConditional(STAGE_NAME)
            }
        }
    }
}

150
rocmtest clang_debug: rocmnode('mi100+') { cmake_build ->
151
    stage('hipRTC Debug') {
152
153
154
155
156
157
158
        // Disable MLIR since it doesnt work with all ub sanitizers
        withEnv(['MIGRAPHX_DISABLE_MLIR=1']) {
            def sanitizers = "undefined"
            def debug_flags = "-g -O2 -fsanitize=${sanitizers} -fno-sanitize-recover=${sanitizers}"
            def gpu_targets = getgputargets()
            cmake_build(flags: "-DCMAKE_BUILD_TYPE=debug -DMIGRAPHX_ENABLE_PYTHON=Off -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}' -DCMAKE_C_FLAGS_DEBUG='${debug_flags}' -DMIGRAPHX_USE_HIPRTC=On -DGPU_TARGETS='${gpu_targets}'", gpu_debug: true)
        }
Paul Fultz II's avatar
Paul Fultz II committed
159
    }
160
}, clang_release: rocmnode('mi100+') { cmake_build ->
161
    stage('Hip Clang Release') {
Chris Austen's avatar
Chris Austen committed
162
163
        def gpu_targets = getgputargets()
        cmake_build(flags: "-DCMAKE_BUILD_TYPE=release -DGPU_TARGETS='${gpu_targets}'")
Paul Fultz II's avatar
Paul Fultz II committed
164
        stash includes: 'build/*.deb', name: 'migraphx-package'
Paul's avatar
Paul committed
165
    }
166
167
168
169
// }, hidden_symbols: rocmnode('cdna') { cmake_build ->
//     stage('Hidden symbols') {
//         cmake_build(flags: "-DMIGRAPHX_ENABLE_PYTHON=Off -DMIGRAPHX_ENABLE_GPU=On -DMIGRAPHX_ENABLE_CPU=On -DCMAKE_CXX_VISIBILITY_PRESET=hidden -DCMAKE_C_VISIBILITY_PRESET=hidden")
//     }
170
}, all_targets_debug : rocmnode('mi100+') { cmake_build ->
171
    stage('All targets Release') {
Chris Austen's avatar
Chris Austen committed
172
173
        def gpu_targets = getgputargets()
        cmake_build(flags: "-DCMAKE_BUILD_TYPE=release -DMIGRAPHX_ENABLE_GPU=On -DMIGRAPHX_ENABLE_CPU=On -DMIGRAPHX_ENABLE_FPGA=On -DGPU_TARGETS='${gpu_targets}'")
174
    }
175
}, mlir_debug: rocmnode('mi100+') { cmake_build ->
176
    stage('MLIR Debug') {
177
        withEnv(['MIGRAPHX_ENABLE_EXTRA_MLIR=1']) {
178
            def sanitizers = "undefined"
179
180
181
            // Note: the -fno-sanitize= is copied from upstream LLVM_UBSAN_FLAGS.
            def debug_flags_cxx = "-g -O2 -fsanitize=${sanitizers} -fno-sanitize=vptr,function -fno-sanitize-recover=${sanitizers}"
            def debug_flags = "-g -O2 -fsanitize=${sanitizers} -fno-sanitize=vptr -fno-sanitize-recover=${sanitizers}"
Chris Austen's avatar
Chris Austen committed
182
183
            def gpu_targets = getgputargets()
            cmake_build(flags: "-DCMAKE_BUILD_TYPE=debug -DMIGRAPHX_ENABLE_PYTHON=Off -DMIGRAPHX_ENABLE_MLIR=On -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags_cxx}' -DCMAKE_C_FLAGS_DEBUG='${debug_flags}' -DGPU_TARGETS='${gpu_targets}'")
184
        }
185
    }
186
187
}, ck_hiprtc: rocmnode('mi100+') { cmake_build ->
    stage('CK hipRTC') {
188
        withEnv(['MIGRAPHX_ENABLE_CK=1', 'MIGRAPHX_TUNE_CK=1', 'MIGRAPHX_DISABLE_MLIR=1']) {
Chris Austen's avatar
Chris Austen committed
189
190
            def gpu_targets = getgputargets()
            cmake_build(flags: "-DCMAKE_BUILD_TYPE=release -DMIGRAPHX_USE_HIPRTC=On -DGPU_TARGETS='${gpu_targets}'")
191
192
        }
    }
Paul's avatar
Paul committed
193
194
195
}, clang_asan: rocmnode('nogpu') { cmake_build ->
    stage('Clang ASAN') {
        def sanitizers = "undefined,address"
Paul's avatar
Paul committed
196
        def debug_flags = "-g -O2 -fno-omit-frame-pointer -fsanitize=${sanitizers} -fno-sanitize-recover=${sanitizers}"
Chris Austen's avatar
Chris Austen committed
197
198
        def gpu_targets = getgputargets()
        cmake_build(flags: "-DCMAKE_BUILD_TYPE=debug -DMIGRAPHX_ENABLE_PYTHON=Off -DMIGRAPHX_ENABLE_GPU=Off -DMIGRAPHX_ENABLE_CPU=On -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}' -DCMAKE_C_FLAGS_DEBUG='${debug_flags}' -DGPU_TARGETS='${gpu_targets}'")
Paul's avatar
Paul committed
199
    }
200
201
}//, clang_release_navi: rocmnode('navi21') { cmake_build ->
//    stage('HIP Clang Release Navi') {
Paul Fultz II's avatar
Paul Fultz II committed
202
//        cmake_build(flags: "-DCMAKE_BUILD_TYPE=release")
203
204
//    }
//}
Paul Fultz II's avatar
Paul Fultz II committed
205

206
207
208
209
def onnxnode(name, body) {
    return { label ->
        rocmtestnode(variant: label, node: rocmnodename(name), docker_args: '-u root', body: body, pre: {
            sh 'rm -rf ./build/*.deb'
210
            unstash 'migraphx-package'
211
212
213
214
        })
    }
}

215
rocmtest onnx: onnxnode('mi100+') { cmake_build ->
Paul Fultz II's avatar
Paul Fultz II committed
216
217
    stage("Onnx runtime") {
        sh '''
218
            apt install half
Chris Austen's avatar
Chris Austen committed
219
220
            #ls -lR
            md5sum ./build/*.deb
221
            dpkg -i ./build/*.deb
222
            env
Paul Fultz II's avatar
Paul Fultz II committed
223
224
225
226
            cd /onnxruntime && ./build_and_test_onnxrt.sh
        '''
    }
}