Skip to content

GitLab

"git@developer.sourcefind.cn:change/sglang.git" did not exist on "75f4ccb7ddea2fd1abaa6475855da141b6c63980"
Unverified Commit 7fc3ed76 authored by Illia Silin's avatar Illia Silin Committed by GitHub
Browse files

Allow setting ROCM version, activate cchache, etc. (#462) 

* enable ccache and decouple it from MIOpen ccache use

* fix the ccache check script

* use another method to get server name

* fix syntax

* add quotes around the server name variable

* use check_host as function

* change syntax

* fix syntax

* test if server name is parsed correctly

* try different syntax

* check the env var value

* test new check node function

* add ROCMVERSION parameter and fix script syntax

* fix script syntax

* add missing instances of rocm version

* install ccache in the docker image

* do not check GPU in clang format stage, clean up old code

* update defaults and clean up
parent b8825547
Show whitespace changes
Inline Side-by-side
Showing with 49 additions and 111 deletions
Dockerfile
View file @ 7fc3ed76
FROM ubuntu:20.04 FROM ubuntu:20.04
ARG ROCMVERSION=5.2.3 ARG ROCMVERSION=5.3
ARG compiler_version ARG compiler_version="release"
ARG compiler_commit ARG compiler_commit=""
RUN set -xe RUN set -xe
...@@ -19,6 +19,7 @@ RUN sh -c "echo deb http://mirrors.kernel.org/ubuntu focal main universe | tee - ...@@ -19,6 +19,7 @@ RUN sh -c "echo deb http://mirrors.kernel.org/ubuntu focal main universe | tee -
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \
apt-utils \ apt-utils \
build-essential \ build-essential \
ccache \
cmake-data \ cmake-data \
cmake \ cmake \
curl \ curl \
......
Jenkinsfile
View file @ 7fc3ed76
...@@ -19,10 +19,24 @@ def runShell(String command){ ...@@ -19,10 +19,24 @@ def runShell(String command){
} }
def getDockerImageName(){ def getDockerImageName(){
def img = "${env.CK_DOCKERHUB}:ck_ub20.04_rocm5.2.3_${params.COMPILER_VERSION}" def img = "${env.CK_DOCKERHUB}:ck_ub20.04_rocm${params.ROCMVERSION}_${params.COMPILER_VERSION}"
return img return img
} }
def check_host() {
if ("${env.CK_CCACHE}" != "null"){
def CCACHE_SERVER="${env.CK_CCACHE.split(':')[0]}"
echo "ccache server: ${CCACHE_SERVER}"
sh '''ping -c 1 -p 6379 "${CCACHE_SERVER}" | echo $? > tmp.txt'''
def output = readFile(file: "tmp.txt")
echo "tmp.txt contents: \$output"
return (output != "0")
}
else{
return 1
}
}
def build_compiler(){ def build_compiler(){
def compiler def compiler
if (params.BUILD_COMPILER == "hipcc"){ if (params.BUILD_COMPILER == "hipcc"){
...@@ -43,21 +57,21 @@ def getDockerImage(Map conf=[:]){ ...@@ -43,21 +57,21 @@ def getDockerImage(Map conf=[:]){
env.DOCKER_BUILDKIT=1 env.DOCKER_BUILDKIT=1
def prefixpath = conf.get("prefixpath", "/opt/rocm") // prefix:/opt/rocm def prefixpath = conf.get("prefixpath", "/opt/rocm") // prefix:/opt/rocm
def no_cache = conf.get("no_cache", false) def no_cache = conf.get("no_cache", false)
def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' " def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
if(env.CCACHE_HOST) echo "ccache server: ${env.CK_CCACHE}"
if(env.CK_CCACHE)
{ {
def check_host = sh(script:"""(printf "PING\r\n";) | nc -N ${env.CCACHE_HOST} 6379 """, returnStdout: true).trim() if(check_host())
if(check_host == "+PONG")
{ {
echo "FOUND CCACHE SERVER: ${CCACHE_HOST}" echo "FOUND CCACHE SERVER: ${env.CK_CCACHE}"
} }
else else
{ {
echo "CCACHE SERVER: ${CCACHE_HOST} NOT FOUND, got ${check_host} response" echo "CCACHE SERVER: ${env.CK_CCACHE} NOT FOUND, got ${check_host} response"
} }
dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CCACHE_HOST}' --build-arg COMPILER_LAUNCHER='ccache' " dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CK_CCACHE}' --build-arg COMPILER_LAUNCHER='ccache' "
env.CCACHE_DIR = """/tmp/ccache_store""" env.CCACHE_DIR = """/tmp/ccache_store"""
env.CCACHE_SECONDARY_STORAGE="""redis://${env.CCACHE_HOST}""" env.CCACHE_SECONDARY_STORAGE="""redis://${env.CK_CCACHE}"""
} }
if(no_cache) if(no_cache)
{ {
...@@ -86,21 +100,21 @@ def buildDocker(install_prefix){ ...@@ -86,21 +100,21 @@ def buildDocker(install_prefix){
checkout scm checkout scm
def image_name = getDockerImageName() def image_name = getDockerImageName()
echo "Building Docker for ${image_name}" echo "Building Docker for ${image_name}"
def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${install_prefix} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}'" def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${install_prefix} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
if(env.CCACHE_HOST) echo "ccache server: ${env.CK_CCACHE}"
if(env.CK_CCACHE)
{ {
def check_host = sh(script:"""(printf "PING\\r\\n";) | nc -N ${env.CCACHE_HOST} 6379 """, returnStdout: true).trim() if(check_host())
if(check_host == "+PONG")
{ {
echo "FOUND CCACHE SERVER: ${CCACHE_HOST}" echo "FOUND CCACHE SERVER: ${env.CK_CCACHE}"
} }
else else
{ {
echo "CCACHE SERVER: ${CCACHE_HOST} NOT FOUND, got ${check_host} response" echo "CCACHE SERVER: ${env.CK_CCACHE} NOT FOUND, got ${check_host} response"
} }
dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CCACHE_HOST}' --build-arg COMPILER_LAUNCHER='ccache' " dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CK_CCACHE}' --build-arg COMPILER_LAUNCHER='ccache' "
env.CCACHE_DIR = """/tmp/ccache_store""" env.CCACHE_DIR = """/tmp/ccache_store"""
env.CCACHE_SECONDARY_STORAGE="""redis://${env.CCACHE_HOST}""" env.CCACHE_SECONDARY_STORAGE="""redis://${env.CK_CCACHE}"""
} }
echo "Build Args: ${dockerArgs}" echo "Build Args: ${dockerArgs}"
...@@ -161,10 +175,11 @@ def cmake_build(Map conf=[:]){ ...@@ -161,10 +175,11 @@ def cmake_build(Map conf=[:]){
}else{ }else{
setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args
} }
if(env.CCACHE_HOST) if(env.CK_CCACHE)
{ {
setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER='ccache' -DCMAKE_C_COMPILER_LAUNCHER='ccache' " + setup_args setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER='ccache' -DCMAKE_C_COMPILER_LAUNCHER='ccache' " + setup_args
} }
echo "ccache server: ${env.CK_CCACHE}"
def pre_setup_cmd = """ def pre_setup_cmd = """
echo \$HSA_ENABLE_SDMA echo \$HSA_ENABLE_SDMA
...@@ -210,7 +225,7 @@ def buildHipClangJob(Map conf=[:]){ ...@@ -210,7 +225,7 @@ def buildHipClangJob(Map conf=[:]){
if (conf.get("enforce_xnack_on", false)) { if (conf.get("enforce_xnack_on", false)) {
dockerOpts = dockerOpts + " --env HSA_XNACK=1 " dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
} }
def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' " def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
if (params.COMPILER_VERSION != "release"){ if (params.COMPILER_VERSION != "release"){
dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' " dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
} }
...@@ -220,39 +235,6 @@ def buildHipClangJob(Map conf=[:]){ ...@@ -220,39 +235,6 @@ def buildHipClangJob(Map conf=[:]){
def retimage def retimage
gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') { gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') {
try {
(retimage, image) = getDockerImage(conf)
withDockerContainer(image: image, args: dockerOpts) {
timeout(time: 5, unit: 'MINUTES'){
sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo | tee clinfo.log'
if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
throw new Exception ("GPU not found")
}
else{
echo "GPU is OK"
}
}
}
}
catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){
echo "The job was cancelled or aborted"
throw e
}
catch(Exception ex) {
retimage = docker.build("${image}", dockerArgs + " --no-cache .")
withDockerContainer(image: image, args: dockerOpts) {
timeout(time: 5, unit: 'MINUTES'){
sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo |tee clinfo.log'
if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){
throw new Exception ("GPU not found")
}
else{
echo "GPU is OK"
}
}
}
}
withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') { withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') {
timeout(time: 5, unit: 'HOURS') timeout(time: 5, unit: 'HOURS')
{ {
...@@ -297,7 +279,7 @@ def runCKProfiler(Map conf=[:]){ ...@@ -297,7 +279,7 @@ def runCKProfiler(Map conf=[:]){
if (conf.get("enforce_xnack_on", false)) { if (conf.get("enforce_xnack_on", false)) {
dockerOpts = dockerOpts + " --env HSA_XNACK=1 " dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
} }
def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' " def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
if (params.COMPILER_VERSION != "release"){ if (params.COMPILER_VERSION != "release"){
dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' " dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
} }
...@@ -430,7 +412,7 @@ def Build_CK(Map conf=[:]){ ...@@ -430,7 +412,7 @@ def Build_CK(Map conf=[:]){
if (conf.get("enforce_xnack_on", false)) { if (conf.get("enforce_xnack_on", false)) {
dockerOpts = dockerOpts + " --env HSA_XNACK=1 " dockerOpts = dockerOpts + " --env HSA_XNACK=1 "
} }
def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' " def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' "
if (params.COMPILER_VERSION != "release"){ if (params.COMPILER_VERSION != "release"){
dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' " dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' "
} }
...@@ -581,15 +563,19 @@ pipeline { ...@@ -581,15 +563,19 @@ pipeline {
booleanParam( booleanParam(
name: "BUILD_DOCKER", name: "BUILD_DOCKER",
defaultValue: false, defaultValue: false,
description: "Force building docker image (default: false)") description: "Force building docker image (default: false), set to true if docker image needs to be updated.")
string(
name: 'ROCMVERSION',
defaultValue: '5.3',
description: 'Specify which ROCM version to use: 5.2.3, or 5.3 (default), etc.')
string( string(
name: 'COMPILER_VERSION', name: 'COMPILER_VERSION',
defaultValue: 'amd-stg-open', defaultValue: 'release',
description: 'Specify which version of compiler to use: ck-9110, release, or amd-stg-open (default).') description: 'Specify which version of compiler to use: ck-9110, release (default), or amd-stg-open.')
string( string(
name: 'COMPILER_COMMIT', name: 'COMPILER_COMMIT',
defaultValue: '8a82e4eb7ba28521ba9a9424a0315a8a16590424', defaultValue: '',
description: 'Specify which commit of compiler branch to use: leave empty to use the latest commit, or use 10738 commit (default).') description: 'Specify which commit of compiler branch to use: leave empty to use the latest commit (default), or use 8a82e4eb7ba28521ba9a9424a0315a8a16590424 commit of amd-stg-open branch.')
string( string(
name: 'BUILD_COMPILER', name: 'BUILD_COMPILER',
defaultValue: 'hipcc', defaultValue: 'hipcc',
...@@ -627,17 +613,6 @@ pipeline { ...@@ -627,17 +613,6 @@ pipeline {
} }
stage("Static checks") { stage("Static checks") {
parallel{ parallel{
// enable after we move from hipcc to hip-clang
// stage('Tidy') {
// agent{ label rocmnode("nogpu") }
// environment{
// // setup_cmd = "CXX='/opt/rocm/bin/hipcc' cmake -DBUILD_DEV=On .. "
// build_cmd = "make -j\$(nproc) -k analyze"
// }
// steps{
// buildHipClangJobAndReboot(build_cmd: build_cmd, no_reboot:true, prefixpath: '/opt/rocm', build_type: 'debug')
// }
// }
stage('Clang Format') { stage('Clang Format') {
agent{ label rocmnode("nogpu") } agent{ label rocmnode("nogpu") }
environment{ environment{
...@@ -676,28 +651,6 @@ pipeline { ...@@ -676,28 +651,6 @@ pipeline {
} }
} }
/*
//at present this stage only builds binaries.
//we will now build all binaries in a separate stage.
//once we have some tests to run in this stage, we can enable it again.
stage("Client App")
{
parallel
{
stage("Run Client App")
{
agent{ label rocmnode("gfx908")}
environment{
setup_args = "${params.COMPILER_VERSION == "ck-9110" ? """ -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx908;gfx90a" -DCMAKE_CXX_FLAGS="-O3 -Xclang -mlink-builtin-bitcode -Xclang /opt/rocm/amdgcn/bitcode/oclc_abi_version_400.bc" """ : """ -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx908;gfx90a" -DCMAKE_CXX_FLAGS="-O3 " """ }"
execute_args = "${params.COMPILER_VERSION == "ck-9110" ? """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -DGPU_TARGETS="gfx908;gfx90a" -DCMAKE_CXX_FLAGS="-O3 -Xclang -mlink-builtin-bitcode -Xclang /opt/rocm/amdgcn/bitcode/oclc_abi_version_400.bc" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """ : """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -DGPU_TARGETS="gfx908;gfx90a" -DCMAKE_CXX_FLAGS="-O3" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """ }"
}
steps{
buildHipClangJobAndReboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
}
}
}
}
*/
stage("Performance Tests") stage("Performance Tests")
{ {
parallel parallel
...@@ -746,21 +699,5 @@ pipeline { ...@@ -746,21 +699,5 @@ pipeline {
} }
} }
} }
/* enable after the cmake file supports packaging
stage("Packages") {
when {
expression { params.BUILD_PACKAGES && params.TARGET_NOGPU && params.DATATYPE_NA }
}
parallel {
stage("Package /opt/rocm") {
agent{ label rocmnode("nogpu") }
steps{
buildHipClangJobAndReboot( package_build: "true", prefixpath: '/opt/rocm', gpu_arch: "gfx906;gfx908;gfx90a")
}
}
}
}
*/
} }
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment