Skip to content

GitLab

Unverified Commit ceb4ca09 authored by Paul Fultz II's avatar Paul Fultz II Committed by GitHub
Browse files

Revamp CI infrastucture (#706) 



* Add build and test github workflow

* Fix cget command

* Remove def-requirements.txt

* Add tmate session to debug workflow

* Run tmate session after installing dependencies

* Print date periodically

* Add clang tidy action

* Seperate build and run container in two different jobs

* Run bash script

* Remove interactive flag

* Try to mount the files

* Try to use the github workspace

* WIthout double braces

* Use env variable

* Pipe bash script in

* Run using hip-clang

* Use correct path

* Add verbose

* Remove j flag

* Only run for onnx file to debug

* Manually run clang-tidy

* Remove quiet flag

* Print header file

* Printout environment

* Remove extra defines

* Remove fixits and config flag

* Show ldd

* Add tmate session

* Run onnx protobuf first

* Generate proto for tensorflow

* Update cppcheck version

* Fix some cppcheck issues

* Add const

* Cppcheck fixes

* Formatting

* Fix more cppcheck issues

* Run two jobs

* Cache analysis and run format checking

* Fix yaml issues

* Fix yaml issues

* Fix indentation

* Switch to hip-clang for main docker file

* Use hip-clang in the readme

* Fixes for jenkins

* Use ccache to build

* Combine file

* Set restore keys

* Change stage name

* Build with ccache

* Add missing dependency for ccache

* Build debug with codecov

* Fix workflow syntax

* Fix list

* Use quotes

* Got to correct build path

* Install lcov

* Use sudo

* Echo all commands

* Setup tmate

* Add verbose output

* Build with cmake directly

* Add pthread flag

* Remove python config

* Continue on error

* Use on or off for cmake flag

* Use always upload cache

* Verbose output

* Verbose output from build

* Build one target

* Reduce debug symbols

* Increase garbage collection

* Remove dmesg

* Increase it to 20

* Update rocm cmake version

* Remove jobs from jenkins

* Run on all 3 ubuntus

* Remove gcc 5 jobs

* Dont add flag on 16.04

* Only upload coverage on 18.04

* Dont build for ubuntu 20.04

* Use matrix.os

* Use O2 for hip-clang since lower optimizations are broken

* Use rocm 3.0

* Pass ccache as cmake variable instead of env variable

* Build miopen from source

* Show ccache statistics

* Print log information

* Set compression level

* Use hash dir

* Set hashdir

* Install clang ocl from system

* Up compression level

* Add locale

* Increase cache size to 1G

* Lower compression level to 9

* Remove split dwarf

* Remove Og

* Add back Og

* Seperate debug and codecov

* Add missing backlash

* Garbage collect more often

* Add missing locales package

* Use Os

* Install onednn in docker and run tests

* Include target headers in tests

* Increase timeout

* Remove if condtion

* Make flag public

* Suppress memory leaks in onednn

* Use equal

* Add gh annotations

* Update rocm-cmake version

* Add ldconfig
Co-authored-by: default avatarShucai Xiao <shucai@gmail.com>
parent 1c8fcfc9
Hide whitespace changes
Inline Side-by-side
Showing with 406 additions and 161 deletions
.github/workflows/ci.yaml 0 → 100755
View file @ ceb4ca09
name: migraphx
on: push
jobs:
cancel:
runs-on: ubuntu-latest
steps:
- name: Cancel Previous Runs
uses: styfle/cancel-workflow-action@0.6.0
with:
access_token: ${{ github.token }}
tidy:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
# In this step, this action saves a list of existing images,
# the cache is created without them in the post run.
# It also restores the cache if it exists.
- uses: satackey/action-docker-layer-caching@v0.0.8
# Ignore the failure of a step and avoid terminating the job.
continue-on-error: true
- name: Prepare timestamp
id: cache_timestamp
shell: cmake -P {0}
run: |
string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
message("::set-output name=timestamp::${current_date}")
- name: Cache files for tidy
uses: pat-s/always-upload-cache@v2.1.3
with:
path: tidy-cache
key: tidy-cache-${{ steps.cache_timestamp.outputs.timestamp }}
restore-keys: |
tidy-cache-${{ steps.cache_timestamp.outputs.timestamp }}
tidy-cache-
- name: Build the Docker image
run: docker build . --file hip-clang.docker --tag migraphx
- name: Clang tidy
shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data migraphx bash < {0}"
run: |
mkdir build
cd build
CXX=/opt/rocm/llvm/bin/clang++ CC=/opt/rocm/llvm/bin/clang cmake \
-DMIGRAPHX_ENABLE_GPU=On \
-DMIGRAPHX_ENABLE_CPU=On \
-DROCM_ENABLE_GH_ANNOTATIONS=On \
-DCLANG_TIDY_DEPEND_ON_TARGET=Off \
-DCLANG_TIDY_CACHE=/data/tidy-cache \
..
make -j2 -k onnx-proto tf-proto tidy
cppcheck:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
# In this step, this action saves a list of existing images,
# the cache is created without them in the post run.
# It also restores the cache if it exists.
- uses: satackey/action-docker-layer-caching@v0.0.8
# Ignore the failure of a step and avoid terminating the job.
continue-on-error: true
- name: Prepare timestamp
id: cache_timestamp
shell: cmake -P {0}
run: |
string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
message("::set-output name=timestamp::${current_date}")
- name: Cache files for cppcheck
uses: pat-s/always-upload-cache@v2.1.3
with:
path: cppcheck-cache
key: cppcheck-cache-${{ steps.cache_timestamp.outputs.timestamp }}
restore-keys: |
cppcheck-cache-${{ steps.cache_timestamp.outputs.timestamp }}
cppcheck-cache-
- name: Build the Docker image
run: docker build . --file hip-clang.docker --tag migraphx
- name: Cppcheck
shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data migraphx bash < {0}"
run: |
mkdir build
cd build
CXX=/opt/rocm/llvm/bin/clang++ CC=/opt/rocm/llvm/bin/clang cmake \
-DCPPCHECK_BUILD_DIR=/data/cppcheck-cache \
-DROCM_ENABLE_GH_ANNOTATIONS=On \
..
make -j2 cppcheck
format:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
# In this step, this action saves a list of existing images,
# the cache is created without them in the post run.
# It also restores the cache if it exists.
- uses: satackey/action-docker-layer-caching@v0.0.8
# Ignore the failure of a step and avoid terminating the job.
continue-on-error: true
- name: Build the Docker image
run: docker build . --file hip-clang.docker --tag migraphx
- name: Check formatting
shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data migraphx bash < {0}"
run: |
find . -iname '*.h' \
-o -iname '*.hpp' \
-o -iname '*.cpp' \
-o -iname '*.h.in' \
-o -iname '*.hpp.in' \
-o -iname '*.cpp.in' \
-o -iname '*.cl' \
| grep -v 'build/' \
| xargs -n 1 -P 1 -I{} -t sh -c 'clang-format-5.0 -style=file {} | diff - {}'
find . -iname '*.py' \
| grep -v 'build/' \
| xargs -n 1 -P 1 -I{} -t sh -c 'yapf {} | diff - {}'
linux:
runs-on: ${{ matrix.os }}
env:
CCACHE_COMPRESSLEVEL: 10
CCACHE_DIR: ${{github.workspace}}/ccache
CCACHE_NOHASHDIR: true
CCACHE_BASEDIR: ${{github.workspace}}
CCACHE_MAXSIZE: 1
strategy:
matrix:
os:
- ubuntu-16.04
- ubuntu-18.04
configuration:
- debug
- release
- codecov
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.6
- name: Cache dependencies
# Ignore the failure of a step and avoid terminating the job.
continue-on-error: true
uses: actions/cache@v2
with:
# This path is specific to Ubuntu
path: ${{ github.workspace }}/cget
# Look to see if there is a cache hit for the corresponding requirements file
key: ${{ matrix.os }}-cget-2-${{ hashFiles('requirements.txt') }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install cget
cget install pfultz2/rocm-recipes
cget ignore ROCmSoftwarePlatform/rocBLAS ROCmSoftwarePlatform/MIOpen
cget install -f requirements.txt
cget install oneapi-src/oneDNN@v1.7
cget install facebook/zstd@v1.4.5 -X subdir -DCMAKE_DIR=build/cmake
cget install ccache@v4.1
- name: Prepare timestamp
id: cache_timestamp
shell: cmake -P {0}
run: |
string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
message("::set-output name=timestamp::${current_date}")
- name: Cache files for ccache
# Ignore the failure of a step and avoid terminating the job.
continue-on-error: true
uses: pat-s/always-upload-cache@v2.1.3
with:
path: ccache
key: ${{ matrix.os }}-${{ matrix.configuration }}-ccache-${{ steps.cache_timestamp.outputs.timestamp }}
restore-keys: |
${{ matrix.os }}-${{ matrix.configuration }}-ccache-${{ steps.cache_timestamp.outputs.timestamp }}
${{ matrix.os }}-${{ matrix.configuration }}-ccache-
- name: Build and test
env:
CMAKE_PREFIX_PATH: ${{ github.workspace }}/cget
CCACHE_LOGFILE: /tmp/ccache.log
CXXFLAGS: -Werror -pthread --param ggc-min-expand=5 --param ggc-min-heapsize=8192
run: |
echo "leak:dnnl::impl::malloc" > suppressions.txt
export LSAN_OPTIONS="suppressions=$(pwd)/suppressions.txt"
mkdir build
cd build
cmake .. \
-DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/cget/bin/ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/cget/bin/ccache \
-DCMAKE_BUILD_TYPE=${{matrix.configuration}} \
-DMIGRAPHX_ENABLE_CPU=On \
-DMIGRAPHX_ENABLE_PYTHON=${{matrix.configuration == 'release' && 'On' || 'Off'}} \
-DCMAKE_CXX_FLAGS_DEBUG="-g1 -Os -fdebug-prefix-map=$PWD=. -fdebug-types-section -fno-omit-frame-pointer ${{matrix.os != 'ubuntu-16.04' && '-fsanitize-address-use-after-scope' || ''}} -fsanitize=undefined,address -fno-sanitize-recover=undefined,address" \
-DCMAKE_CXX_FLAGS_CODECOV="-g1 -Og -fdebug-prefix-map=$PWD=. -fdebug-types-section -fprofile-arcs -ftest-coverage -fno-omit-frame-pointer" \
-DCMAKE_EXE_LINKER_FLAGS='-fuse-ld=gold' \
-DCMAKE_SHARED_LINKER_FLAGS='-fuse-ld=gold'
make -j2 check
${{ github.workspace }}/cget/bin/ccache -s
- name: Upload code coverage
if: "matrix.configuration == 'codecov'"
env:
CODECOV_TOKEN: "8545af1c-f90b-4345-92a5-0d075503ca56"
run: |
sudo apt-get install -y lcov
cd build
lcov --directory . --capture --output-file $(pwd)/coverage.info
lcov --remove $(pwd)/coverage.info '/usr/*' --output-file $(pwd)/coverage.info
lcov --list $(pwd)/coverage.info
curl -s https://codecov.io/bash | bash
echo "Uploaded"
CMakeLists.txt
View file @ ceb4ca09
......@@ -165,6 +165,7 @@ rocm_enable_cppcheck(
unusedStructMember
functionStatic
functionConst:*program.*
functionConst:*module.*
shadowFunction
shadowVar
shadowVariable
......
Dockerfile
View file @ ceb4ca09
......@@ -6,55 +6,43 @@ ARG PREFIX=/usr/local
RUN dpkg --add-architecture i386
# Add rocm repository
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl apt-utils wget software-properties-common
RUN curl https://raw.githubusercontent.com/RadeonOpenCompute/ROCm-docker/master/add-rocm.sh | bash
# Add ubuntu toolchain
RUN apt-get update && add-apt-repository ppa:ubuntu-toolchain-r/test -y
RUN sh -c 'echo deb [arch=amd64 trusted=yes] http://repo.radeon.com/rocm/apt/.apt_3.7/ xenial main > /etc/apt/sources.list.d/rocm.list'
# Install dependencies
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \
apt-utils \
build-essential \
clang-5.0 \
clang-format-5.0 \
clang-tidy-5.0 \
cmake \
comgr \
curl \
doxygen \
g++-5 \
g++-7 \
gdb \
git \
hsa-rocr-dev \
hsakmt-roct-dev \
lcov \
libelf-dev \
libfile-which-perl \
libncurses5-dev \
libnuma-dev \
libpthread-stubs0-dev \
libssl-dev \
locales \
pkg-config \
python \
python-dev \
python-pip \
python3 \
python3-dev \
python3-pip \
python-pip \
python-dev \
rocm-device-libs \
rocm-opencl \
rocm-opencl-dev \
software-properties-common \
sudo \
wget \
rocm-device-libs \
miopen-hip \
rocblas \
zlib1g-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Wokaround broken rocm packages for rocm >= 3.1
RUN [ -d /opt/rocm ] || ln -sd $(realpath /opt/rocm-*) /opt/rocm
# Workaround broken rocm packages
RUN ln -s /opt/rocm-* /opt/rocm
RUN echo "/opt/rocm/lib" > /etc/ld.so.conf.d/rocm.conf
RUN echo "/opt/rocm/llvm/lib" > /etc/ld.so.conf.d/rocm-llvm.conf
RUN ldconfig
RUN locale-gen en_US.UTF-8
RUN update-locale LANG=en_US.UTF-8
......@@ -62,11 +50,8 @@ RUN update-locale LANG=en_US.UTF-8
ENV LC_ALL=C.UTF-8
ENV LANG=C.UTF-8
# Install cget
RUN pip3 install cget
# Install rclone
RUN pip install https://github.com/pfultz2/rclone/archive/master.tar.gz
# Install rbuild
RUN pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
# Install yapf
RUN pip3 install yapf==0.28.0
......@@ -75,22 +60,6 @@ RUN pip3 install yapf==0.28.0
ADD doc/requirements.txt /doc-requirements.txt
RUN pip3 install -r /doc-requirements.txt
# Install hcc
RUN rclone -b roc-3.0.x -c 286651a04d9c3a8e3052dd84b1822985498cd27d https://github.com/RadeonOpenCompute/hcc.git /hcc
RUN cget -p $PREFIX install hcc,/hcc
# Use hcc
RUN cget -p $PREFIX init --cxx $PREFIX/bin/hcc
# Workaround hip's broken cmake
RUN ln -s $PREFIX /opt/rocm/hip
RUN ln -s $PREFIX /opt/rocm/hcc
# Install dependencies
ADD dev-requirements.txt /dev-requirements.txt
ADD requirements.txt /requirements.txt
RUN cget -p $PREFIX install -f /dev-requirements.txt -DMIOPEN_CACHE_DIR=""
RUN pip3 install onnx==1.7.0 numpy==1.18.5 typing==3.7.4 pytest==6.0.1
# Download real models to run onnx unit tests
......@@ -98,6 +67,24 @@ ENV ONNX_HOME=$HOME
COPY ./tools/download_models.sh /
RUN chmod +x /download_models.sh && /download_models.sh && rm /download_models.sh
# Install dependencies
ADD dev-requirements.txt /dev-requirements.txt
ADD requirements.txt /requirements.txt
# Manually ignore rocm dependencies
RUN cget -p $PREFIX ignore \
RadeonOpenCompute/clang-ocl \
ROCm-Developer-Tools/HIP \
ROCmSoftwarePlatform/MIOpen \
ROCmSoftwarePlatform/MIOpenGEMM \
ROCmSoftwarePlatform/rocBLAS
RUN cget -p $PREFIX init --cxx /opt/rocm/llvm/bin/clang++
RUN cget -p $PREFIX install -f dev-requirements.txt
RUN cget -p $PREFIX install oneapi-src/oneDNN@v1.7
# Install latest ccache version
RUN cget -p $PREFIX install facebook/zstd@v1.4.5 -X subdir -DCMAKE_DIR=build/cmake
RUN cget -p $PREFIX install ccache@v4.1
# Install newer cmake for onnx runtime
RUN cget -p /opt/cmake install kitware/cmake@v3.13.0
......
Jenkinsfile
View file @ ceb4ca09
......@@ -7,7 +7,10 @@ def rocmtestnode(Map conf) {
def docker_args = conf.get("docker_args", "")
def docker_build_args = conf.get("docker_build_args", "")
def pre = conf.get("pre", {})
def ccache = "/var/jenkins/.cache/ccache"
def image = 'migraphxlib'
env.CCACHE_COMPRESSLEVEL = 7
env.CCACHE_DIR = ccache
def cmake_build = { compiler, flags ->
def cmd = """
env
......@@ -15,12 +18,12 @@ def rocmtestnode(Map conf) {
rm -rf build
mkdir build
cd build
CXX=${compiler} CXXFLAGS='-Werror -Wno-fallback' cmake ${flags} ..
CTEST_PARALLEL_LEVEL=32 make -j\$(nproc) generate all doc package check
CXX=${compiler} CXXFLAGS='-Werror -Wno-fallback' cmake -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ${flags} ..
CTEST_PARALLEL_LEVEL=32 make -j\$(nproc) generate all doc package check VERBOSE=1
"""
echo cmd
sh cmd
if (compiler == "hcc") {
if (compiler != "hcc") {
// Only archive from master or develop
if (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "master") {
archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true
......@@ -42,7 +45,7 @@ def rocmtestnode(Map conf) {
}
}
withDockerContainer(image: image, args: "--device=/dev/kfd --device=/dev/dri --group-add video --cap-add SYS_PTRACE ${docker_args}") {
withDockerContainer(image: image, args: "--device=/dev/kfd --device=/dev/dri --group-add video --cap-add SYS_PTRACE -v=/var/jenkins/:/var/jenkins ${docker_args}") {
timeout(time: 2, unit: 'HOURS') {
body(cmake_build)
}
......@@ -80,92 +83,30 @@ def rocmnode(name, body) {
}
}
def rocmhipclangnode(name, body) {
def rochccmnode(name, body) {
return { label ->
rocmtestnode(variant: label, node: rocmnodename(name), docker_build_args: '-f hip-clang.docker', body: body)
rocmtestnode(variant: label, node: rocmnodename(name), docker_build_args: '-f hcc.docker', body: body)
}
}
// Static checks
rocmtest format: rocmnode('rocmtest') { cmake_build ->
stage('Format') {
sh '''
find . -iname \'*.h\' \
-o -iname \'*.hpp\' \
-o -iname \'*.cpp\' \
-o -iname \'*.h.in\' \
-o -iname \'*.hpp.in\' \
-o -iname \'*.cpp.in\' \
-o -iname \'*.cl\' \
| grep -v 'build/' \
| xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-5.0 -style=file {} | diff - {}\'
find . -iname \'*.py\' \
| grep -v 'build/' \
| xargs -n 1 -P 1 -I{} -t sh -c \'yapf {} | diff - {}\'
'''
}
}, clang_debug: rocmnode('vega') { cmake_build ->
stage('Clang Debug') {
// TODO: Enable integer
def sanitizers = "undefined"
def debug_flags = "-O2 -fsanitize=${sanitizers} -fno-sanitize-recover=${sanitizers}"
cmake_build("hcc", "-DCMAKE_BUILD_TYPE=debug -DMIGRAPHX_ENABLE_PYTHON=Off -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'")
rocmtest clang_debug: rocmnode('vega') { cmake_build ->
stage('Hip Clang Debug') {
// def sanitizers = "undefined"
// def debug_flags = "-O2 -fsanitize=${sanitizers} -fno-sanitize-recover=${sanitizers}"
def debug_flags = "-g -O2"
cmake_build("/opt/rocm/llvm/bin/clang++", "-DCMAKE_BUILD_TYPE=debug -DMIGRAPHX_ENABLE_PYTHON=Off -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'")
}
}, clang_release: rocmnode('vega') { cmake_build ->
stage('Clang Release') {
cmake_build("hcc", "-DCMAKE_BUILD_TYPE=release")
stash includes: 'build/*.deb', name: 'migraphx-package'
}
}, hip_clang_release: rocmhipclangnode('vega') { cmake_build ->
stage('Hip Clang Release') {
cmake_build("/opt/rocm/llvm/bin/clang++", "-DCMAKE_BUILD_TYPE=release")
// stash includes: 'build/*.deb', name: 'migraphx-package'
}
}, hip_clang_tidy: rocmhipclangnode('rocmtest') { cmake_build ->
stage('Hip Clang Tidy') {
sh '''
rm -rf build
mkdir build
cd build
CXX=/opt/rocm/llvm/bin/clang++ cmake ..
make -j$(nproc) -k analyze
'''
}
}, gcc5: rocmnode('rocmtest') { cmake_build ->
stage('GCC 5 Debug') {
cmake_build("g++-5", "-DCMAKE_BUILD_TYPE=debug")
}
stage('GCC 5 Release') {
cmake_build("g++-5", "-DCMAKE_BUILD_TYPE=release")
}
}, gcc7: rocmhipclangnode('rocmtest') { cmake_build ->
stage('GCC 7 Debug') {
def linker_flags = '-fuse-ld=gold'
def cmake_linker_flags = "-DCMAKE_EXE_LINKER_FLAGS='${linker_flags}' -DCMAKE_SHARED_LINKER_FLAGS='${linker_flags}'"
// TODO: Add bounds-strict
def sanitizers = "undefined,address"
def debug_flags = "-g -fno-omit-frame-pointer -fsanitize-address-use-after-scope -fsanitize=${sanitizers} -fno-sanitize-recover=${sanitizers}"
cmake_build("g++-7", "-DCMAKE_BUILD_TYPE=debug -DMIGRAPHX_ENABLE_CPU=On -DMIGRAPHX_ENABLE_PYTHON=Off ${cmake_linker_flags} -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'")
}
}, codecov: rocmnode('rocmtest') { cmake_build ->
stage('GCC 7 Codecov') {
def linker_flags = '-fuse-ld=gold'
def cmake_linker_flags = "-DCMAKE_EXE_LINKER_FLAGS='${linker_flags}' -DCMAKE_SHARED_LINKER_FLAGS='${linker_flags}'"
def debug_flags = "-g -fprofile-arcs -ftest-coverage -fno-omit-frame-pointer"
cmake_build("g++-7", "-DCMAKE_BUILD_TYPE=debug -DMIGRAPHX_ENABLE_CPU=Off -DMIGRAPHX_ENABLE_PYTHON=Off ${cmake_linker_flags} -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'")
stash includes: 'build/*.deb', name: 'migraphx-package'
}
stage('Codecov') {
env.CODECOV_TOKEN="8545af1c-f90b-4345-92a5-0d075503ca56"
sh '''
cd build
lcov --directory . --capture --output-file $(pwd)/coverage.info
lcov --remove $(pwd)/coverage.info '/usr/*' --output-file $(pwd)/coverage.info
lcov --list $(pwd)/coverage.info
curl -s https://codecov.io/bash | bash
echo "Uploaded"
'''
}, hcc_debug: rochccmnode('vega') { cmake_build ->
stage('Hcc Debug') {
// TODO: Enable integer
def sanitizers = "undefined"
def debug_flags = "-O2 -fsanitize=${sanitizers} -fno-sanitize-recover=${sanitizers}"
cmake_build("/opt/rocm/bin/hcc", "-DCMAKE_BUILD_TYPE=debug -DMIGRAPHX_ENABLE_PYTHON=Off -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'")
}
}
......
README.md 100644 → 100755
View file @ ceb4ca09
......@@ -21,7 +21,7 @@ pip install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
To build dependencies along with MIGraphX
```
rbuild build -d depend --cxx=/opt/rocm/bin/hcc
rbuild build -d depend --cxx=/opt/rocm/llvm/bin/clang++
```
This builds dependencies in the subdirectory named depend and then builds MIGraphX using these dependencies.
......@@ -41,14 +41,14 @@ Next configure cmake. The hcc compiler is required to build the MIOpen backend:
```
CXX=/opt/rocm/bin/hcc cmake ..
CXX=/opt/rocm/llvm/bin/clang++ cmake ..
```
If the dependencies from `install_deps.cmake` was installed to another directory, the `CMAKE_PREFIX_PATH` needs to be set to what `--prefix` was set to from `install_deps.cmake`:
```
CXX=/opt/rocm/bin/hcc cmake -DCMAKE_PREFIX_PATH=/some/dir ..
CXX=/opt/rocm/llvm/bin/clang++ cmake -DCMAKE_PREFIX_PATH=/some/dir ..
```
......
cppcheck.rules 100644 → 100755
View file @ ceb4ca09
<?xml version="1.0"?>
<rule>
<tokenlist>normal</tokenlist>
<pattern> [;{}] [*] \w+? (\+\+|\-\-) ; </pattern>
<message>
<id>UnusedDeref</id>
......@@ -8,6 +9,7 @@
</message>
</rule>
<rule>
<tokenlist>normal</tokenlist>
<pattern> if \( ([!] )*?(strlen) \( \w+? \) ([>] [0] )*?\) { </pattern>
<message>
<id>StrlenEmptyString</id>
......@@ -16,6 +18,7 @@
</message>
</rule>
<rule>
<tokenlist>normal</tokenlist>
<pattern> [;{}] [*] \w+? (\+\+|\-\-) ; </pattern>
<message>
<id>UnusedDeref</id>
......@@ -42,6 +45,7 @@
</message>
</rule>
<rule>
<tokenlist>normal</tokenlist>
<pattern>mutable \w+</pattern>
<message>
<id>MutableVariable</id>
......@@ -50,6 +54,7 @@
</message>
</rule>
<rule>
<tokenlist>normal</tokenlist>
<pattern>(memcpy|strcpy|strncpy|strcat|strncat) \(</pattern>
<message>
<id>useStlAlgorithms</id>
......@@ -58,6 +63,7 @@
</message>
</rule>
<rule>
<tokenlist>normal</tokenlist>
<pattern>memset \(</pattern>
<message>
<id>useStlAlgorithms</id>
......@@ -66,6 +72,7 @@
</message>
</rule>
<rule>
<tokenlist>normal</tokenlist>
<pattern>memcmp \(</pattern>
<message>
<id>useStlAlgorithms</id>
......@@ -74,6 +81,7 @@
</message>
</rule>
<rule>
<tokenlist>normal</tokenlist>
<pattern>memchr \(</pattern>
<message>
<id>useStlAlgorithms</id>
......@@ -82,6 +90,7 @@
</message>
</rule>
<rule>
<tokenlist>normal</tokenlist>
<pattern>\\W(fclose|free|hipFree|hipHostFree|hipFreeArray|hipMemFree|hipStreamDestroy|hipEventDestroy|hipArrayDestroy|hipCtxDestroy|hipDestroyTextureObject|hipDestroySurfaceObject) \(</pattern>
<message>
<id>useManagePointer</id>
......@@ -89,7 +98,7 @@
<summary>Use manage pointer for resource management</summary>
</message>
</rule>
<rule>
<!-- <rule>
<tokenlist>raw</tokenlist>
<pattern><![CDATA[] (__device__ |__host__ )+(\(|{)]]></pattern>
<message>
......@@ -97,9 +106,9 @@
<severity>warning</severity>
<summary>Attributes to lambdas must come after parameter list.</summary>
</message>
</rule>
</rule> -->
<rule>
<tokenlist>raw</tokenlist>
<tokenlist>normal</tokenlist>
<pattern><![CDATA[hipLaunchKernelGGL \( (?!\( \w+ < \w+ > \))]]></pattern>
<message>
<id>UseDeviceLaunch</id>
......@@ -315,7 +324,7 @@
</message>
</rule>
<rule>
<tokenlist>simple</tokenlist>
<tokenlist>normal</tokenlist>
<pattern><![CDATA[for \( (?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)* (\w+) = \w+ ; \1 < \w+ ; (\1 \+\+|\+\+ \1|\1 \-\-|\-\- \1) \) { \w+ \[ \1 \] = \w+ \[ \1 \] ; }]]></pattern>
<message>
<id>useStlAlgorithm</id>
......@@ -324,7 +333,7 @@
</message>
</rule>
<rule>
<tokenlist>simple</tokenlist>
<tokenlist>normal</tokenlist>
<pattern><![CDATA[for \( (?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)* (\w+) = \w+ ; \1 < \w+ ; (\1 \+\+|\+\+ \1|\1 \-\-|\-\- \1) \) { \w+ \[ \1 \] = \w+ ; }]]></pattern>
<message>
<id>useStlAlgorithm</id>
......@@ -333,7 +342,7 @@
</message>
</rule>
<rule>
<tokenlist>simple</tokenlist>
<tokenlist>normal</tokenlist>
<pattern><![CDATA[for \( (?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)* (\w+) = \w+ ; \1 < \w+ ; (\1 \+\+|\+\+ \1|\1 \-\-|\-\- \1) \) { \w+ \[ \1 \] = (?:\w+ :: )*\w+ \( \) ; }]]></pattern>
<message>
<id>useStlAlgorithm</id>
......@@ -342,7 +351,7 @@
</message>
</rule>
<rule>
<tokenlist>simple</tokenlist>
<tokenlist>normal</tokenlist>
<pattern><![CDATA[for \( (?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)* (\w+) = \w+ ; \1 < \w+ ; (\1 \+\+|\+\+ \1|\1 \-\-|\-\- \1) \) { \w+ \[ \1 \] = (?:\w+ :: )*\w+ \( \w+ \[ \1 \] \) ; }]]></pattern>
<message>
<id>useStlAlgorithm</id>
......@@ -351,7 +360,7 @@
</message>
</rule>
<rule>
<tokenlist>simple</tokenlist>
<tokenlist>normal</tokenlist>
<pattern><![CDATA[for \( (?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)* (\w+) = \w+ ; \1 < \w+ ; (\1 \+\+|\+\+ \1|\1 \-\-|\-\- \1) \) { \w+ \[ \1 \] = (?:\w+ :: )*\w+ \( \w+ \[ \1 \] , \w+ \[ \1 \] \) ; }]]></pattern>
<message>
<id>useStlAlgorithm</id>
......@@ -360,7 +369,7 @@
</message>
</rule>
<rule>
<tokenlist>simple</tokenlist>
<tokenlist>normal</tokenlist>
<pattern><![CDATA[for \( (?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)* (\w+) : (?:[^()]*(\([^()]*(?-1)*[^()]*\)))*[^)]*\) { (?:(?<idx1>\w+) \+\+|\+\+ (?<idx2>\w+)) ; if (\([^()]*(?-1)*[^()]*\)) { \w+ = \g{idx1}|\g{idx2} ; (?:break ; )?(?:return [^;]*; )?} }]]></pattern>
<message>
<id>useStlAlgorithm</id>
......@@ -369,7 +378,7 @@
</message>
</rule>
<rule>
<tokenlist>simple</tokenlist>
<tokenlist>normal</tokenlist>
<pattern><![CDATA[for \( (?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)* (\w+) : (?:[^()]*(\([^()]*(?-1)*[^()]*\)))*[^)]*\) { if (\([^()]*(?-1)*[^()]*\)) { \w+ = (?<idx>\w) ; (?:break ; )?(?:return [^;]*; )?} (?:(\g{idx}) \+\+|\+\+ (\g{idx})) ; }]]></pattern>
<message>
<id>useStlAlgorithm</id>
......@@ -378,7 +387,7 @@
</message>
</rule>
<rule>
<tokenlist>simple</tokenlist>
<tokenlist>normal</tokenlist>
<pattern><![CDATA[for \( (?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)* (\w+) : (?:[^()]*(\([^()]*(?-1)*[^()]*\)))*[^)]*\) { (?:(?<idx1>\w+) \+\+|\+\+ (?<idx2>\w+)) ; if (\([^()]*(?-1)*[^()]*\)) { return \g{idx1}|\g{idx2} ; } }]]></pattern>
<message>
<id>useStlAlgorithm</id>
......@@ -387,7 +396,7 @@
</message>
</rule>
<rule>
<tokenlist>simple</tokenlist>
<tokenlist>normal</tokenlist>
<pattern><![CDATA[for \( (?:(?:\w+|<|>|::) )*(?:\w+|>)(?: &|\*)* (\w+) : (?:[^()]*(\([^()]*(?-1)*[^()]*\)))*[^)]*\) { if (\([^()]*(?-1)*[^()]*\)) { return (?<idx>\w+) ; } (?:(\g{idx}) \+\+|\+\+ (\g{idx})) ; }]]></pattern>
<message>
<id>useStlAlgorithm</id>
......
hcc.docker 0 → 100755
View file @ ceb4ca09
FROM ubuntu:18.04
ARG PREFIX=/usr/local
# Support multiarch
RUN dpkg --add-architecture i386
# Add rocm repository
RUN sh -c 'echo deb [arch=amd64 trusted=yes] http://repo.radeon.com/rocm/apt/3.0/ xenial main > /etc/apt/sources.list.d/rocm.list'
# Install dependencies
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \
apt-utils \
build-essential \
clang-format-5.0 \
cmake \
curl \
doxygen \
gdb \
git \
lcov \
pkg-config \
python \
python-dev \
python-pip \
python3 \
python3-dev \
python3-pip \
software-properties-common \
wget \
rocm-clang-ocl \
rocm-device-libs \
rocblas \
zlib1g-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Workaround broken rocm packages
RUN ln -s /opt/rocm-* /opt/rocm
RUN echo "/opt/rocm/lib" > /etc/ld.so.conf.d/rocm.conf
RUN echo "/opt/rocm/llvm/lib" > /etc/ld.so.conf.d/rocm-llvm.conf
RUN ldconfig
ENV LC_ALL=C.UTF-8
ENV LANG=C.UTF-8
# Install rbuild
RUN pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
# Install doc requirements
ADD doc/requirements.txt /doc-requirements.txt
RUN pip3 install -r /doc-requirements.txt
# Install dependencies
ADD dev-requirements.txt /dev-requirements.txt
ADD requirements.txt /requirements.txt
# Manually ignore rocm dependencies
RUN cget -p $PREFIX ignore \
RadeonOpenCompute/clang-ocl \
ROCm-Developer-Tools/HIP \
ROCmSoftwarePlatform/MIOpenGEMM \
ROCmSoftwarePlatform/rocBLAS
RUN cget -p $PREFIX init --cxx /opt/rocm/bin/hcc
RUN cget -p $PREFIX install -f dev-requirements.txt
# Install latest ccache version
RUN cget -p $PREFIX install facebook/zstd@v1.4.5 -X subdir -DCMAKE_DIR=build/cmake
RUN cget -p $PREFIX install ccache@v4.1
hip-clang.docker
View file @ ceb4ca09
......@@ -37,6 +37,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-
# Workaround broken rocm packages
RUN ln -s /opt/rocm-* /opt/rocm
RUN echo "/opt/rocm/lib" > /etc/ld.so.conf.d/rocm.conf
RUN echo "/opt/rocm/llvm/lib" > /etc/ld.so.conf.d/rocm-llvm.conf
RUN ldconfig
ENV LC_ALL=C.UTF-8
ENV LANG=C.UTF-8
......@@ -44,17 +47,13 @@ ENV LANG=C.UTF-8
# Install rbuild
RUN pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
# Install yapf
RUN pip3 install yapf==0.28.0
# Install doc requirements
ADD doc/requirements.txt /doc-requirements.txt
RUN pip3 install -r /doc-requirements.txt
RUN pip3 install onnx==1.7.0 numpy==1.18.5 typing==3.7.4 pytest==6.0.1
# Download real models to run onnx unit tests
ENV ONNX_HOME=$HOME
COPY ./tools/download_models.sh /
RUN chmod +x /download_models.sh && /download_models.sh && rm /download_models.sh
# Install dependencies
ADD dev-requirements.txt /dev-requirements.txt
ADD requirements.txt /requirements.txt
......
src/include/migraphx/matcher.hpp 100644 → 100755
View file @ ceb4ca09
......@@ -44,7 +44,7 @@ struct predicate_matcher
{
P p;
instruction_ref match(matcher_context& ctx, instruction_ref ins) const
instruction_ref match(const matcher_context& ctx, instruction_ref ins) const
{
assert(ins != ctx.not_found());
if(p(ins))
......
src/include/migraphx/raw_data.hpp 100644 → 100755
View file @ ceb4ca09
......@@ -202,7 +202,8 @@ void visit_all_impl(const shape& s, V&& v, Ts&&... xs)
template <class T, class... Ts>
auto visit_all(T&& x, Ts&&... xs)
{
auto&& s = x.get_shape();
auto&& s = x.get_shape();
// cppcheck-suppress redundantInitialization
std::initializer_list<shape::type_t> types = {xs.get_shape().type()...};
if(!std::all_of(types.begin(), types.end(), [&](shape::type_t t) { return t == s.type(); }))
MIGRAPHX_THROW("Types must be the same");
......
src/targets/cpu/CMakeLists.txt
View file @ ceb4ca09
......@@ -26,9 +26,8 @@ rocm_clang_tidy_check(migraphx_cpu)
target_link_libraries(migraphx_cpu PRIVATE migraphx Threads::Threads)
target_include_directories(migraphx_cpu PRIVATE ${BLAZE_INCLUDE})
target_compile_definitions(migraphx_cpu PRIVATE -DBLAZE_USE_CPP_THREADS)
if(dnnl_FOUND)
target_link_libraries(migraphx_cpu PRIVATE DNNL::dnnl)
target_compile_definitions(migraphx_cpu PRIVATE -DUSE_DNNL)
target_compile_definitions(migraphx_cpu PUBLIC -DUSE_DNNL)
find_package(OpenMP)
target_link_libraries(migraphx_cpu PUBLIC OpenMP::OpenMP_CXX)
# Add library path to rpath to workaround issues with our broken packages
......@@ -38,7 +37,6 @@ foreach(LIBRARY ${OpenMP_CXX_LIBRARIES})
target_link_libraries(migraphx_cpu PUBLIC -Wl,-rpath=${LIBRARY_PATH} -Wl,-rpath-link=${LIBRARY_PATH})
endif()
endforeach()
endif()
target_link_libraries(migraphx_all_targets INTERFACE migraphx_cpu)
......
src/targets/gpu/device/include/migraphx/gpu/device/launch.hpp 100644 → 100755
View file @ ceb4ca09
......@@ -56,6 +56,7 @@ inline auto launch(hipStream_t stream, index_int global, index_int local)
using f_type = decltype(f);
dim3 nblocks(global / local);
dim3 nthreads(local);
// cppcheck-suppress UseDeviceLaunch
hipLaunchKernelGGL((launcher<f_type>), nblocks, nthreads, 0, stream, f);
};
}
......
src/targets/gpu/device/include/migraphx/gpu/device/reduce.hpp 100644 → 100755
View file @ ceb4ca09
......@@ -53,7 +53,7 @@ struct max
template <class T, class U>
MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x, U y) const
{
return x > y ? x : y;
return (x > y) ? x : y;
}
};
......@@ -62,7 +62,7 @@ struct min
template <class T, class U>
MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x, U y) const
{
return x < y ? x : y;
return (x < y) ? x : y;
}
};
......
src/targets/gpu/fuse_ops.cpp
View file @ ceb4ca09
......@@ -458,10 +458,10 @@ struct find_gelu_new
auto x_ins = r.instructions["x"];
auto args = ins->inputs();
if(not fast_math)
p.replace_instruction(ins, hip_gelu_new{}, x_ins, args.back());
else
if(fast_math)
p.replace_instruction(ins, hip_gelu{}, x_ins, args.back());
else
p.replace_instruction(ins, hip_gelu_new{}, x_ins, args.back());
}
};
......
test/CMakeLists.txt
View file @ ceb4ca09
......@@ -7,7 +7,7 @@ find_package(Threads REQUIRED)
include(ProcessorCount)
ProcessorCount(N)
set(CTEST_PARALLEL_LEVEL ${N} CACHE STRING "CTest parallel level")
add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -j ${CTEST_PARALLEL_LEVEL} -C ${CMAKE_CFG_INTDIR} --timeout 2000)
add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -j ${CTEST_PARALLEL_LEVEL} -C ${CMAKE_CFG_INTDIR} --timeout 5000)
add_custom_target(tests)
find_program(MIGRAPHX_GDB gdb)
......
test/verify/main.cpp
View file @ ceb4ca09
......@@ -4,6 +4,10 @@
#ifdef HAVE_GPU
#include <migraphx/gpu/analyze_streams.hpp>
#include <migraphx/gpu/target.hpp>
#endif
#ifdef HAVE_CPU
#include <migraphx/cpu/target.hpp>
#endif
inline void check_gpu_streams(const migraphx::program& p)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment