Commit fb75dfaf authored by Paul's avatar Paul
Browse files

Only use no-cache on jenkins

parents e596eec2 f0604d78
......@@ -100,4 +100,4 @@ CheckOptions:
# - key: readability-identifier-naming.MacroDefinitionCase
# value: UPPER_CASE
# - key: readability-identifier-naming.MacroDefinitionPrefix
# value: RTG_
# value: MIGRAPH_
cmake_minimum_required(VERSION 3.5)
project(rtglib)
project(migraphlib)
find_package(ROCM REQUIRED)
option( BUILD_SHARED_LIBS "Build as a shared library" ON )
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.4")
message(FATAL_ERROR "RTGLib requires at least gcc 5.4")
message(FATAL_ERROR "MIGraph requires at least gcc 5.4")
endif()
endif()
if(CMAKE_CXX_COMPILER MATCHES ".*hcc")
message(STATUS "Enable miopen backend")
set(RTG_ENABLE_MIOPEN On CACHE BOOL "")
set(MIGRAPH_ENABLE_GPU On CACHE BOOL "")
else()
set(RTG_ENABLE_MIOPEN Off CACHE BOOL "")
set(MIGRAPH_ENABLE_GPU Off CACHE BOOL "")
endif()
add_compile_options(-std=c++14)
......@@ -83,8 +85,7 @@ rocm_enable_clang_tidy(
HEADER_FILTER
".*hpp"
EXTRA_ARGS
-DRTG_USE_CLANG_TIDY
ANALYZE_TEMPORARY_DTORS ON
-DMIGRAPH_USE_CLANG_TIDY
)
include(ROCMCppCheck)
......@@ -106,8 +107,12 @@ rocm_enable_cppcheck(
${CMAKE_CURRENT_SOURCE_DIR}/src/include
${CMAKE_CURRENT_SOURCE_DIR}/src/targets/cpu/include
${CMAKE_CURRENT_SOURCE_DIR}/src/targets/miopen/include
DEFINE
CPPCHECK=1
)
enable_testing()
add_subdirectory(src)
add_subdirectory(doc)
add_subdirectory(test)
......
......@@ -21,13 +21,17 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-
doxygen \
gdb \
git \
hcc \
hip_hcc \
hsa-rocr-dev \
hsakmt-roct-dev \
lcov \
libelf-dev \
libncurses5-dev \
libpthread-stubs0-dev \
libnuma-dev \
python \
python-dev \
python-pip \
rocminfo \
rocm-opencl \
rocm-opencl-dev \
software-properties-common \
......@@ -38,8 +42,19 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-
# Install cget
RUN pip install cget
# Install rclone
RUN pip install https://github.com/pfultz2/rclone/archive/master.tar.gz
# Install hcc
RUN rclone -b sanitizer1 https://github.com/RadeonOpenCompute/hcc.git /hcc
RUN cget -p $PREFIX install hcc,/hcc
# Use hcc
RUN cget -p $PREFIX init --cxx /opt/rocm/bin/hcc
RUN cget -p $PREFIX init --cxx $PREFIX/bin/hcc
# Workaround hip's broken cmake
RUN ln -s $PREFIX /opt/rocm/hip
RUN ln -s $PREFIX /opt/rocm/hcc
# Install dependencies
ADD dev-requirements.txt /dev-requirements.txt
......@@ -49,5 +64,5 @@ RUN cget -p $PREFIX install -f /dev-requirements.txt -DMIOPEN_CACHE_DIR=""
ENV LD_LIBRARY_PATH=$PREFIX/lib
# Install doc requirements
# ADD doc/requirements.txt /doc-requirements.txt
# RUN pip install -r /doc-requirements.txt
ADD doc/requirements.txt /doc-requirements.txt
RUN pip install -r /doc-requirements.txt
def rocmtestnode(variant, name, body) {
def image = 'rtglib'
def image = 'migraphlib'
def cmake_build = { compiler, flags ->
def cmd = """
rm -rf build
mkdir build
cd build
CXX=${compiler} CXXFLAGS='-Werror' cmake -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} ..
CXX=${compiler} CXXFLAGS='-Werror -Wno-fallback' cmake -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} ..
CTEST_PARALLEL_LEVEL=32 make -j32 all doc check
"""
echo cmd
......@@ -19,9 +19,9 @@ def rocmtestnode(variant, name, body) {
}
stage("image ${variant}") {
try {
docker.build("${image}", ".")
docker.build("${image}", '.')
} catch(Exception ex) {
docker.build("${image}", "--no-cache .")
docker.build("${image}", '--no-cache .')
}
}
......@@ -92,10 +92,10 @@ rocmtest tidy: rocmnode('rocmtest') { cmake_build ->
}
}, clang: rocmnode('rocmtest') { cmake_build ->
stage('Clang Debug') {
cmake_build('/opt/rocm/bin/hcc', '-DCMAKE_BUILD_TYPE=debug')
cmake_build('hcc', '-DCMAKE_BUILD_TYPE=debug')
}
stage('Clang Release') {
cmake_build('/opt/rocm/bin/hcc', '-DCMAKE_BUILD_TYPE=release')
cmake_build('hcc', '-DCMAKE_BUILD_TYPE=release')
}
}, gcc: rocmnode('rocmtest') { cmake_build ->
stage('GCC Debug') {
......
# MIGraph
AMD's library for graph optimizations.
## Prerequisites
* [ROCm cmake modules](https://github.com/RadeonOpenCompute/rocm-cmake) **required**
* [MIOpen](https://github.com/ROCmSoftwarePlatform/MIOpen) for running on the GPU
* [HIP](https://github.com/ROCm-Developer-Tools/HIP) for running on the GPU
* [Protobuf](https://github.com/google/protobuf) for reading [onxx](https://github.com/onnx/onnx) files
## Installing the dependencies
The dependencies can be installed with the `install_deps.cmake`, script: `cmake -P install_deps.cmake`.
This will install by default to `/usr/local` but it can be installed in another location with `--prefix` argument:
```
cmake -P install_deps.cmake --prefix /some/local/dir
```
## Building MIGraph from source
## Configuring with cmake
First create a build directory:
```
mkdir build;
cd build;
```
Next configure cmake. The hcc compiler is required to build the MIOpen backend:
```
CXX=/opt/rocm/bin/hcc cmake ..
```
If the dependencies from `install_deps.cmake` was installed to another directory, the `CMAKE_PREFIX_PATH` needs to be set to what `--prefix` was set to from `install_deps.cmake`:
```
CXX=/opt/rocm/bin/hcc cmake -DCMAKE_PREFIX_PATH=/some/dir ..
```
#### Changing the cmake configuration
The configuration can be changed after running cmake by using `ccmake`:
` ccmake .. ` **OR** `cmake-gui`: ` cmake-gui ..`
## Building the library
The library can be built, from the `build` directory using the 'Release' configuration:
` cmake --build . --config Release ` **OR** ` make `
And can be installed by using the 'install' target:
` cmake --build . --config Release --target install ` **OR** ` make install `
This will install the library to the `CMAKE_INSTALL_PREFIX` path that was set.
## Running the tests
The tests can be run by using the 'check' target:
` cmake --build . --config Release --target check ` **OR** ` make check `
## Building the documentation
HTML and PDF documentation can be built using:
`cmake --build . --config Release --target doc` **OR** `make doc`
The generated documentation will be located in `doc/doxygen/`.
## Formatting the code
All the code is formatted using clang-format. To format a file, use:
```
clang-format-5.0 -style=file -i <path-to-source-file>
```
Also, githooks can be installed to format the code per-commit:
```
./.githooks/install
```
## Using docker
The easiest way to setup the development environment is to use docker. You can build the top-level docker file:
docker build -t migraph .
Then to enter the developement environment use `docker run`:
docker run --device='/dev/kfd' --device='/dev/dri' -v=`pwd`:/data -w /data --group-add video -it migraph
......@@ -16,7 +16,7 @@ add_doxygen_doc(
CALL_GRAPH YES
CALLER_GRAPH YES
BUILTIN_STL_SUPPORT YES
PROJECT_NAME RTGLib
PROJECT_NAME MIGraph
SORT_MEMBERS_CTORS_1ST YES
SOURCE_BROWSER YES
GENERATE_TREEVIEW YES
......@@ -26,15 +26,30 @@ add_doxygen_doc(
EXTRACT_ALL YES
ENUM_VALUES_PER_LINE 1
FULL_PATH_NAMES YES
PREDEFINED DOXYGEN
)
# include(SphinxDoc)
# add_sphinx_doc(src
# BUILDER html
# OUTPUT_DIR html
# VARS
# breathe_projects.proj=${DOXYGEN_OUTPUT}/xml
# breathe_default_project=proj
# DEPENDS doxygen
# )
include(SphinxDoc)
add_sphinx_doc(src
BUILDER html
OUTPUT_DIR html
VARS
breathe_projects.proj=${DOXYGEN_OUTPUT}/xml
breathe_default_project=proj
DEPENDS doxygen
)
find_package(LATEX)
if(LATEX_FOUND)
add_sphinx_doc(src
BUILDER latex
OUTPUT_DIR pdf
VARS
breathe_projects.proj=${DOXYGEN_OUTPUT}/xml
breathe_default_project=proj
DEPENDS doxygen
)
else()
message("Latex builder not found. Latex builder is required only for building the PDF documentation for MIGraph and is not necessary for building the library, or any other components. To build PDF documentation run make in ${CMAKE_CURRENT_SOURCE_DIR}/pdf, once a latex builder is installed.")
endif()
sphinx
breathe==4.9.1
# git+https://github.com/arximboldi/breathe@fix-node-parent
# -*- coding: utf-8 -*-
#
# MIGraph documentation build configuration file, created by
# sphinx-quickstart on Thu Jul 19 11:38:13 2018.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['breathe', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'MIGraph'
copyright = u'2018, AMD'
author = u'AMD'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = u'0.1'
# The full version, including alpha/beta/rc tags.
release = u'0.1'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = []
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
highlight_language = 'cpp'
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# -- Options for HTMLHelp output ------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'MIGraphdoc'
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'MIGraph.tex', u'MIGraph Documentation',
u'AMD', 'manual'),
]
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'migraph', u'MIGraph Documentation',
[author], 1)
]
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'MIGraph', u'MIGraph Documentation',
author, 'MIGraph', 'One line description of project.',
'Miscellaneous'),
]
breathe_default_members = ('members', 'undoc-members')
cpp_index_common_prefix = ['migraph::']
default_role = 'any'
primary_domain = 'cpp'
.. MIGraph documentation master file, created by
sphinx-quickstart on Thu Jul 19 11:38:13 2018.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to MIGraph's documentation!
===================================
.. toctree::
:maxdepth: 2
:caption: Contents:
overview
reference/data
reference/operators
reference/program
reference/targets
reference/pass
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
Overview
========
MIGraph provides an optimized execution engine for deep learning neural networks.
Building a program
------------------
A program consists of a set of instructions to be executed when calling `eval <migraph::program::eval>`. Each instruction has an associated `operation <migraph::operation>` which represents the computation to be performed by the instruction.
We can start by building a simple program to add two numbers together::
program p;
instruction_ref one = p.add_literal(1);
instruction_ref two = p.add_literal(2);
p.add_instruction(add{}, one, two);
The `add_literal <migraph::program::add_literal>` function will add an instruction to the program to store a literal number. The `instruction_ref <migraph::instruction_ref>` is a reference to the instruction in the program, which can be used to compose the output of the instruction with another instruction.
After creating the literals, we then create the instruction to add the numbers together. This is done by using the `add{} <migraph::add>` operation class along with the `instruction_ref <migraph::instruction_ref>` for the input arguments of the instruction.
Finally, we can run this `program <migraph::program>` by compiling it for the cpu and then running it with `eval <migraph::program::eval>`::
p.compile(cpu::target{});
argument result = p.eval({});
The easiest way to see the result is to print it::
std::cout << result;
Which will print ``3``.
We can also compile the program for the gpu as well.
Adding parameters
-----------------
Of course, this program will always produce the same value which is quite uninteresting. Instead, we want to pass an input to a program and compute a value based on the input. This can be done with a parameter. For example, we can modify the program to take an input ``x``::
program p;
instruction_ref x = p.add_parameter("x", {shape::int64_type});
instruction_ref two = p.add_literal(2);
p.add_instruction(add{}, x, two);
p.compile(cpu::target{});
This adds a parameter of type ``int64``, and compiles it for the ``cpu``. To run the program, we need to pass the parameter to it when we call `eval <migraph::program::eval>`::
argument result = p.eval({
{"x", literal{1}.get_argument()}
});
std::cout << result;
This will print ``3``.
A parameter is given as an `argument <migraph::argument>`. In this case, the simplest way of creating an `argument <migraph::argument>` is from a `literal <migraph::literal>`.
Tensor data
-----------
In this example we are just creating numbers, but the `shape <migraph::shape>` class can describe multi-dimensional tensors. For example, we can build a simple network with convolution and relu::
program p;
instruction_ref input = p.add_parameter("x", shape{shape::float_type, {1, 3, 32, 32}});
instruction_ref weights = p.add_parameter("w", shape{shape::float_type, {1, 3, 5, 5}});
instruction_ref conv = p.add_instruction(convolution{}, input, weights);
p.add_instruction(activation{"relu"}, conv);
Here we create two parameters for both the ``input`` and ``weights``. In the previous examples, we just created simple literals, however, most programs will take data from already allocated buffers(usually on the GPU). In this case, we can create `argument <migraph::argument>` objects directly from the pointers to the buffers::
// Compile the program
p.compile(gpu::target{});
// Allocated buffers by the user
float* input = ...;
float* weights = ...;
// Create the arguments
argument input_arg{shape{shape::float_type, {1, 3, 32, 32}}, input};
argument weights_arg{shape{shape::float_type, {1, 3, 32, 32}}, weights};
p.eval({{"x", input_arg}, {"w", weights_arg}})
An `argument <migraph::argument>` can handle memory buffers from either the GPU or the CPU, but when running the `program <migraph::program>`, buffers should be allocated for the corresponding target. That is, when compiling for the CPU, the buffers should be allocated on the CPU, and when compiling for the GPU the buffers should be allocated on the GPU.
Importing from onnx
-------------------
A `program <migraph::program>` can be built directly from an onnx file, which makes it easier to use neural networks directly from other frameworks. In this case, there is an ``parse_onnx`` function::
program p = parse_onnx("model.onnx");
p.compile(gpu::target{});
Data types
==========
shape
-----
.. doxygenstruct:: migraph::shape
literal
-------
.. doxygenstruct:: migraph::literal
argument
--------
.. doxygenstruct:: migraph::argument
raw_data
--------
.. doxygenstruct:: migraph::raw_data
.. doxygenfunction:: migraph::visit_all
tensor_view
-----------
.. doxygenstruct:: migraph::tensor_view
Operators
=========
operation
---------
.. doxygenstruct:: migraph::operation
operators
---------
.. doxygenfile:: operators.hpp
Passes
======
pass
----
.. doxygenstruct:: migraph::pass
dead_code_elimination
---------------------
.. doxygenstruct:: migraph::dead_code_elimination
auto_contiguous
---------------
.. doxygenstruct:: migraph::gpu::auto_contiguous
write_literals
--------------
.. doxygenstruct:: migraph::gpu::write_literals
Program
=======
instruction
-----------
.. doxygenstruct:: migraph::instruction
instruction_ref
---------------
.. cpp:type:: migraph::instruction_ref
References an instruction in the program.
program
-------
.. doxygenstruct:: migraph::program
parse_onnx
----------
.. doxygenfunction:: migraph::parse_onnx
Targets
=======
target
------
.. doxygenstruct:: migraph::target
gpu::target
-----------
.. doxygenstruct:: migraph::gpu::target
cpu::target
-----------
.. doxygenstruct:: migraph::cpu::cpu_target
......@@ -47,4 +47,14 @@ if(NOT CMakeGet_FOUND)
find_package(CMakeGet REQUIRED PATHS ${PREFIX})
endif()
# Set compiler to hcc if not set
if(NOT DEFINED ENV{CXX} AND NOT DEFINED CMAKE_CXX_COMPILER AND NOT DEFINED CMAKE_TOOLCHAIN_FILE)
find_program(HCC hcc PATHS /opt/rocm PATH_SUFFIXES bin)
if(HCC)
set(ENV{CXX} ${HCC})
else()
message(FATAL_ERROR "Cannot find hcc")
endif()
endif()
cmake_get_from(${CMAKE_CURRENT_LIST_DIR}/dev-requirements.txt PREFIX ${PREFIX} CMAKE_ARGS -DCMAKE_INSTALL_RPATH=${PREFIX}/lib ${PARSE_UNPARSED_ARGUMENTS})
add_library(rtg
add_library(migraph
auto_contiguous.cpp
dead_code_elimination.cpp
generate.cpp
program.cpp
shape.cpp
simplify_reshapes.cpp
)
rocm_clang_tidy_check(rtg)
target_include_directories(rtg PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>)
rocm_clang_tidy_check(migraph)
target_include_directories(migraph PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>)
add_subdirectory(onnx)
add_subdirectory(targets/cpu)
if(RTG_ENABLE_MIOPEN)
add_subdirectory(targets/miopen)
if(MIGRAPH_ENABLE_GPU)
add_subdirectory(targets/gpu)
endif()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment