Merge branch 'develop' into blas_tuning

doc/src/driver/compile.rst → docs/driver/compile.rst

@@ -32,10 +32,6 @@ Disable fast math optimization
 
 Perform an exhaustive search to find the fastest version of generated kernels for selected backend
 
-.. options:: --split-single-dyn-dim
-
-Enable the split single dynamic dimension pass
-
 .. option::  --fp16
 
 Quantize for fp16

doc/src/driver/read.rst → docs/driver/read.rst

@@ -82,6 +82,10 @@ Print out program in text format.
 
 Print out program in binary format.
 
+.. option::  --py
+
+Print out program using python API.
+
 .. option::  --output, -o [std::string]
 
 Output to file.

doc/src/index.rst → docs/index.rst

@@ -3,18 +3,10 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to AMD MIGraphX's documentation!
-========================================
-
-.. toctree::
-   :maxdepth: 3
-   :caption: Contents:
-
-   py_user_guide
-   cpp_user_guide
-   driver
-   contributor_guide
+AMD MIGraphX documentation
+==========================
 
+AMD MIGraphX is AMD's graph inference engine that accelerates machine learning model inference. 
 
 Indices and tables
 ==================

doc/src/reference/py.rst → docs/reference/py.rst

@@ -6,9 +6,9 @@ Python Reference
 shape
 -----
 
-.. py:class:: shape(type, lens, strides=None)
+.. py:class:: shape(type, lens, strides=None, dyn_dims)
 
-    Describes the shape of a tensor. This includes size, layout, and data type/
+    Describes the shape of a tensor. This includes size, layout, and data type. Can be a dynamic shape by using dyn_dims.
 
 .. py:method:: type()
 
@@ -34,6 +34,12 @@ shape
 
     :rtype: int
 
+.. py:method:: dyn_dims()
+
+    The dynamic dimensions of the shape.
+
+    :rtype: list[dynamic_dimension]
+
 .. py:method:: bytes()
 
     The number of bytes the shape uses.
@@ -46,6 +52,12 @@ shape
 
     :rtype: int
 
+.. py:method:: ndim()
+
+    The number of dimensions for the shape.
+
+    :rtype: int
+
 .. py:method:: packed()
 
     Returns true if the shape is packed.
@@ -64,6 +76,12 @@ shape
 
     :rtype: bool
 
+.. py:method:: dynamic()
+
+    Returns true if the shape is dynamic.
+
+    :rtype: bool
+
 .. py:method:: standard()
 
     Returns true if the shape is a standard shape. That is, the shape is both packed and not transposed.
@@ -76,6 +94,18 @@ shape
 
     :rtype: bool
 
+dynamic_dimension
+--------
+
+.. py:class:: dynamic_dimension(min, max, optimals)
+
+    Construct a dynamic_dimension from a minimum, a maximum, and optionally a set of optimals.
+
+.. py:method:: is_fixed()
+    
+    Returns true if the dynamic_dimension is fixed.
+
+    :rtype : int
 
 argument
 --------
@@ -121,6 +151,15 @@ argument
 
     :rtype: argument
 
+.. py:function:: create_argument(s, values)
+
+    Create an argument of shape s with a set of values.
+
+    :param shape s: Shape of argument to create.
+    :param list values: Values to put in the argument. Must be the same number of elements as the shape.
+
+    :rtype: argument
+
 .. py:function:: argument_from_pointer(shape, address)
 
     Create argument from data stored in given address without copy.
@@ -292,8 +331,10 @@ parse_onnx
     Load and parse an onnx file.
 
     :param str filename: Path to file.
-    :param str default_dim_value: default batch size to use (if not specified in onnx file).
+    :param str default_dim_value: default dimension to use (if not specified in onnx file).
+    :param dynamic_dimension default_dyn_dim_value: default dynamic_dimension value to use.
     :param str map_input_dims: Explicitly specify the dims of an input.
+    :param list[dynamic_dimension] map_dyn_input_dims: Explicitly specify the dynamic_dimensions of an input.
     :param str skip_unknown_operators: Continue parsing onnx file if an unknown operator is found.
     :param str print_program_on_error: Print program if an error occurs.
     :param int max_loop_iterations: Maximum iteration number for the loop operator.

examples/migraphx/cpp_dynamic_batch/CMakeLists.txt

+#####################################################################################
+# The MIT License (MIT)
+#
+# Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#####################################################################################
+cmake_minimum_required(VERSION 3.5)
+project (cpp_dynamic_batch)
+
+set (CMAKE_CXX_STANDARD 14)
+set (EXAMPLE dynamic_batch)
+
+list (APPEND CMAKE_PREFIX_PATH /opt/rocm)
+find_package (migraphx)
+
+message("source file: " ${EXAMPLE}.cpp " ---> bin: " ${EXAMPLE})
+add_executable(${EXAMPLE} ${EXAMPLE}.cpp)
+
+target_link_libraries(${EXAMPLE} migraphx::c)

examples/migraphx/cpp_dynamic_batch/README.md

+# Running ONNX model with dynamic batch
+
+
+## Description
+This examples demonstrates how to run a graph program with dynamic batch using the MIGraphX C++ API. 
+
+
+## Creating dynamic dimension objects
+`dynamic_dimension` objects are used in MIGraphX to specify a range of dimension values from a minimum value to a maximum value and optimal values that the tensor can be at model evaluation time.
+A dynamic shape is defined by a list of `dynamic_dimensions` while a static shape only has fixed dimension values.
+For example, a `dynamic_dimension` with `{min:1, max:10, optimals:{1, 4, 10}}` means that the dimension can be any value from 1 through 10 with the optimal values being 1, 4, and 10.
+Supplied optimal values may allow MIGraphX to optimize the program for those specific shapes.
+
+A fixed `dynamic_dimension` can be specified by setting the `min` and `max` to the same value (ex. `{min:3, max:3}`).
+A dynamic shape specified solely by fixed `dynamic_dimension` objects will be converted to a static shape during parsing.
+This can be useful for setting a static shape using the `set_dyn_input_parameter_shape()` method discussed later in this document.
+
+
+## Parsing
+ONNX graphs [ONNX](https://onnx.ai/get-started.html) can be parsed by MIGraphX to create a runnable program with dynamic batch sizes.
+The dynamic batch range must be specified by a `dynamic_dimension` object.
+
+One method to set the `dynamic_dimension` object works for ONNX files that only have symbolic variables for the batch dimensions:
+```
+migraphx::program p;
+migraphx::onnx_options options;
+options.set_default_dyn_dim_value(migraphx::dynamic_dimension{1, 4, {2, 4}});
+p = parse_onnx(input_file, options);
+```
+
+Another option that can run any ONNX model with dynamic batch sizes uses the dynamic input map where the entire shape of the input parameter is supplied:
+```
+migraphx::program p;
+migraphx::onnx_options options;
+migraphx::dynamic_dimensions dyn_dims = {migraphx::dynamic_dimension{1, 4, {2, 4}},
+                                         migraphx::dynamic_dimension{3, 3},
+                                         migraphx::dynamic_dimension{4, 4},
+                                         migraphx::dynamic_dimension{4, 4}};
+options.set_dyn_input_parameter_shape("input", dyn_dims);
+p = parse_onnx(input_file, options);
+```
+
+## Compiling
+Currently the MIGraphX C/C++ API requires that `offload_copy` be enabled for compiling dynamic batch programs.
+Here is a snippet of compiling a model with `offload_copy` enabled:
+```
+migraphx::compile_options c_options;
+c_options.set_offload_copy();
+p.compile(migraphx::target("gpu"), c_options);
+```
+where `p` is the `migraphx::program`.
+
+
+## Saving and Loading
+A dynamic batch MIGraphX program can be saved and loaded to/from a MXR file the same way as a fully static shape program.
+
+
+## Executing the dynamic batch model
+The compiled dynamic batch model can be executed the same way as a static model by supplying the input data as `arguments` in a `program_parameters` object.
+
+
+## Running the Example
+Your ROCm installation could be installed in a location other than the one specified in the CMakeLists.txt.
+You can set `LD_LIBRARY_PATH` or `CMAKE_PREFIX_PATH` to that location so that this program can still build.
+
+The provided example is [`dynamic_batch.cpp`](./dynamic_batch.cpp)
+To compile and run the example from this directory:
+```
+$ mkdir build
+$ cd build
+$ cmake ..
+$ make
+```
+There will now be an executable named `dynamic_batch` with the following usage:
+```
+$ ./dynamic_batch
+```

examples/migraphx/cpp_dynamic_batch/dynamic_batch.cpp

+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <string>
+#include <algorithm>
+
+// MIGraphX C++ API
+#include <migraphx/migraphx.hpp>
+
+int main(int argc, char** argv)
+{
+    migraphx::onnx_options o_options;
+    migraphx::dynamic_dimensions dyn_dims = {migraphx::dynamic_dimension{1, 4, {2, 4}},
+                                             migraphx::dynamic_dimension{3, 3},
+                                             migraphx::dynamic_dimension{4, 4},
+                                             migraphx::dynamic_dimension{5, 5}};
+    o_options.set_dyn_input_parameter_shape("0", dyn_dims);
+    auto p = migraphx::parse_onnx("../add_scalar_test.onnx", o_options);
+    migraphx::compile_options c_options;
+    c_options.set_offload_copy();
+    p.compile(migraphx::target("gpu"), c_options);
+
+    // batch size = 2
+    std::vector<uint8_t> a(2 * 3 * 4 * 5, 3);
+    std::vector<uint8_t> b = {2};
+    migraphx::program_parameters pp;
+    migraphx::shape s = migraphx::shape(migraphx_shape_uint8_type, {2, 3, 4, 5});
+    pp.add("0", migraphx::argument(s, a.data()));
+    pp.add("1", migraphx::argument(migraphx::shape(migraphx_shape_uint8_type, {1}, {0}), b.data()));
+    auto outputs = p.eval(pp);
+    auto result  = outputs[0];
+    std::vector<uint8_t> c(2 * 3 * 4 * 5, 5);
+    if(bool{result == migraphx::argument(s, c.data())})
+    {
+        std::cout << "Successfully executed dynamic batch add\n";
+    }
+    else
+    {
+        std::cout << "Failed dynamic batch add\n";
+    }
+
+    return 0;
+}

examples/migraphx/cpp_parse_load_save/parse_load_save.cpp

@@ -114,7 +114,7 @@ int main(int argc, char** argv)
 char* getCmdOption(char** begin, char** end, const std::string& option)
 {
     char** itr = std::find(begin, end, option);
-    if(itr != end && ++itr != end)
+    if(itr != end and ++itr != end)
     {
         return *itr;
     }

examples/migraphx/migraphx_driver/README.md

@@ -53,7 +53,6 @@ See below for a comprehensive list of commands and option arguments, as well as
 | --enable-offload-copy                    | Enable implicit offload copying                           |
 | --disable-fast-math                      | Disable fast math optimization                            |
 | --exhaustive-tune                        | Enable exhaustive search to find fastest kernel           |
-| --split-single-dyn-dim                   | Enable split_single_dyn_dim compiler pass                 |
 | --fp16                                   | Quantize for fp16                                         |
 | --int8                                   | Quantize for int8                                         |
 | --tolerance                              | Tolerance for errors                                      |

examples/nlp/python_bert_squad/requirements_bertsquad.txt

@@ -21,6 +21,6 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #####################################################################################
-tensorflow==2.9.3
+tensorflow==2.11.1
 onnxruntime
 tokenizers
\ No newline at end of file

examples/vision/cpp_mnist/mnist_inference.cpp

@@ -160,9 +160,9 @@ int main(int argc, char** argv)
     auto lengths = shape.lengths();
     auto num_results =
         std::accumulate(lengths.begin(), lengths.end(), 1, std::multiplies<size_t>());
-    float* results = reinterpret_cast<float*>(outputs[0].data());
-    float* max     = std::max_element(results, results + num_results);
-    int answer     = max - results;
+    float* results   = reinterpret_cast<float*>(outputs[0].data());
+    const float* max = std::max_element(results, results + num_results);
+    int answer       = max - results;
 
     std::cout << std::endl
               << "Randomly chosen digit: " << rand_digit << std::endl
@@ -192,12 +192,12 @@ void read_nth_digit(const int n, std::vector<float>& digit)
         for(int i = 0; i < HEIGHT * WIDTH; ++i)
         {
             unsigned char temp = 0;
-            file.read((char*)&temp, sizeof(temp));
+            file.read(reinterpret_cast<char*>(&temp), sizeof(temp));
             if(d == n)
             {
                 float data = temp / 255.0;
                 digit.push_back(data);
-                std::cout << SYMBOLS[(int)(data * 10) % 11];
+                std::cout << SYMBOLS[static_cast<int>(data * 10) % 11];
                 if((i + 1) % WIDTH == 0)
                     std::cout << std::endl;
             }

hip-clang.docker

@@ -6,13 +6,12 @@ ARG PREFIX=/usr/local
 RUN dpkg --add-architecture i386
 
 # Add rocm repository
-RUN sh -c 'echo deb [arch=amd64 trusted=yes] http://repo.radeon.com/rocm/apt/5.4.2/ ubuntu main > /etc/apt/sources.list.d/rocm.list'
+RUN sh -c 'echo deb [arch=amd64 trusted=yes] http://repo.radeon.com/rocm/apt/5.6/ focal main > /etc/apt/sources.list.d/rocm.list'
 
 # Install dependencies
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \
     apt-utils \
     build-essential \
-    clang-format-10 \
     cmake \
     curl \
     doxygen \
@@ -47,7 +46,7 @@ ENV LANG=C.UTF-8
 RUN pip3 install yapf==0.28.0
 
 # Install doc requirements
-ADD doc/requirements.txt /doc-requirements.txt
+ADD docs/.sphinx/requirements.txt /doc-requirements.txt
 RUN pip3 install -r /doc-requirements.txt
 
 # Install dependencies
@@ -55,6 +54,12 @@ ADD dev-requirements.txt /dev-requirements.txt
 ADD requirements.txt /requirements.txt
 ADD rbuild.ini /rbuild.ini
 
+# Temporarily install a new cmake until switching to ubuntu 22.04
+RUN pip3 install cmake==3.22.1
+
 COPY ./tools/install_prereqs.sh /
 RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh
 
+# Install MLIR
+ADD mlir-requirements.txt /mlir-requirements.txt
+RUN cget -p /usr/local install -f /mlir-requirements.txt

doc/requirements.txt → mlir-requirements.txt

@@ -21,8 +21,4 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #####################################################################################
-docutils==0.17.1
-sphinx==4.2.0
-breathe==4.31.0
-sphinx_rtd_theme==1.0.0
-# git+https://github.com/arximboldi/breathe@fix-node-parent
+ROCmSoftwarePlatform/rocMLIR@ea15b3597ce55b9088621818228595dd48fb6ec0 -DBUILD_FAT_LIBROCKCOMPILER=On

rbuild.ini

@@ -6,7 +6,9 @@ deps =
     -f requirements.txt
 
 [gh]
-ignore = danmar/cppcheck
+ignore =
+    danmar/cppcheck
+    ROCmSoftwarePlatform/rocMLIR
 deps =
     -f dev-requirements.txt
     oneapi-src/oneDNN@v1.7

requirements.txt

@@ -24,7 +24,8 @@
 google/protobuf@v3.11.0 -DCMAKE_POSITION_INDEPENDENT_CODE=On -X subdir -Dprotobuf_BUILD_TESTS=Off
 nlohmann/json@v3.8.0
 live-clones/blaze@v3.8 -X header -DHEADER_DIR=blaze -H sha256:d0ff011f47538285178908ea5f2cab46bb6a8f55b1edb6e03224a82dbc1a3212
-ROCmSoftwarePlatform/half@rocm-5.4.2
+ROCmSoftwarePlatform/half@rocm-5.6.0
 pybind/pybind11@d159a563383d10c821ba7b2a71905d1207db6de4 --build
 msgpack/msgpack-c@cpp-3.3.0 -DMSGPACK_BUILD_TESTS=Off
 sqlite3@3.17 -DCMAKE_POSITION_INDEPENDENT_CODE=On
+ROCmSoftwarePlatform/composable_kernel@5172ec5280f14974beee2acf1af1db3b2670244c -DCK_BUILD_JIT_LIB=On -DCMAKE_POSITION_INDEPENDENT_CODE=On

src/CMakeLists.txt

@@ -22,6 +22,8 @@
 # THE SOFTWARE.
 #####################################################################################
 
+include(ExportHeader)
+
 include(ROCMInstallTargets)
 include(ROCMPackageConfigHelpers)
 include(RegisterOp)
@@ -94,6 +96,7 @@ add_library(migraphx
     simplify_algebra.cpp
     simplify_reshapes.cpp
     split_single_dyn_dim.cpp
+    target.cpp
     tmp_dir.cpp
     value.cpp
     verify_args.cpp
@@ -126,10 +129,11 @@ register_migraphx_ops(
     contiguous
     convert
     convolution
+    convolution_backwards
     cosh
     cos
-    deconvolution
     dequantizelinear
+    dimensions_of
     div
     dot
     elu
@@ -195,6 +199,7 @@ register_migraphx_ops(
     roialign
     round
     rsqrt
+    run_on_target
     scalar
     scatter_add
     scatter_mul
@@ -243,12 +248,11 @@ endif()
 target_link_libraries(migraphx PRIVATE -ldl)
 
 target_include_directories(migraphx SYSTEM PUBLIC $<BUILD_INTERFACE:${HALF_INCLUDE_DIR}>)
-
-find_package(Threads)
 target_link_libraries(migraphx PUBLIC Threads::Threads)
 
 find_package(nlohmann_json 3.8.0 REQUIRED)
 target_link_libraries(migraphx PRIVATE nlohmann_json::nlohmann_json)
+migraphx_generate_export_header(migraphx)
 
 find_package(PkgConfig)
 pkg_check_modules(SQLITE3 REQUIRED IMPORTED_TARGET sqlite3)

src/api/CMakeLists.txt

@@ -26,6 +26,7 @@ add_library(migraphx_c
     api.cpp
 )
 set_target_properties(migraphx_c PROPERTIES EXPORT_NAME c)
+migraphx_generate_export_header(migraphx_c DIRECTORY migraphx/api)
 
 # migraphx_c is stable API interface library. SO version of this should be 
 # bumped when binary compatibility is broken.