merge from develop branch and resolve merge conflicts

.github/workflows/ci.yaml

@@ -19,7 +19,7 @@ jobs:
     # In this step, this action saves a list of existing images,
     # the cache is created without them in the post run.
     # It also restores the cache if it exists.
-    - uses: satackey/action-docker-layer-caching@v0.0.8
+    - uses: satackey/action-docker-layer-caching@v0.0.11
       # Ignore the failure of a step and avoid terminating the job.
       continue-on-error: true
 
@@ -65,7 +65,7 @@ jobs:
     # In this step, this action saves a list of existing images,
     # the cache is created without them in the post run.
     # It also restores the cache if it exists.
-    - uses: satackey/action-docker-layer-caching@v0.0.8
+    - uses: satackey/action-docker-layer-caching@v0.0.11
       # Ignore the failure of a step and avoid terminating the job.
       continue-on-error: true
 
@@ -108,7 +108,7 @@ jobs:
     # In this step, this action saves a list of existing images,
     # the cache is created without them in the post run.
     # It also restores the cache if it exists.
-    - uses: satackey/action-docker-layer-caching@v0.0.8
+    - uses: satackey/action-docker-layer-caching@v0.0.11
       # Ignore the failure of a step and avoid terminating the job.
       continue-on-error: true
 
@@ -142,10 +142,12 @@ jobs:
       with:
         python-version: 3.6
     - name: Install pyflakes
-      run: pip install pyflakes==2.3.1
+      run: pip install pyflakes==2.3.1 mypy==0.931
 
     - name: Run pyflakes
-      run: pyflakes examples/ tools/ src/ test/ doc/
+      run: |
+        pyflakes examples/ tools/ src/ test/ doc/
+        mypy tools/api.py
 
   linux:
 

CMakeLists.txt

@@ -200,6 +200,7 @@ rocm_enable_cppcheck(
     RULE_FILE
         ${CMAKE_CURRENT_SOURCE_DIR}/cppcheck.rules
     SOURCES
+        examples/
         src/
         test/
     INCLUDE

cmake/TargetFlags.cmake

 
+function(eval_and_strip_genex OUTPUT_VAR INPUT)
+    string(REPLACE "$<LINK_LANGUAGE:CXX>" "1" INPUT "${INPUT}")
+    string(REPLACE "$<COMPILE_LANGUAGE:CXX>" "1" INPUT "${INPUT}")
+    string(REPLACE "SHELL:" "" INPUT "${INPUT}")
+    string(REPLACE "$<BOOL:>" "0" INPUT "${INPUT}")
+    string(REGEX REPLACE "\\$<BOOL:(0|FALSE|false|OFF|off|N|n|IGNORE|ignore|NOTFOUND|notfound)>" "0" INPUT "${INPUT}")
+    string(REGEX REPLACE "\\$<BOOL:[^<>]*-NOTFOUND>" "0" INPUT "${INPUT}")
+    string(REGEX REPLACE "\\$<BOOL:[^$<>]*>" "1" INPUT "${INPUT}")
+    string(REPLACE "$<NOT:0>" "1" INPUT "${INPUT}")
+    string(REPLACE "$<NOT:1>" "0" INPUT "${INPUT}")
+    string(REGEX REPLACE "\\$<0:[^<>]*>" "" INPUT "${INPUT}")
+    string(REGEX REPLACE "\\$<1:([^<>]*)>" "\\1" INPUT "${INPUT}")
+    string(GENEX_STRIP "${INPUT}" INPUT)
+    set(${OUTPUT_VAR} "${INPUT}" PARENT_SCOPE)
+endfunction()
+
 function(get_target_property2 VAR TARGET PROPERTY)
     get_target_property(_pflags ${TARGET} ${PROPERTY})
     if(_pflags)
+        eval_and_strip_genex(_pflags "${_pflags}")
         set(${VAR} ${_pflags} PARENT_SCOPE)
     else()
         set(${VAR} "" PARENT_SCOPE)
     endif()
 endfunction()
 
+function(flags_requires_arg OUTPUT_VAR FLAG)
+    set(_args -x -isystem)
+    if(FLAG IN_LIST _args)
+        set(${OUTPUT_VAR} 1 PARENT_SCOPE)
+    else()
+        set(${OUTPUT_VAR} 0 PARENT_SCOPE)
+    endif()
+endfunction()
 
 macro(append_flags FLAGS TARGET PROPERTY PREFIX)
     get_target_property2(_pflags ${TARGET} ${PROPERTY})
+    set(_requires_arg 0)
     foreach(FLAG ${_pflags})
-        if(TARGET ${FLAG})
+        string(STRIP "${FLAG}" FLAG)
+        if(FLAG)
+            if(TARGET ${FLAG} AND NOT _requires_arg)
                 target_flags(_pflags2 ${FLAG})
                 string(APPEND ${FLAGS} " ${_pflags2}")
             else()
                 string(APPEND ${FLAGS} " ${PREFIX}${FLAG}")
             endif()
+            flags_requires_arg(_requires_arg "${FLAG}")
+        endif()
     endforeach()
 endmacro()
 
 macro(append_link_flags FLAGS TARGET PROPERTY)
     get_target_property2(_pflags ${TARGET} ${PROPERTY})
+    set(_requires_arg 0)
     foreach(FLAG ${_pflags})
-        if(TARGET ${FLAG})
+        string(STRIP "${FLAG}" FLAG)
+        if(FLAG)
+            if(TARGET ${FLAG} AND NOT _requires_arg)
                 target_flags(_pflags2 ${FLAG})
                 string(APPEND ${FLAGS} " ${_pflags2}")
             elseif(FLAG MATCHES "^-.*")
@@ -34,6 +67,8 @@ macro(append_link_flags FLAGS TARGET PROPERTY)
             else()
                 string(APPEND ${FLAGS} " -l${FLAG}")
             endif()
+            flags_requires_arg(_requires_arg "${FLAG}")
+        endif()
     endforeach()
 endmacro()
 

doc/src/conf.py

@@ -32,8 +32,10 @@ import re
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'breathe', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', 'sphinx_rtd_theme'
+    'breathe', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', 'sphinx_rtd_theme',
+    'sphinx.ext.autosectionlabel'
 ]
+autosectionlabel_prefix_document = True
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

doc/src/dev/tools.rst

+Tools
+=====
+
+roctx.py
+--------
+MIGraphX driver provides `roctx` command which can be used with `rocprof` binary to get marker timing information for each MIGraphX operator.  
+In order to help user to process timing information, rocTX helper script is provided at `tools/roctx.py`.
+The `roctx.py` helper script provides two main functionality: `run` and `parse`. Available knobs and usage are given below:
+
+::
+
+    Usage: roctx.py [-h] [--json-path json_path] [--out out]
+    [--study-name study-name] [--repeat repeat] [--parse]
+    [--run run] [--debug]
+
+.. option::  --run
+
+Runs `migraphx-driver roctx` command and given `migraphx-driver` knobs, and then parses the results, providing GPU kernel timing information.
+MIGraphX knobs can be given via a string to `--run` knob. Please see the examples below.
+
+.. option::  --parse
+
+Given `--json-path`, parses JSON file and provides GPU kernel timing information.
+
+.. option::  --out
+
+Output folder
+
+.. option::  --study-name
+
+Optional. Allows user to name a study for easier interpretation. Defaults to timestamp.
+
+.. option::  --repeat
+
+Number of iterations. Set to **2** by default.
+
+.. option::  --debug
+
+Provides additional debug information related to data. Only use for debugging purposes.
+
+**Examples:**
+
+**Running inference with rocTX for a given ONNX file:**
+::
+    python roctx.py --run '--onnx --gpu fcn-resnet50-11.onnx' --out output_folder --repeat 5
+
+After a run, similar to output given below is expected at terminal. The output will provide `SUM`, `MIN`, `MAX` and `COUNT` information for each kernel executed for a given model.
+Average total time is also provided. There are three files provided for reference:
+
+1. `OUTPUT CSV FILE` provides a summary of the run, providing utilized MIGraphX knobs and related kernel timing information
+2. `KERNEL TIMING DETAILS` provides the hotspot kernel timing information
+3. This will provide all output data related to all iterations executed during a run.
+
+An example output:
+
+.. image:: ./roctx1.jpg
+
+Hotspot kerel timing information:
+
+.. image:: ./roctx2.jpg
+
+**Parsing an already existing JSON file:**
+::
+
+    python roctx.py --parse --json-path ../trace.json
\ No newline at end of file

doc/src/developer_guide.rst

@@ -13,3 +13,4 @@ Developer Guide
    dev/quantization
    dev/pass
    dev/matchers
+   dev/tools

doc/src/driver.rst

@@ -61,3 +61,21 @@ Verify each instruction
 .. option::  -r, --reduce
 
 Reduce program and verify
+
+roctx
+----
+
+.. program:: migraphx-driver roctx
+
+Provides marker information for each operation, allowing MIGraphX to be used with `rocprof <https://rocmdocs.amd.com/en/latest/ROCm_Tools/ROCm-Tools.html>`_ for performance analysis.
+This allows user to get GPU-level kernel timing information.
+An example command line combined with rocprof for tracing purposes is given below:
+
+.. code-block:: bash
+
+    /opt/rocm/bin/rocprof --hip-trace --roctx-trace --flush-rate 1ms --timestamp on -d <OUTPUT_PATH> --obj-tracking on /opt/rocm/bin/migraphx-driver roctx <ONNX_FILE> <MIGRAPHX_OPTIONS>
+
+After `rocprof` is run, the output directory will contain trace information for HIP, HCC and ROCTX in seperate `.txt` files.
+To understand the interactions between API calls, it is recommended to utilize `roctx.py` helper script as desribed in :ref:`dev/tools:rocTX` section. 
+
+.. include:: ./driver/compile.rst
\ No newline at end of file

examples/migraphx/cpp_parse_load_save/parse_load_save.cpp

@@ -24,7 +24,6 @@ int main(int argc, char** argv)
         return 0;
     }
 
-    char* parse_arg        = getCmdOption(argv + 2, argv + argc, "--parse");
     char* load_arg         = getCmdOption(argv + 2, argv + argc, "--load");
     char* save_arg         = getCmdOption(argv + 2, argv + argc, "--save");
     const char* input_file = argv[1];

examples/nlp/python_bert_squad/requirements_bertsquad.txt

-tensorflow==2.5.1
+tensorflow==2.5.2
 onnxruntime
 tokenizers
\ No newline at end of file

examples/vision/python_3dunet/3dunet_inference.ipynb

@@ -10,6 +10,16 @@
     "https://github.com/naomifridman/Unet_Brain_tumor_segmentation"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "09ceec31",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install SimpleITK matplotlib scikit-image"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

examples/vision/python_nfnet/nfnet_inference.ipynb

@@ -17,7 +17,9 @@
     "- How to optimize NFNet ONNX model with AMD MIGraphX.\n",
     "- How to run inference on AMD GPU with the optimized ONNX model.\n",
     "\n",
-    "The NFNet utilized in this example is the smallest NFNet version, F0: 71.5M parameters (83.6% top-1 accuracy on ImageNet)"
+    "The NFNet utilized in this example is the smallest NFNet version, F0: 71.5M parameters (83.6% top-1 accuracy on ImageNet)\n",
+    "\n",
+    "Please make sure MIGraphX Python API is installed following the instructions at Github page."
    ]
   },
   {
@@ -107,7 +109,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "with open('../python_api_inference/imagenet_simple_labels.json') as json_data:\n",
+    "with open('../python_resnet50/imagenet_simple_labels.json') as json_data:\n",
     "    labels = json.load(json_data)"
    ]
   },

examples/vision/python_nfnet/requirements_nfnet.txt

 opencv-python
 onnxruntime
+image
\ No newline at end of file

src/cpp_generator.cpp

 #include <migraphx/cpp_generator.hpp>
 #include <migraphx/module.hpp>
 #include <migraphx/operation.hpp>
+#include <migraphx/ranges.hpp>
 #include <migraphx/instruction.hpp>
 #include <migraphx/builtin.hpp>
 #include <migraphx/stringutils.hpp>
@@ -51,6 +52,7 @@ cpp_generator::function& cpp_generator::function::set_types(const module& m)
 cpp_generator::function&
 cpp_generator::function::set_types(const module& m, const std::function<std::string(shape)>& parse)
 {
+    this->params.clear();
     auto pmap = m.get_parameter_shapes();
     std::map<std::string, shape> input_map(pmap.begin(), pmap.end());
     std::transform(
@@ -63,11 +65,31 @@ cpp_generator::function::set_types(const module& m, const std::function<std::str
     return *this;
 }
 
+cpp_generator::function& cpp_generator::function::set_generic_types(const module& m)
+{
+    this->params.clear();
+    auto pmap = m.get_parameter_shapes();
+    std::map<std::string, shape> input_map(pmap.begin(), pmap.end());
+    std::transform(
+        input_map.begin(), input_map.end(), std::back_inserter(this->params), [&](auto&& p) {
+            return param{p.first, "T" + p.first};
+        });
+
+    std::transform(input_map.begin(),
+                   input_map.end(),
+                   std::back_inserter(this->tparams),
+                   [&](auto&& p) { return "class T" + p.first; });
+    this->return_type = "auto";
+    return *this;
+}
+
 struct cpp_generator_impl
 {
     std::stringstream fs{};
     std::size_t function_count                                = 0;
     std::function<std::string(std::string)> fmap              = nullptr;
+    std::function<std::string(shape)> fresult                 = nullptr;
+    std::unordered_map<std::string, std::string> point_op_map = {};
 };
 cpp_generator::cpp_generator() : impl(std::make_unique<cpp_generator_impl>()) {}
 
@@ -83,15 +105,30 @@ cpp_generator::~cpp_generator() noexcept = default;
 
 void cpp_generator::fmap(const std::function<std::string(std::string)>& f) { impl->fmap = f; }
 
+void cpp_generator::fresult(const std::function<std::string(shape)>& f) { impl->fresult = f; }
+
+void cpp_generator::add_point_op(const std::string& op_name, const std::string& code)
+{
+    impl->point_op_map[op_name] = code;
+}
+
 std::string cpp_generator::generate_point_op(const operation& op,
                                              const std::vector<std::string>& args)
 {
     auto v = op.to_value();
+    std::string code;
+    if(contains(impl->point_op_map, op.name()))
+    {
+        code = impl->point_op_map.at(op.name());
+    }
+    else
+    {
         auto attributes = op.attributes();
         if(not attributes.contains("point_op"))
             MIGRAPHX_THROW("op is missing point_op attribute: " + op.name());
-    return interpolate_string(attributes["point_op"].to<std::string>(),
-                              [&](auto start, auto last) -> std::string {
+        code = attributes["point_op"].to<std::string>();
+    }
+    return interpolate_string(code, [&](auto start, auto last) -> std::string {
         auto key = trim({start, last});
         if(key.empty())
             MIGRAPHX_THROW("Empty parameter");
@@ -140,7 +177,12 @@ cpp_generator::function cpp_generator::generate_module(const module& m)
                            ins->inputs().end(),
                            std::back_inserter(args),
                            [&](auto i) { return names.at(i); });
-            return this->generate_point_op(ins->get_operator(), args);
+
+            auto s = this->generate_point_op(ins->get_operator(), args);
+            if(impl->fresult)
+                return impl->fresult(ins->get_shape()) + '(' + s + ')';
+            else
+                return s;
         });
     return f;
 }
@@ -148,6 +190,8 @@ cpp_generator::function cpp_generator::generate_module(const module& m)
 std::string cpp_generator::create_function(const cpp_generator::function& f)
 {
     impl->function_count++;
+    if(not f.tparams.empty())
+        impl->fs << "template<" << join_strings(f.tparams, ", ") << ">\n";
     std::string name = f.name.empty() ? "f" + std::to_string(impl->function_count) : f.name;
     impl->fs << join_strings(f.attributes, " ") << " " << f.return_type << " " << name;
     char delim = '(';

src/fuse_pointwise.cpp

@@ -33,9 +33,6 @@ static void create_pointwise_modules(module_pass_manager& mpm)
     {
         if(not ins->get_operator().attributes().get("pointwise", false))
             continue;
-        // Skip convert op for now
-        if(ins->name() == "convert")
-            continue;
         assert(ins->get_operator().attributes().contains("point_op"));
         auto* pm = mpm.create_module(mpm.get_module().name() + ":pointwise" + std::to_string(n++));
         pm->set_bypass();
@@ -129,22 +126,25 @@ static std::vector<instruction_ref> append_pointwise_module(instruction_ref ins,
 static bool find_pointwise_modules(module& m)
 {
     bool changed = false;
+    auto last    = std::prev(m.end());
     for(auto ins : iterator_for(m))
     {
         if(ins->name() != "pointwise")
             continue;
-        if(ins->outputs().empty())
+        if(ins->outputs().empty() and ins != last)
             continue;
         auto it = std::find_if(ins->inputs().begin(), ins->inputs().end(), [&](auto i) {
             return i->name() == "pointwise" and i->outputs().size() == 1;
         });
         if(it == ins->inputs().end())
             continue;
+        auto input = *it;
+
+        auto new_inputs = append_pointwise_module(input, ins);
+        m.replace_instruction(input, input->get_operator(), new_inputs, input->module_inputs());
+        m.replace_instruction(ins, input);
+        m.move_instruction(input, ins);
 
-        auto new_inputs = append_pointwise_module(*it, ins);
-        m.replace_instruction(*it, (*it)->get_operator(), new_inputs, (*it)->module_inputs());
-        m.replace_instruction(ins, *it);
-        m.move_instruction(*it, ins);
         changed = true;
     }
     return changed;

src/include/migraphx/cpp_generator.hpp

@@ -34,6 +34,7 @@ struct cpp_generator
         std::string return_type             = "void";
         std::string name                    = "";
         std::vector<std::string> attributes = {};
+        std::vector<std::string> tparams    = {};
         function& set_body(const module& m, const generate_module_callback& g);
         function& set_body(const std::string& s)
         {
@@ -52,6 +53,7 @@ struct cpp_generator
         }
         function& set_types(const module& m);
         function& set_types(const module& m, const std::function<std::string(shape)>& parse);
+        function& set_generic_types(const module& m);
     };
 
     cpp_generator();
@@ -66,6 +68,10 @@ struct cpp_generator
 
     void fmap(const std::function<std::string(std::string)>& f);
 
+    void fresult(const std::function<std::string(shape)>& f);
+
+    void add_point_op(const std::string& op_name, const std::string& code);
+
     std::string generate_point_op(const operation& op, const std::vector<std::string>& args);
 
     std::string str() const;

src/include/migraphx/iota_iterator.hpp

@@ -89,6 +89,13 @@ inline std::ptrdiff_t operator-(basic_iota_iterator<F, Iterator> x,
     return x.index - y.index;
 }
 
+template <class F, class Iterator>
+inline basic_iota_iterator<F, Iterator> operator-(basic_iota_iterator<F, Iterator> x,
+                                                  std::ptrdiff_t y)
+{
+    return x -= y;
+}
+
 template <class F, class Iterator>
 inline bool operator==(basic_iota_iterator<F, Iterator> x, basic_iota_iterator<F, Iterator> y)
 {