Rocm56 dynbatch (#1737)

* Removed split_single_dyn_dim compile flag (#1711)
* Update C/C++ API for dynamic batch (#1712)
* Python API update for dynamic batch (#1723)
* Dynamic batch C++ API example #1728
* Optimize file space of github runners (#1743)
Co-authored-by: Charlie Lin <charlie.lin@amd.com>

.github/workflows/ci.yaml

@@ -23,7 +23,7 @@ jobs:
     steps:
     - name: Free space
       run: |
-        sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku 
+        sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku /opt/az /opt/hostedtoolcache/Ruby /opt/hostedtoolcache/node /opt/hostedtoolcache/go /opt/hostedtoolcache/Python/2* /opt/hostedtoolcache/CodeQL /opt/microsoft /etc/skel /home/runner/.rustup/toolchains 
 
     - uses: actions/checkout@v3
 
@@ -48,7 +48,8 @@ jobs:
         restore-keys: tidy-cache-
         
     - name: Build the Docker image
-      run: docker build . --file hip-clang.docker --tag migraphx
+      run: |
+        docker build . --file hip-clang.docker --tag migraphx
 
     - name: Clang tidy
       shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data migraphx bash < {0}"
@@ -90,7 +91,8 @@ jobs:
 
     steps:
     - name: Free space
-      run: sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku
+      run: |
+        sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku /opt/az /opt/hostedtoolcache/Ruby /opt/hostedtoolcache/node /opt/hostedtoolcache/go /opt/hostedtoolcache/Python/2* /opt/hostedtoolcache/CodeQL /opt/microsoft /etc/skel /home/runner/.rustup/toolchains 
     - uses: actions/checkout@v3
 
     # In this step, this action saves a list of existing images,
@@ -152,7 +154,8 @@ jobs:
 
     steps:
     - name: Free space
-      run: sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku 
+      run: |
+        sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku /opt/az /opt/hostedtoolcache/Ruby /opt/hostedtoolcache/node /opt/hostedtoolcache/go /opt/hostedtoolcache/Python/2* /opt/hostedtoolcache/CodeQL /opt/microsoft /etc/skel /home/runner/.rustup/toolchains 
     - uses: actions/checkout@v3
 
     # In this step, this action saves a list of existing images,
@@ -193,7 +196,8 @@ jobs:
 
     steps:
     - name: Free space
-      run: sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku
+      run: |
+        sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku /opt/az /opt/hostedtoolcache/Ruby /opt/hostedtoolcache/node /opt/hostedtoolcache/go /opt/hostedtoolcache/Python/2* /opt/hostedtoolcache/CodeQL /opt/microsoft /etc/skel /home/runner/.rustup/toolchains 
     - uses: actions/checkout@v3
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -214,7 +218,8 @@ jobs:
 
     steps:
     - name: Free space
-      run: sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku
+      run: |
+        sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku /opt/az /opt/hostedtoolcache/Ruby /opt/hostedtoolcache/node /opt/hostedtoolcache/go /opt/hostedtoolcache/Python/2* /opt/hostedtoolcache/CodeQL /opt/microsoft /etc/skel /home/runner/.rustup/toolchains 
     - uses: actions/checkout@v3
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -246,7 +251,7 @@ jobs:
     steps:
     - name: Free space and install rbuild, lld
       run: |
-        sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws*  /usr/local/lib/heroku
+        sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku /opt/az /opt/hostedtoolcache/Ruby /opt/hostedtoolcache/node /opt/hostedtoolcache/go /opt/hostedtoolcache/Python/2* /opt/hostedtoolcache/CodeQL /opt/microsoft /etc/skel /home/runner/.rustup/toolchains 
         sudo apt-get install -y lld
         python -m pip install --upgrade pip
         pip install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
@@ -352,7 +357,8 @@ jobs:
 
     steps:
     - name: Free space
-      run: sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku
+      run: |
+        sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku /opt/az /opt/hostedtoolcache/Ruby /opt/hostedtoolcache/node /opt/hostedtoolcache/go /opt/hostedtoolcache/Python/2* /opt/hostedtoolcache/CodeQL /opt/microsoft /etc/skel /home/runner/.rustup/toolchains 
     - uses: actions/checkout@v3
     - name: Set up Python
       uses: actions/setup-python@v4

doc/src/driver/compile.rst

@@ -32,10 +32,6 @@ Disable fast math optimization
 
 Perform an exhaustive search to find the fastest version of generated kernels for selected backend
 
-.. options:: --split-single-dyn-dim
-
-Enable the split single dynamic dimension pass
-
 .. option::  --fp16
 
 Quantize for fp16

doc/src/reference/py.rst

@@ -6,9 +6,9 @@ Python Reference
 shape
 -----
 
-.. py:class:: shape(type, lens, strides=None)
+.. py:class:: shape(type, lens, strides=None, dyn_dims)
 
-    Describes the shape of a tensor. This includes size, layout, and data type/
+    Describes the shape of a tensor. This includes size, layout, and data type. Can be a dynamic shape by using dyn_dims.
 
 .. py:method:: type()
 
@@ -34,6 +34,12 @@ shape
 
     :rtype: int
 
+.. py:method:: dyn_dims()
+
+    The dynamic dimensions of the shape.
+
+    :rtype: list[dynamic_dimension]
+
 .. py:method:: bytes()
 
     The number of bytes the shape uses.
@@ -46,6 +52,12 @@ shape
 
     :rtype: int
 
+.. py:method:: ndim()
+
+    The number of dimensions for the shape.
+
+    :rtype: int
+
 .. py:method:: packed()
 
     Returns true if the shape is packed.
@@ -64,6 +76,12 @@ shape
 
     :rtype: bool
 
+.. py:method:: dynamic()
+
+    Returns true if the shape is dynamic.
+
+    :rtype: bool
+
 .. py:method:: standard()
 
     Returns true if the shape is a standard shape. That is, the shape is both packed and not transposed.
@@ -76,6 +94,18 @@ shape
 
     :rtype: bool
 
+dynamic_dimension
+--------
+
+.. py:class:: dynamic_dimension(min, max, optimals)
+
+    Construct a dynamic_dimension from a minimum, a maximum, and optionally a set of optimals.
+
+.. py:method:: is_fixed()
+    
+    Returns true if the dynamic_dimension is fixed.
+
+    :rtype : int
 
 argument
 --------
@@ -292,8 +322,10 @@ parse_onnx
     Load and parse an onnx file.
 
     :param str filename: Path to file.
-    :param str default_dim_value: default batch size to use (if not specified in onnx file).
+    :param str default_dim_value: default dimension to use (if not specified in onnx file).
+    :param dynamic_dimension default_dyn_dim_value: default dynamic_dimension value to use.
     :param str map_input_dims: Explicitly specify the dims of an input.
+    :param list[dynamic_dimension] map_dyn_input_dims: Explicitly specify the dynamic_dimensions of an input.
     :param str skip_unknown_operators: Continue parsing onnx file if an unknown operator is found.
     :param str print_program_on_error: Print program if an error occurs.
     :param int max_loop_iterations: Maximum iteration number for the loop operator.

examples/migraphx/cpp_dynamic_batch/CMakeLists.txt

+#####################################################################################
+# The MIT License (MIT)
+#
+# Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#####################################################################################
+cmake_minimum_required(VERSION 3.5)
+project (cpp_dynamic_batch)
+
+set (CMAKE_CXX_STANDARD 14)
+set (EXAMPLE dynamic_batch)
+
+list (APPEND CMAKE_PREFIX_PATH /opt/rocm)
+find_package (migraphx)
+
+message("source file: " ${EXAMPLE}.cpp " ---> bin: " ${EXAMPLE})
+add_executable(${EXAMPLE} ${EXAMPLE}.cpp)
+
+target_link_libraries(${EXAMPLE} migraphx::c)
+

examples/migraphx/cpp_dynamic_batch/README.md

+# Running ONNX model with dynamic batch
+
+
+## Description
+This examples demonstrates how to run a graph program with dynamic batch using the MIGraphX C++ API. 
+
+
+## Creating dynamic dimension objects
+`dynamic_dimension` objects are used in MIGraphX to specify a range of dimension values from a minimum value to a maximum value and optimal values that the tensor can be at model evaluation time.
+A dynamic shape is defined by a list of `dynamic_dimensions` while a static shape only has fixed dimension values.
+For example, a `dynamic_dimension` with `{min:1, max:10, optimals:{1, 4, 10}}` means that the dimension can be any value from 1 through 10 with the optimal values being 1, 4, and 10.
+Supplied optimal values may allow MIGraphX to optimize the program for those specific shapes.
+
+A fixed `dynamic_dimension` can be specified by setting the `min` and `max` to the same value (ex. `{min:3, max:3}`).
+A dynamic shape specified solely by fixed `dynamic_dimension` objects will be converted to a static shape during parsing.
+This can be useful for setting a static shape using the `set_dyn_input_parameter_shape()` method discussed later in this document.
+
+
+## Parsing
+ONNX graphs [ONNX](https://onnx.ai/get-started.html) can be parsed by MIGraphX to create a runnable program with dynamic batch sizes.
+The dynamic batch range must be specified by a `dynamic_dimension` object.
+
+One method to set the `dynamic_dimension` object works for ONNX files that only have symbolic variables for the batch dimensions:
+```
+migraphx::program p;
+migraphx::onnx_options options;
+options.set_default_dyn_dim_value(migraphx::dynamic_dimension{1, 4, {2, 4}});
+p = parse_onnx(input_file, options);
+```
+
+Another option that can run any ONNX model with dynamic batch sizes uses the dynamic input map where the entire shape of the input parameter is supplied:
+```
+migraphx::program p;
+migraphx::onnx_options options;
+migraphx::dynamic_dimensions dyn_dims = {migraphx::dynamic_dimension{1, 4, {2, 4}},
+                                         migraphx::dynamic_dimension{3, 3},
+                                         migraphx::dynamic_dimension{4, 4},
+                                         migraphx::dynamic_dimension{4, 4}};
+options.set_dyn_input_parameter_shape("input", dyn_dims);
+p = parse_onnx(input_file, options);
+```
+
+## Compiling
+Currently the MIGraphX C/C++ API requires that `offload_copy` be enabled for compiling dynamic batch programs.
+Here is a snippet of compiling a model with `offload_copy` enabled:
+```
+migraphx::compile_options c_options;
+c_options.set_offload_copy();
+p.compile(migraphx::target("gpu"), c_options);
+```
+where `p` is the `migraphx::program`.
+
+
+## Saving and Loading
+A dynamic batch MIGraphX program can be saved and loaded to/from a MXR file the same way as a fully static shape program.
+
+
+## Executing the dynamic batch model
+The compiled dynamic batch model can be executed the same way as a static model by supplying the input data as `arguments` in a `program_parameters` object.
+
+
+## Running the Example
+Your ROCm installation could be installed in a location other than the one specified in the CMakeLists.txt.
+You can set `LD_LIBRARY_PATH` or `CMAKE_PREFIX_PATH` to that location so that this program can still build.
+
+The provided example is [`dynamic_batch.cpp`](./dynamic_batch.cpp)
+To compile and run the example from this directory:
+```
+$ mkdir build
+$ cd build
+$ cmake ..
+$ make
+```
+There will now be an executable named `dynamic_batch` with the following usage:
+```
+$ ./dynamic_batch
+```

examples/migraphx/cpp_dynamic_batch/dynamic_batch.cpp

+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <string>
+#include <algorithm>
+
+// MIGraphX C++ API
+#include <migraphx/migraphx.hpp>
+
+int main(int argc, char** argv)
+{
+    migraphx::onnx_options o_options;
+    migraphx::dynamic_dimensions dyn_dims = {migraphx::dynamic_dimension{1, 4, {2, 4}},
+                                             migraphx::dynamic_dimension{3, 3},
+                                             migraphx::dynamic_dimension{4, 4},
+                                             migraphx::dynamic_dimension{5, 5}};
+    o_options.set_dyn_input_parameter_shape("0", dyn_dims);
+    auto p = migraphx::parse_onnx("../add_scalar_test.onnx", o_options);
+    migraphx::compile_options c_options;
+    c_options.set_offload_copy();
+    p.compile(migraphx::target("gpu"), c_options);
+
+    // batch size = 2
+    std::vector<uint8_t> a(2 * 3 * 4 * 5, 3);
+    std::vector<uint8_t> b = {2};
+    migraphx::program_parameters pp;
+    migraphx::shape s = migraphx::shape(migraphx_shape_uint8_type, {2, 3, 4, 5});
+    pp.add("0", migraphx::argument(s, a.data()));
+    pp.add("1", migraphx::argument(migraphx::shape(migraphx_shape_uint8_type, {1}, {0}), b.data()));
+    auto outputs = p.eval(pp);
+    auto result  = outputs[0];
+    std::vector<uint8_t> c(2 * 3 * 4 * 5, 5);
+    if(bool{result == migraphx::argument(s, c.data())})
+    {
+        std::cout << "Successfully executed dynamic batch add\n";
+    }
+    else
+    {
+        std::cout << "Failed dynamic batch add\n";
+    }
+
+    return 0;
+}

examples/migraphx/migraphx_driver/README.md

@@ -53,7 +53,6 @@ See below for a comprehensive list of commands and option arguments, as well as
 | --enable-offload-copy                    | Enable implicit offload copying                           |
 | --disable-fast-math                      | Disable fast math optimization                            |
 | --exhaustive-tune                        | Enable exhaustive search to find fastest kernel           |
-| --split-single-dyn-dim                   | Enable split_single_dyn_dim compiler pass                 |
 | --fp16                                   | Quantize for fp16                                         |
 | --int8                                   | Quantize for int8                                         |
 | --tolerance                              | Tolerance for errors                                      |

src/api/api.cpp

@@ -24,6 +24,7 @@
 #include <migraphx/execution_environment.hpp>
 #include <migraphx/migraphx.h>
 #include <migraphx/rank.hpp>
+#include <migraphx/ranges.hpp>
 #include <migraphx/shape.hpp>
 #include <migraphx/program.hpp>
 #include <migraphx/onnx.hpp>
@@ -145,6 +146,11 @@ void set_default_dim_value(onnx_options& options, size_t value)
     options.default_dim_value = value;
 }
 
+void set_default_dyn_dim_value(onnx_options& options, const shape::dynamic_dimension& dd)
+{
+    options.default_dyn_dim_value = dd;
+}
+
 void set_default_loop_iterations(onnx_options& options, int64_t value)
 {
     options.max_loop_iterations = value;
@@ -161,6 +167,13 @@ void set_input_parameter_shape(onnx_options& options,
     options.map_input_dims[std::string(name)] = std::move(dims);
 }
 
+void set_dyn_input_parameter_shape(onnx_options& options,
+                                   const char* name,
+                                   std::vector<shape::dynamic_dimension> dyn_dims)
+{
+    options.map_dyn_input_dims[std::string(name)] = std::move(dyn_dims);
+}
+
 void set_input_parameter_shape(tf_options& options, const char* name, std::vector<std::size_t> dims)
 {
     options.map_input_dims[std::string(name)] = std::move(dims);
@@ -187,6 +200,12 @@ std::vector<const char*> get_names(const std::unordered_map<std::string, Value>&
     return result;
 }
 
+template <class T>
+std::set<T> make_set(const T* x, std::size_t n)
+{
+    return {x, x + n};
+}
+
 void quantize_fp16_with_op_names(program& prog, std::vector<std::string>& names)
 {
     if(names.empty())
@@ -346,7 +365,10 @@ const Target* object_cast(const U* x)
 template <class T, class... Ts, class Target = std::remove_pointer_t<T>>
 Target* allocate(Ts&&... xs)
 {
-    return new Target(std::forward<Ts>(xs)...); // NOLINT
+    if constexpr(std::is_aggregate<Target>{})
+        return new Target{std::forward<Ts>(xs)...}; // NOLINT
+    else
+        return new Target(std::forward<Ts>(xs)...); // NOLINT
 }
 
 template <class T>
@@ -409,6 +431,39 @@ struct manage_generic_ptr
     D deleter                = nullptr;
 };
 
+extern "C" struct migraphx_optimals;
+struct migraphx_optimals
+{
+    template <class... Ts>
+    migraphx_optimals(Ts&&... xs)
+        : object(std::forward<Ts>(xs)...) // NOLINT(readability-redundant-member-init)
+    {
+    }
+    std::set<size_t> object;
+};
+
+extern "C" struct migraphx_dynamic_dimension;
+struct migraphx_dynamic_dimension
+{
+    template <class... Ts>
+    migraphx_dynamic_dimension(Ts&&... xs)
+        : object(std::forward<Ts>(xs)...) // NOLINT(readability-redundant-member-init)
+    {
+    }
+    migraphx::shape::dynamic_dimension object;
+};
+
+extern "C" struct migraphx_dynamic_dimensions;
+struct migraphx_dynamic_dimensions
+{
+    template <class... Ts>
+    migraphx_dynamic_dimensions(Ts&&... xs)
+        : object(std::forward<Ts>(xs)...) // NOLINT(readability-redundant-member-init)
+    {
+    }
+    std::vector<migraphx::shape::dynamic_dimension> object;
+};
+
 extern "C" struct migraphx_shape;
 struct migraphx_shape
 {
@@ -736,6 +791,152 @@ struct migraphx_experimental_custom_op
     }
 };
 
+extern "C" migraphx_status migraphx_optimals_destroy(migraphx_optimals_t optimals)
+{
+    auto api_error_result = migraphx::try_([&] { destroy((optimals)); });
+    return api_error_result;
+}
+
+extern "C" migraphx_status migraphx_optimals_assign_to(migraphx_optimals_t output,
+                                                       const_migraphx_optimals_t input)
+{
+    auto api_error_result = migraphx::try_([&] { *output = *input; });
+    return api_error_result;
+}
+
+extern "C" migraphx_status
+migraphx_optimals_create(migraphx_optimals_t* optimals, const size_t* ptr, size_t size)
+{
+    auto api_error_result = migraphx::try_([&] {
+        *optimals = object_cast<migraphx_optimals_t>(
+            allocate<std::set<size_t>>(migraphx::make_set<size_t>((ptr), (size))));
+    });
+    return api_error_result;
+}
+
+extern "C" migraphx_status
+migraphx_dynamic_dimension_destroy(migraphx_dynamic_dimension_t dynamic_dimension)
+{
+    auto api_error_result = migraphx::try_([&] { destroy((dynamic_dimension)); });
+    return api_error_result;
+}
+
+extern "C" migraphx_status
+migraphx_dynamic_dimension_assign_to(migraphx_dynamic_dimension_t output,
+                                     const_migraphx_dynamic_dimension_t input)
+{
+    auto api_error_result = migraphx::try_([&] { *output = *input; });
+    return api_error_result;
+}
+
+extern "C" migraphx_status migraphx_dynamic_dimension_create_min_max(
+    migraphx_dynamic_dimension_t* dynamic_dimension, size_t min, size_t max)
+{
+    auto api_error_result = migraphx::try_([&] {
+        *dynamic_dimension = object_cast<migraphx_dynamic_dimension_t>(
+            allocate<migraphx::shape::dynamic_dimension>((min), (max)));
+    });
+    return api_error_result;
+}
+
+extern "C" migraphx_status
+migraphx_dynamic_dimension_create_min_max_optimals(migraphx_dynamic_dimension_t* dynamic_dimension,
+                                                   size_t min,
+                                                   size_t max,
+                                                   migraphx_optimals_t optimals)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(optimals == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter optimals: Null pointer");
+        *dynamic_dimension = object_cast<migraphx_dynamic_dimension_t>(
+            allocate<migraphx::shape::dynamic_dimension>((min), (max), (optimals->object)));
+    });
+    return api_error_result;
+}
+
+extern "C" migraphx_status
+migraphx_dynamic_dimension_is_fixed(bool* out, const_migraphx_dynamic_dimension_t dynamic_dimension)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(dynamic_dimension == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param,
+                           "Bad parameter dynamic_dimension: Null pointer");
+        *out = (dynamic_dimension->object).is_fixed();
+    });
+    return api_error_result;
+}
+
+extern "C" migraphx_status
+migraphx_dynamic_dimension_equal(bool* out,
+                                 const_migraphx_dynamic_dimension_t dynamic_dimension,
+                                 const_migraphx_dynamic_dimension_t x)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(dynamic_dimension == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param,
+                           "Bad parameter dynamic_dimension: Null pointer");
+        if(x == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter x: Null pointer");
+        *out = migraphx::equal((dynamic_dimension->object), (x->object));
+    });
+    return api_error_result;
+}
+
+extern "C" migraphx_status
+migraphx_dynamic_dimensions_destroy(migraphx_dynamic_dimensions_t dynamic_dimensions)
+{
+    auto api_error_result = migraphx::try_([&] { destroy((dynamic_dimensions)); });
+    return api_error_result;
+}
+
+extern "C" migraphx_status
+migraphx_dynamic_dimensions_assign_to(migraphx_dynamic_dimensions_t output,
+                                      const_migraphx_dynamic_dimensions_t input)
+{
+    auto api_error_result = migraphx::try_([&] { *output = *input; });
+    return api_error_result;
+}
+
+extern "C" migraphx_status
+migraphx_dynamic_dimensions_create(migraphx_dynamic_dimensions_t* dynamic_dimensions,
+                                   const_migraphx_dynamic_dimension_t* ptr,
+                                   size_t size)
+{
+    auto api_error_result = migraphx::try_([&] {
+        *dynamic_dimensions = object_cast<migraphx_dynamic_dimensions_t>(
+            allocate<std::vector<migraphx::shape::dynamic_dimension>>(
+                migraphx::to_obj_vector<const_migraphx_dynamic_dimension_t>((ptr), (size))));
+    });
+    return api_error_result;
+}
+
+extern "C" migraphx_status
+migraphx_dynamic_dimensions_size(size_t* out, migraphx_dynamic_dimensions_t dynamic_dimensions)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(dynamic_dimensions == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param,
+                           "Bad parameter dynamic_dimensions: Null pointer");
+        *out = (dynamic_dimensions->object).size();
+    });
+    return api_error_result;
+}
+
+extern "C" migraphx_status
+migraphx_dynamic_dimensions_get(const_migraphx_dynamic_dimension_t* out,
+                                migraphx_dynamic_dimensions_t dynamic_dimensions,
+                                size_t idx)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(dynamic_dimensions == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param,
+                           "Bad parameter dynamic_dimensions: Null pointer");
+        *out = object_cast<const_migraphx_dynamic_dimension_t>(
+            &((dynamic_dimensions->object).at((idx))));
+    });
+    return api_error_result;
+}
+
 extern "C" migraphx_status migraphx_shape_destroy(migraphx_shape_t shape)
 {
     auto api_error_result = migraphx::try_([&] { destroy((shape)); });
@@ -794,6 +995,19 @@ extern "C" migraphx_status migraphx_shape_create_scalar(migraphx_shape_t* shape,
     return api_error_result;
 }
 
+extern "C" migraphx_status migraphx_shape_create_dynamic(migraphx_shape_t* shape,
+                                                         migraphx_shape_datatype_t type,
+                                                         migraphx_dynamic_dimensions_t dims)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(dims == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter dims: Null pointer");
+        *shape = object_cast<migraphx_shape_t>(
+            allocate<migraphx::shape>((migraphx::to_shape_type(type)), (dims->object)));
+    });
+    return api_error_result;
+}
+
 extern "C" migraphx_status
 migraphx_shape_lengths(const size_t** out, size_t* out_size, const_migraphx_shape_t shape)
 {
@@ -824,6 +1038,17 @@ migraphx_shape_strides(const size_t** out, size_t* out_size, const_migraphx_shap
     return api_error_result;
 }
 
+extern "C" migraphx_status migraphx_shape_dyn_dims(migraphx_dynamic_dimensions_t* out,
+                                                   const_migraphx_shape_t shape)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(shape == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer");
+        *out = allocate<migraphx_dynamic_dimensions_t>((shape->object).dyn_dims());
+    });
+    return api_error_result;
+}
+
 extern "C" migraphx_status migraphx_shape_type(migraphx_shape_datatype_t* out,
                                                const_migraphx_shape_t shape)
 {
@@ -857,6 +1082,16 @@ extern "C" migraphx_status migraphx_shape_bytes(size_t* out, const_migraphx_shap
     return api_error_result;
 }
 
+extern "C" migraphx_status migraphx_shape_ndim(size_t* out, const_migraphx_shape_t shape)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(shape == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer");
+        *out = (shape->object).ndim();
+    });
+    return api_error_result;
+}
+
 extern "C" migraphx_status
 migraphx_shape_equal(bool* out, const_migraphx_shape_t shape, const_migraphx_shape_t x)
 {
@@ -880,6 +1115,16 @@ extern "C" migraphx_status migraphx_shape_standard(bool* out, const_migraphx_sha
     return api_error_result;
 }
 
+extern "C" migraphx_status migraphx_shape_dynamic(bool* out, const_migraphx_shape_t shape)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(shape == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer");
+        *out = (shape->object).dynamic();
+    });
+    return api_error_result;
+}
+
 extern "C" migraphx_status migraphx_shape_index(size_t* out, const_migraphx_shape_t shape, size_t i)
 {
     auto api_error_result = migraphx::try_([&] {
@@ -915,6 +1160,17 @@ migraphx_argument_create(migraphx_argument_t* argument, const_migraphx_shape_t s
     return api_error_result;
 }
 
+extern "C" migraphx_status migraphx_argument_create_empty(migraphx_argument_t* argument,
+                                                          const_migraphx_shape_t shape)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(shape == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer");
+        *argument = object_cast<migraphx_argument_t>(allocate<migraphx::argument>((shape->object)));
+    });
+    return api_error_result;
+}
+
 extern "C" migraphx_status migraphx_argument_shape(const_migraphx_shape_t* out,
                                                    const_migraphx_argument_t argument)
 {
@@ -1590,6 +1846,19 @@ extern "C" migraphx_status migraphx_onnx_options_set_input_parameter_shape(
     return api_error_result;
 }
 
+extern "C" migraphx_status migraphx_onnx_options_set_dyn_input_parameter_shape(
+    migraphx_onnx_options_t onnx_options, const char* name, migraphx_dynamic_dimensions_t dims)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(onnx_options == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter onnx_options: Null pointer");
+        if(dims == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter dims: Null pointer");
+        migraphx::set_dyn_input_parameter_shape((onnx_options->object), (name), (dims->object));
+    });
+    return api_error_result;
+}
+
 extern "C" migraphx_status
 migraphx_onnx_options_set_default_dim_value(migraphx_onnx_options_t onnx_options, size_t value)
 {
@@ -1601,6 +1870,20 @@ migraphx_onnx_options_set_default_dim_value(migraphx_onnx_options_t onnx_options
     return api_error_result;
 }
 
+extern "C" migraphx_status
+migraphx_onnx_options_set_default_dyn_dim_value(migraphx_onnx_options_t onnx_options,
+                                                const_migraphx_dynamic_dimension_t dd)
+{
+    auto api_error_result = migraphx::try_([&] {
+        if(onnx_options == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter onnx_options: Null pointer");
+        if(dd == nullptr)
+            MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter dd: Null pointer");
+        migraphx::set_default_dyn_dim_value((onnx_options->object), (dd->object));
+    });
+    return api_error_result;
+}
+
 extern "C" migraphx_status
 migraphx_onnx_options_set_default_loop_iterations(migraphx_onnx_options_t onnx_options,
                                                   int64_t value)

src/api/include/migraphx/migraphx.h

@@ -66,6 +66,15 @@ typedef enum
 } migraphx_shape_datatype_t;
 #undef MIGRAPHX_SHAPE_GENERATE_ENUM_TYPES
 
+typedef struct migraphx_optimals* migraphx_optimals_t;
+typedef const struct migraphx_optimals* const_migraphx_optimals_t;
+
+typedef struct migraphx_dynamic_dimension* migraphx_dynamic_dimension_t;
+typedef const struct migraphx_dynamic_dimension* const_migraphx_dynamic_dimension_t;
+
+typedef struct migraphx_dynamic_dimensions* migraphx_dynamic_dimensions_t;
+typedef const struct migraphx_dynamic_dimensions* const_migraphx_dynamic_dimensions_t;
+
 typedef struct migraphx_shape* migraphx_shape_t;
 typedef const struct migraphx_shape* const_migraphx_shape_t;
 
@@ -157,6 +166,55 @@ typedef migraphx_status (*migraphx_experimental_custom_op_copy)(void** out, void
 
 typedef migraphx_status (*migraphx_experimental_custom_op_delete)(void* input);
 
+migraphx_status migraphx_optimals_destroy(migraphx_optimals_t optimals);
+
+migraphx_status migraphx_optimals_assign_to(migraphx_optimals_t output,
+                                            const_migraphx_optimals_t input);
+
+migraphx_status
+migraphx_optimals_create(migraphx_optimals_t* optimals, const size_t* ptr, size_t size);
+
+migraphx_status migraphx_dynamic_dimension_destroy(migraphx_dynamic_dimension_t dynamic_dimension);
+
+migraphx_status migraphx_dynamic_dimension_assign_to(migraphx_dynamic_dimension_t output,
+                                                     const_migraphx_dynamic_dimension_t input);
+
+migraphx_status migraphx_dynamic_dimension_create_min_max(
+    migraphx_dynamic_dimension_t* dynamic_dimension, size_t min, size_t max);
+
+migraphx_status
+migraphx_dynamic_dimension_create_min_max_optimals(migraphx_dynamic_dimension_t* dynamic_dimension,
+                                                   size_t min,
+                                                   size_t max,
+                                                   migraphx_optimals_t optimals);
+
+migraphx_status
+migraphx_dynamic_dimension_is_fixed(bool* out,
+                                    const_migraphx_dynamic_dimension_t dynamic_dimension);
+
+migraphx_status
+migraphx_dynamic_dimension_equal(bool* out,
+                                 const_migraphx_dynamic_dimension_t dynamic_dimension,
+                                 const_migraphx_dynamic_dimension_t x);
+
+migraphx_status
+migraphx_dynamic_dimensions_destroy(migraphx_dynamic_dimensions_t dynamic_dimensions);
+
+migraphx_status migraphx_dynamic_dimensions_assign_to(migraphx_dynamic_dimensions_t output,
+                                                      const_migraphx_dynamic_dimensions_t input);
+
+migraphx_status
+migraphx_dynamic_dimensions_create(migraphx_dynamic_dimensions_t* dynamic_dimensions,
+                                   const_migraphx_dynamic_dimension_t* ptr,
+                                   size_t size);
+
+migraphx_status migraphx_dynamic_dimensions_size(size_t* out,
+                                                 migraphx_dynamic_dimensions_t dynamic_dimensions);
+
+migraphx_status migraphx_dynamic_dimensions_get(const_migraphx_dynamic_dimension_t* out,
+                                                migraphx_dynamic_dimensions_t dynamic_dimensions,
+                                                size_t idx);
+
 migraphx_status migraphx_shape_destroy(migraphx_shape_t shape);
 
 migraphx_status migraphx_shape_assign_to(migraphx_shape_t output, const_migraphx_shape_t input);
@@ -176,23 +234,34 @@ migraphx_status migraphx_shape_create_with_strides(migraphx_shape_t* shape,
 migraphx_status migraphx_shape_create_scalar(migraphx_shape_t* shape,
                                              migraphx_shape_datatype_t type);
 
+migraphx_status migraphx_shape_create_dynamic(migraphx_shape_t* shape,
+                                              migraphx_shape_datatype_t type,
+                                              migraphx_dynamic_dimensions_t dims);
+
 migraphx_status
 migraphx_shape_lengths(const size_t** out, size_t* out_size, const_migraphx_shape_t shape);
 
 migraphx_status
 migraphx_shape_strides(const size_t** out, size_t* out_size, const_migraphx_shape_t shape);
 
+migraphx_status migraphx_shape_dyn_dims(migraphx_dynamic_dimensions_t* out,
+                                        const_migraphx_shape_t shape);
+
 migraphx_status migraphx_shape_type(migraphx_shape_datatype_t* out, const_migraphx_shape_t shape);
 
 migraphx_status migraphx_shape_elements(size_t* out, const_migraphx_shape_t shape);
 
 migraphx_status migraphx_shape_bytes(size_t* out, const_migraphx_shape_t shape);
 
+migraphx_status migraphx_shape_ndim(size_t* out, const_migraphx_shape_t shape);
+
 migraphx_status
 migraphx_shape_equal(bool* out, const_migraphx_shape_t shape, const_migraphx_shape_t x);
 
 migraphx_status migraphx_shape_standard(bool* out, const_migraphx_shape_t shape);
 
+migraphx_status migraphx_shape_dynamic(bool* out, const_migraphx_shape_t shape);
+
 migraphx_status migraphx_shape_index(size_t* out, const_migraphx_shape_t shape, size_t i);
 
 migraphx_status migraphx_argument_destroy(migraphx_argument_t argument);
@@ -203,6 +272,9 @@ migraphx_status migraphx_argument_assign_to(migraphx_argument_t output,
 migraphx_status
 migraphx_argument_create(migraphx_argument_t* argument, const_migraphx_shape_t shape, void* buffer);
 
+migraphx_status migraphx_argument_create_empty(migraphx_argument_t* argument,
+                                               const_migraphx_shape_t shape);
+
 migraphx_status migraphx_argument_shape(const_migraphx_shape_t* out,
                                         const_migraphx_argument_t argument);
 
@@ -397,9 +469,16 @@ migraphx_status migraphx_onnx_options_create(migraphx_onnx_options_t* onnx_optio
 migraphx_status migraphx_onnx_options_set_input_parameter_shape(
     migraphx_onnx_options_t onnx_options, const char* name, size_t* dims, size_t dims_size);
 
+migraphx_status migraphx_onnx_options_set_dyn_input_parameter_shape(
+    migraphx_onnx_options_t onnx_options, const char* name, migraphx_dynamic_dimensions_t dims);
+
 migraphx_status migraphx_onnx_options_set_default_dim_value(migraphx_onnx_options_t onnx_options,
                                                             size_t value);
 
+migraphx_status
+migraphx_onnx_options_set_default_dyn_dim_value(migraphx_onnx_options_t onnx_options,
+                                                const_migraphx_dynamic_dimension_t dd);
+
 migraphx_status
 migraphx_onnx_options_set_default_loop_iterations(migraphx_onnx_options_t onnx_options,
                                                   int64_t value);

src/api/include/migraphx/migraphx.hpp

@@ -571,10 +571,90 @@ using require_interface =
 // NOLINTNEXTLINE
 #define MIGRAPHX_CONST_HANDLE_BASE(name) MIGRAPHX_DETAIL_HANDLE_BASE(name, const)
 
+/**
+ * Container to hold optimal dynamic dimension values.
+ */
+struct optimals : MIGRAPHX_HANDLE_BASE(optimals)
+{
+    MIGRAPHX_HANDLE_CONSTRUCTOR(optimals)
+
+    optimals(std::initializer_list<size_t> init_list)
+    {
+        this->make_handle(&migraphx_optimals_create, init_list.begin(), init_list.size());
+    }
+};
+
+/**
+ * @brief Dynamic dimension object.
+ * @details minimum, maximum, and optimal dimensions
+ */
+struct dynamic_dimension : MIGRAPHX_CONST_HANDLE_BASE(dynamic_dimension)
+{
+    MIGRAPHX_HANDLE_CONSTRUCTOR(dynamic_dimension)
+
+    dynamic_dimension(size_t min, size_t max)
+    {
+        this->make_handle(&migraphx_dynamic_dimension_create_min_max, min, max);
+    }
+
+    dynamic_dimension(size_t min, size_t max, const optimals& opts)
+    {
+        this->make_handle(
+            &migraphx_dynamic_dimension_create_min_max_optimals, min, max, opts.get_handle_ptr());
+    }
+
+    bool is_fixed() const
+    {
+        bool result = false;
+        call(&migraphx_dynamic_dimension_is_fixed, &result, this->get_handle_ptr());
+        return result;
+    }
+
+    friend bool operator==(const dynamic_dimension& x, const dynamic_dimension& y)
+    {
+        bool pout;
+        call(&migraphx_dynamic_dimension_equal, &pout, x.get_handle_ptr(), y.get_handle_ptr());
+        return pout;
+    }
+
+    friend bool operator!=(const dynamic_dimension& x, const dynamic_dimension& y)
+    {
+        return not(x == y);
+    }
+};
+
+/**
+ * Container to hold dynamic_dimension objects.
+ */
+struct dynamic_dimensions : MIGRAPHX_HANDLE_BASE(dynamic_dimensions)
+{
+    MIGRAPHX_HANDLE_CONSTRUCTOR(dynamic_dimensions)
+
+    template <class... Ts>
+    dynamic_dimensions(Ts... xs)
+    {
+        std::array<const_migraphx_dynamic_dimension_t, sizeof...(Ts)> a{xs.get_handle_ptr()...};
+        this->make_handle(&migraphx_dynamic_dimensions_create, a.data(), a.size());
+    }
+
+    size_t size() const
+    {
+        size_t pout;
+        call(&migraphx_dynamic_dimensions_size, &pout, this->get_handle_ptr());
+        return pout;
+    }
+
+    dynamic_dimension operator[](size_t pidx) const
+    {
+        const_migraphx_dynamic_dimension_t pout;
+        call(&migraphx_dynamic_dimensions_get, &pout, this->get_handle_ptr(), pidx);
+        return {pout, this->share_handle()};
+    }
+};
+
 /**
  * @brief Describe shape of tensor
  * @details A shape consists of a data type, lengths of multi-dimension tensor, and strides
- *
  */
 struct shape : MIGRAPHX_CONST_HANDLE_BASE(shape)
 {
@@ -598,6 +678,13 @@ struct shape : MIGRAPHX_CONST_HANDLE_BASE(shape)
         this->make_handle(&migraphx_shape_create, type, plengths.data(), plengths.size());
     }
 
+    // Force all calls of the format `shape( type_t, { size_t compatibles } )` to map to
+    // shape(type_t, std::vector<std::size_t> l)
+    shape(migraphx_shape_datatype_t t, std::initializer_list<std::size_t> d)
+        : shape::shape(t, std::vector<std::size_t>{d.begin(), d.end()})
+    {
+    }
+
     shape(migraphx_shape_datatype_t type,
           std::vector<size_t> plengths,
           std::vector<size_t> pstrides)
@@ -610,6 +697,11 @@ struct shape : MIGRAPHX_CONST_HANDLE_BASE(shape)
                           pstrides.size());
     }
 
+    shape(migraphx_shape_datatype_t type, const dynamic_dimensions& dyn_dims)
+    {
+        this->make_handle(&migraphx_shape_create_dynamic, type, dyn_dims.get_handle_ptr());
+    }
+
     std::vector<size_t> lengths() const
     {
         const size_t* pout;
@@ -626,6 +718,14 @@ struct shape : MIGRAPHX_CONST_HANDLE_BASE(shape)
         return {pout, pout + pout_size};
     }
 
+    /// Get the dynamic dimensions of the shape
+    dynamic_dimensions dyn_dims() const
+    {
+        migraphx_dynamic_dimensions_t pout;
+        call(&migraphx_shape_dyn_dims, &pout, this->get_handle_ptr());
+        return {pout, own{}};
+    }
+
     migraphx_shape_datatype_t type() const
     {
         migraphx_shape_datatype_t pout;
@@ -654,6 +754,14 @@ struct shape : MIGRAPHX_CONST_HANDLE_BASE(shape)
         return result;
     }
 
+    /// Is the shape dynamic
+    bool dynamic() const
+    {
+        bool result = false;
+        call(&migraphx_shape_dynamic, &result, this->get_handle_ptr());
+        return result;
+    }
+
     // map element index to space index
     size_t index(size_t i) const
     {
@@ -687,6 +795,11 @@ struct argument : MIGRAPHX_CONST_HANDLE_BASE(argument)
     MIGRAPHX_DEPRECATED("Contructor without lifetime annotation is deprecated.")
     argument(const migraphx_argument* p) { this->set_handle(p, borrow{}); }
 
+    argument(shape pshape)
+    {
+        this->make_handle(&migraphx_argument_create_empty, pshape.get_handle_ptr());
+    }
+
     argument(shape pshape, void* pbuffer)
     {
         this->make_handle(&migraphx_argument_create, pshape.get_handle_ptr(), pbuffer);
@@ -1182,12 +1295,27 @@ struct onnx_options : MIGRAPHX_HANDLE_BASE(onnx_options)
              dim.size());
     }
 
+    void set_dyn_input_parameter_shape(const std::string& name, const dynamic_dimensions& dyn_dims)
+    {
+        call(&migraphx_onnx_options_set_dyn_input_parameter_shape,
+             this->get_handle_ptr(),
+             name.c_str(),
+             dyn_dims.get_handle_ptr());
+    }
+
     /// When there is a dimension parameter, then use this default value
     void set_default_dim_value(unsigned int value)
     {
         call(&migraphx_onnx_options_set_default_dim_value, this->get_handle_ptr(), value);
     }
 
+    void set_default_dyn_dim_value(const dynamic_dimension& dd)
+    {
+        call(&migraphx_onnx_options_set_default_dyn_dim_value,
+             this->get_handle_ptr(),
+             dd.get_handle_ptr());
+    }
+
     /// Set default max iteration number for the loop operator
     void set_default_loop_iterations(int64_t value)
     {

src/api/migraphx.py

@@ -45,56 +45,48 @@ def shape_type_wrap(p):
         p.read = 'migraphx::to_shape_type(${name})'
 
 
-@api.cwrap('migraphx::compile_options')
-def compile_options_type_wrap(p):
-    if p.returns:
-        p.add_param('migraphx_compile_options *')
-        p.bad_param('${name} == nullptr', 'Null pointer')
-        p.write = ['*${name} = migraphx::to_compile_options(${result})']
-    else:
-        p.add_param('migraphx_compile_options *')
-        p.read = '${name} == nullptr ? migraphx::compile_options{} : migraphx::to_compile_options(*${name})'
-
-
-@api.cwrap('migraphx::file_options')
-def file_options_type_wrap(p):
-    if p.returns:
-        p.add_param('migraphx_file_options *')
-        p.bad_param('${name} == nullptr', 'Null pointer')
-        p.write = ['*${name} = migraphx::to_file_options(${result})']
-    else:
-        p.add_param('migraphx_file_options *')
-        p.read = '${name} == nullptr ? migraphx::file_options{} : migraphx::to_file_options(*${name})'
+def auto_handle(*args, **kwargs):
+    def with_handle(f):
+        return api.handle('migraphx_' + f.__name__, 'migraphx::' + f.__name__,
+                          *args, **kwargs)(f)
 
+    return with_handle
 
-@api.cwrap('migraphx::onnx_options')
-def onnx_options_type_wrap(p):
-    if p.returns:
-        p.add_param('migraphx_onnx_options *')
-        p.bad_param('${name} == nullptr', 'Null pointer')
-        p.write = ['*${name} = migraphx::to_onnx_options(${result})']
-    else:
-        p.add_param('migraphx_onnx_options *')
-        p.read = '${name} == nullptr ? migraphx::onnx_options{} : migraphx::to_onnx_options(*${name})'
 
+@api.handle('migraphx_optimals', 'std::set<size_t>')
+def optimals(h):
+    h.constructor('create',
+                  api.params(ptr='const size_t*', size='size_t'),
+                  fname='migraphx::make_set<size_t>')
 
-@api.cwrap('migraphx::tf_options')
-def tf_options_type_wrap(p):
-    if p.returns:
-        p.add_param('migraphx_tf_options *')
-        p.bad_param('${name} == nullptr', 'Null pointer')
-        p.write = ['*${name} = migraphx::to_tf_options(${result})']
-    else:
-        p.add_param('migraphx_tf_options *')
-        p.read = '${name} == nullptr ? migraphx::tf_options{} : migraphx::to_tf_options(*${name})'
 
+@api.handle('migraphx_dynamic_dimension', 'migraphx::shape::dynamic_dimension')
+def dynamic_dimension(h):
+    h.constructor('create_min_max', api.params(min='size_t', max='size_t'))
+    h.constructor(
+        'create_min_max_optimals',
+        api.params(min='size_t', max='size_t', optimals='std::set<size_t>'))
+    h.method('is_fixed', returns='bool', const=True)
+    h.method('equal',
+             api.params(x='const migraphx::shape::dynamic_dimension&'),
+             invoke='migraphx::equal($@)',
+             returns='bool',
+             const=True)
 
-def auto_handle(*args, **kwargs):
-    def with_handle(f):
-        return api.handle('migraphx_' + f.__name__, 'migraphx::' + f.__name__,
-                          *args, **kwargs)(f)
 
-    return with_handle
+@api.handle('migraphx_dynamic_dimensions',
+            'std::vector<migraphx::shape::dynamic_dimension>')
+def dynamic_dimensions(h):
+    h.constructor(
+        'create',
+        api.params(ptr='const_migraphx_dynamic_dimension_t*', size='size_t'),
+        fname='migraphx::to_obj_vector<const_migraphx_dynamic_dimension_t>')
+    h.method('size', returns='size_t')
+    h.method('get',
+             api.params(idx='size_t'),
+             fname='at',
+             cpp_name='operator[]',
+             returns='const migraphx::shape::dynamic_dimension&')
 
 
 @auto_handle()
@@ -109,20 +101,29 @@ def shape(h):
                    lengths='std::vector<size_t>',
                    strides='std::vector<size_t>'))
     h.constructor('create_scalar', api.params(type='migraphx::shape::type_t'))
+    h.constructor(
+        'create_dynamic',
+        api.params(type='migraphx::shape::type_t',
+                   dims='std::vector<migraphx::shape::dynamic_dimension>'))
     h.method('lengths',
              fname='lens',
              returns='const std::vector<size_t>&',
              const=True)
     h.method('strides', returns='const std::vector<size_t>&', const=True)
+    h.method('dyn_dims',
+             returns='std::vector<migraphx::shape::dynamic_dimension>',
+             const=True)
     h.method('type', returns='migraphx::shape::type_t', const=True)
     h.method('elements', returns='size_t', const=True)
     h.method('bytes', returns='size_t', const=True)
+    h.method('ndim', returns='size_t', const=True)
     h.method('equal',
              api.params(x='const migraphx::shape&'),
              invoke='migraphx::equal($@)',
              returns='bool',
              const=True)
     h.method('standard', returns='bool', const=True)
+    h.method('dynamic', returns='bool', const=True)
     h.method('index', api.params(i='size_t'), returns='size_t', const=True)
 
 
@@ -130,6 +131,7 @@ def shape(h):
 def argument(h):
     h.constructor('create',
                   api.params(shape='const migraphx::shape&', buffer='void*'))
+    h.constructor('create_empty', api.params(shape='const migraphx::shape&'))
     h.method('shape',
              fname='get_shape',
              cpp_name='get_shape',
@@ -325,11 +327,22 @@ def onnx_options(h):
         api.params(name='const char*', dims='std::vector<size_t>'),
         invoke='migraphx::set_input_parameter_shape($@)',
     )
+    h.method(
+        'set_dyn_input_parameter_shape',
+        api.params(name='const char*',
+                   dims='std::vector<migraphx::shape::dynamic_dimension>'),
+        invoke='migraphx::set_dyn_input_parameter_shape($@)',
+    )
     h.method(
         'set_default_dim_value',
         api.params(value='size_t'),
         invoke='migraphx::set_default_dim_value($@)',
     )
+    h.method(
+        'set_default_dyn_dim_value',
+        api.params(dd='const migraphx::shape::dynamic_dimension&'),
+        invoke='migraphx::set_default_dyn_dim_value($@)',
+    )
     h.method(
         'set_default_loop_iterations',
         api.params(value='int64_t'),

src/driver/main.cpp

@@ -436,11 +436,6 @@ struct compiler
            {"--exhaustive-tune"},
            ap.help("Exhastively search for best tuning parameters for kernels"),
            ap.set_value(true));
-        ap(co.split_single_dyn_dim,
-           {"--split-single-dyn-dim"},
-           ap.help("If there is a single non-fixed dynamic dimension in the model, then split to "
-                   "static submodules"),
-           ap.set_value(true));
         ap(quantize, {"--fp16"}, ap.help("Quantize for fp16"), ap.set_value(precision::fp16));
         ap(quantize, {"--int8"}, ap.help("Quantize for int8"), ap.set_value(precision::int8));
     }

src/include/migraphx/compile_options.hpp

@@ -40,9 +40,6 @@ struct compile_options
 
     bool fast_math       = true;
     bool exhaustive_tune = false;
-
-    /// Use the split_single_dyn_dim pass
-    bool split_single_dyn_dim = false;
     tracer trace{};
 };
 

src/py/migraphx_py.cpp

@@ -95,6 +95,10 @@ void visit_py(T x, F f)
     {
         f(x.template cast<std::string>());
     }
+    else if(py::isinstance<migraphx::shape::dynamic_dimension>(x))
+    {
+        f(migraphx::to_value(x.template cast<migraphx::shape::dynamic_dimension>()));
+    }
     else
     {
         MIGRAPHX_THROW("VISIT_PY: Unsupported data type!");
@@ -165,7 +169,10 @@ template <class T>
 py::buffer_info to_buffer_info(T& x)
 {
     migraphx::shape s = x.get_shape();
-    auto strides      = s.strides();
+    assert(s.type() != migraphx::shape::tuple_type);
+    if(s.dynamic())
+        MIGRAPHX_THROW("MIGRAPHX PYTHON: dynamic shape argument passed to to_buffer_info");
+    auto strides = s.strides();
     std::transform(
         strides.begin(), strides.end(), strides.begin(), [&](auto i) { return i * s.type_size(); });
     py::buffer_info b;
@@ -177,7 +184,7 @@ py::buffer_info to_buffer_info(T& x)
             b = py::buffer_info(x.data(),
                                 as.size(),
                                 py::format_descriptor<bool>::format(),
-                                s.lens().size(),
+                                s.ndim(),
                                 s.lens(),
                                 strides);
         }
@@ -186,7 +193,7 @@ py::buffer_info to_buffer_info(T& x)
             b = py::buffer_info(x.data(),
                                 as.size(),
                                 py::format_descriptor<decltype(as())>::format(),
-                                s.lens().size(),
+                                s.ndim(),
                                 s.lens(),
                                 strides);
         }
@@ -239,8 +246,15 @@ MIGRAPHX_PYBIND11_MODULE(migraphx, m)
     py::class_<migraphx::shape> shape_cls(m, "shape");
     shape_cls
         .def(py::init([](py::kwargs kwargs) {
-            auto v    = migraphx::to_value(kwargs);
-            auto t    = migraphx::shape::parse_type(v.get("type", "float"));
+            auto v = migraphx::to_value(kwargs);
+            auto t = migraphx::shape::parse_type(v.get("type", "float"));
+            if(v.contains("dyn_dims"))
+            {
+                auto dyn_dims =
+                    migraphx::from_value<std::vector<migraphx::shape::dynamic_dimension>>(
+                        v.at("dyn_dims"));
+                return migraphx::shape(t, dyn_dims);
+            }
             auto lens = v.get<std::size_t>("lens", {1});
             if(v.contains("strides"))
                 return migraphx::shape(t, lens, v.at("strides").to_vector<std::size_t>());
@@ -250,15 +264,18 @@ MIGRAPHX_PYBIND11_MODULE(migraphx, m)
         .def("type", &migraphx::shape::type)
         .def("lens", &migraphx::shape::lens)
         .def("strides", &migraphx::shape::strides)
+        .def("ndim", &migraphx::shape::ndim)
         .def("elements", &migraphx::shape::elements)
         .def("bytes", &migraphx::shape::bytes)
         .def("type_string", &migraphx::shape::type_string)
         .def("type_size", &migraphx::shape::type_size)
+        .def("dyn_dims", &migraphx::shape::dyn_dims)
         .def("packed", &migraphx::shape::packed)
         .def("transposed", &migraphx::shape::transposed)
         .def("broadcasted", &migraphx::shape::broadcasted)
         .def("standard", &migraphx::shape::standard)
         .def("scalar", &migraphx::shape::scalar)
+        .def("dynamic", &migraphx::shape::dynamic)
         .def("__eq__", std::equal_to<migraphx::shape>{})
         .def("__ne__", std::not_equal_to<migraphx::shape>{})
         .def("__repr__", [](const migraphx::shape& s) { return migraphx::to_string(s); });
@@ -266,6 +283,15 @@ MIGRAPHX_PYBIND11_MODULE(migraphx, m)
     py::enum_<migraphx::shape::type_t>(shape_cls, "type_t")
         MIGRAPHX_SHAPE_VISIT_TYPES(MIGRAPHX_PYTHON_GENERATE_SHAPE_ENUM);
 
+    py::class_<migraphx::shape::dynamic_dimension>(shape_cls, "dynamic_dimension")
+        .def(py::init<>())
+        .def(py::init<std::size_t, std::size_t>())
+        .def(py::init<std::size_t, std::size_t, std::set<std::size_t>>())
+        .def_readwrite("min", &migraphx::shape::dynamic_dimension::min)
+        .def_readwrite("max", &migraphx::shape::dynamic_dimension::max)
+        .def_readwrite("optimals", &migraphx::shape::dynamic_dimension::optimals)
+        .def("is_fixed", &migraphx::shape::dynamic_dimension::is_fixed);
+
     py::class_<migraphx::argument>(m, "argument", py::buffer_protocol())
         .def_buffer([](migraphx::argument& x) -> py::buffer_info { return to_buffer_info(x); })
         .def(py::init([](py::buffer b) {
@@ -440,13 +466,18 @@ MIGRAPHX_PYBIND11_MODULE(migraphx, m)
         "parse_onnx",
         [](const std::string& filename,
            unsigned int default_dim_value,
+           migraphx::shape::dynamic_dimension default_dyn_dim_value,
            std::unordered_map<std::string, std::vector<std::size_t>> map_input_dims,
+           std::unordered_map<std::string, std::vector<migraphx::shape::dynamic_dimension>>
+               map_dyn_input_dims,
            bool skip_unknown_operators,
            bool print_program_on_error,
            int64_t max_loop_iterations) {
             migraphx::onnx_options options;
             options.default_dim_value      = default_dim_value;
+            options.default_dyn_dim_value  = default_dyn_dim_value;
             options.map_input_dims         = map_input_dims;
+            options.map_dyn_input_dims     = map_dyn_input_dims;
             options.skip_unknown_operators = skip_unknown_operators;
             options.print_program_on_error = print_program_on_error;
             options.max_loop_iterations    = max_loop_iterations;
@@ -454,8 +485,11 @@ MIGRAPHX_PYBIND11_MODULE(migraphx, m)
         },
         "Parse onnx file",
         py::arg("filename"),
-        py::arg("default_dim_value") = 1,
-        py::arg("map_input_dims")    = std::unordered_map<std::string, std::vector<std::size_t>>(),
+        py::arg("default_dim_value")     = 0,
+        py::arg("default_dyn_dim_value") = migraphx::shape::dynamic_dimension{1, 1},
+        py::arg("map_input_dims") = std::unordered_map<std::string, std::vector<std::size_t>>(),
+        py::arg("map_dyn_input_dims") =
+            std::unordered_map<std::string, std::vector<migraphx::shape::dynamic_dimension>>(),
         py::arg("skip_unknown_operators") = false,
         py::arg("print_program_on_error") = false,
         py::arg("max_loop_iterations")    = 10);
@@ -464,20 +498,28 @@ MIGRAPHX_PYBIND11_MODULE(migraphx, m)
         "parse_onnx_buffer",
         [](const std::string& onnx_buffer,
            unsigned int default_dim_value,
+           migraphx::shape::dynamic_dimension default_dyn_dim_value,
            std::unordered_map<std::string, std::vector<std::size_t>> map_input_dims,
+           std::unordered_map<std::string, std::vector<migraphx::shape::dynamic_dimension>>
+               map_dyn_input_dims,
            bool skip_unknown_operators,
            bool print_program_on_error) {
             migraphx::onnx_options options;
             options.default_dim_value      = default_dim_value;
+            options.default_dyn_dim_value  = default_dyn_dim_value;
             options.map_input_dims         = map_input_dims;
+            options.map_dyn_input_dims     = map_dyn_input_dims;
             options.skip_unknown_operators = skip_unknown_operators;
             options.print_program_on_error = print_program_on_error;
             return migraphx::parse_onnx_buffer(onnx_buffer, options);
         },
         "Parse onnx file",
         py::arg("filename"),
-        py::arg("default_dim_value") = 1,
-        py::arg("map_input_dims")    = std::unordered_map<std::string, std::vector<std::size_t>>(),
+        py::arg("default_dim_value")     = 0,
+        py::arg("default_dyn_dim_value") = migraphx::shape::dynamic_dimension{1, 1},
+        py::arg("map_input_dims") = std::unordered_map<std::string, std::vector<std::size_t>>(),
+        py::arg("map_dyn_input_dims") =
+            std::unordered_map<std::string, std::vector<migraphx::shape::dynamic_dimension>>(),
         py::arg("skip_unknown_operators") = false,
         py::arg("print_program_on_error") = false);
 

src/shape.cpp

@@ -706,14 +706,10 @@ void migraphx_from_value(const value& v, shape& s)
         {
             auto v_dd = v.at("dynamic_dimensions");
             std::vector<shape::dynamic_dimension> dyn_dims(v.at("dynamic_dimensions").size());
-            std::transform(v_dd.begin(), v_dd.end(), dyn_dims.begin(), [](migraphx::value x) {
-                auto x_min      = x.at("min").template to<size_t>();
-                auto x_max      = x.at("max").template to<size_t>();
-                auto v_optimals = x.at("optimals");
-                std::set<size_t> set_x_optimals =
-                    from_value<std::set<std::size_t>>(x.at("optimals"));
-                return shape::dynamic_dimension{x_min, x_max, set_x_optimals};
-            });
+            std::transform(
+                v_dd.begin(), v_dd.end(), dyn_dims.begin(), [](const migraphx::value& x) {
+                    return from_value<shape::dynamic_dimension>(x);
+                });
 
             s = shape{shape::parse_type(t), dyn_dims};
         }

src/split_single_dyn_dim.cpp

@@ -100,10 +100,10 @@ struct find_static_2in_broadcasts
 } // namespace
 
 /**
- * Makes all the shapes in the dynamic_dimension range.
- * Probably won't work for `if` and `loop` instructions, depending on how the submodules for those
+ * Makes all the shapes in the dynamic_dimension range.  Probably won't work for `if`
+ * and `loop` instructions, depending on how the submodules for those
  * work. Inserts select_module instruction to the top. Replaces return, bypassing other
- * instructions.
+ * instructions. Skips if the dynamic parameter outputs to a select_module operator.
  */
 void split_single_dyn_dim::apply(module_pass_manager& mpm) const
 {
@@ -111,7 +111,13 @@ void split_single_dyn_dim::apply(module_pass_manager& mpm) const
     auto param_names                            = mm->get_parameter_names();
     auto param_shapes                           = mm->get_parameter_shapes();
     optional<dynamic_dimensions_check> dd_check = has_one_dyn_dim(param_shapes);
-    if(dd_check.has_value())
+    auto any_sm_next                            = [&](auto ddc) {
+        auto p_outputs = mm->get_parameter(ddc->dyn_param_str)->outputs();
+        return std::any_of(p_outputs.cbegin(), p_outputs.cend(), [](auto ins) {
+            return ins->name() == "select_module";
+        });
+    };
+    if(dd_check.has_value() and not any_sm_next(dd_check))
     {
         const auto& dyn_param = mm->get_parameter(dd_check->dyn_param_str);
         auto dyn_param_shape  = mm->get_parameter_shape(dd_check->dyn_param_str);

src/targets/gpu/target.cpp

@@ -101,8 +101,8 @@ std::vector<pass> target::get_passes(migraphx::context& gctx, const compile_opti
     // clang-format off
     return
     {
-        enable_pass(options.split_single_dyn_dim, split_single_dyn_dim{}),
-        enable_pass(options.split_single_dyn_dim, dead_code_elimination{}),
+        split_single_dyn_dim{},
+        dead_code_elimination{},
         normalize_ops{},
         dead_code_elimination{},
         simplify_qdq{},

test/api/CMakeLists.txt

@@ -48,6 +48,7 @@ add_api_test(assign test_assign.cpp ${TEST_ONNX_DIR})
 add_api_test(compile_options test_compile_options.cpp ${TEST_ONNX_DIR})
 add_api_test(lookup test_lookup.cpp ${TEST_ONNX_DIR})
 add_api_test(module_construct test_module_construct.cpp ${TEST_ONNX_DIR})
+add_api_test(dynamic_shape test_dynamic_shape.cpp ${TEST_ONNX_DIR})
 add_api_test(ref test_cpu.cpp ${TEST_ONNX_DIR})
 add_api_test(save_load test_save_load.cpp ${TEST_ONNX_DIR})
 add_api_test(op test_op_construct.cpp ${TEST_ONNX_DIR})

test/api/test_dynamic_shape.cpp

+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include <migraphx/migraphx.h>
+#include <migraphx/migraphx.hpp>
+#include "test.hpp"
+
+TEST_CASE(create_dynamic_dimensions)
+{
+    migraphx::dynamic_dimension dd0{1, 4};
+    EXPECT(not dd0.is_fixed());
+    migraphx::dynamic_dimension dd1{4, 4};
+    EXPECT(dd1.is_fixed());
+    migraphx::optimals opts{1, 2, 4};
+    migraphx::dynamic_dimension dd2{1, 4, opts};
+    migraphx::dynamic_dimensions dyn_dims0{dd0, dd1, dd2};
+    CHECK(bool{dyn_dims0[0] == dd0});
+    CHECK(bool{dyn_dims0[1] == dd1});
+    CHECK(bool{dyn_dims0[2] == dd2});
+    CHECK(bool{dyn_dims0[2] != dd0});
+    EXPECT(dyn_dims0.size() == 3);
+}
+
+TEST_CASE(create_dynamic_shape)
+{
+    migraphx::dynamic_dimensions dyn_dims(migraphx::dynamic_dimension{1, 4},
+                                          migraphx::dynamic_dimension{78, 92},
+                                          migraphx::dynamic_dimension{1, 4, {1, 4}});
+    migraphx::shape dyn_shape{migraphx_shape_float_type, dyn_dims};
+    CHECK(bool{dyn_shape.dynamic()});
+    CHECK(bool{dyn_shape.dyn_dims()[0] == migraphx::dynamic_dimension{1, 4}});
+
+    migraphx::shape static_shape{migraphx_shape_float_type, {3, 8}};
+    EXPECT(not static_shape.dynamic());
+}
+
+int main(int argc, const char* argv[]) { test::run(argc, argv); }