manual merge

test/stringutils.cpp

@@ -99,4 +99,29 @@ TEST_CASE(interpolate_string_custom3)
     EXPECT(s == "****b****");
 }
 
+TEST_CASE(slit_string_simple1)
+{
+    std::string input = "one,two,three";
+    auto resuts       = migraphx::split_string(input, ',');
+    EXPECT(resuts.size() == 3);
+    EXPECT(resuts.front() == "one");
+    EXPECT(resuts.back() == "three");
+}
+
+TEST_CASE(slit_string_simple2)
+{
+    std::string input = "one";
+    auto resuts       = migraphx::split_string(input, ',');
+    EXPECT(resuts.size() == 1);
+    EXPECT(resuts.front() == "one");
+}
+
+TEST_CASE(slit_string_simple3)
+{
+    std::string input = "one two three";
+    auto resuts       = migraphx::split_string(input, ',');
+    EXPECT(resuts.size() == 1);
+    EXPECT(resuts.front() == "one two three");
+}
+
 int main(int argc, const char* argv[]) { test::run(argc, argv); }

test/targets.cpp

 /*
  * The MIT License (MIT)
  *
- * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal

test/tf/tf_test.cpp

@@ -37,7 +37,6 @@
 #include <migraphx/op/convolution.hpp>
 #include <migraphx/op/reduce_mean.hpp>
 #include <migraphx/op/pooling.hpp>
-#include <migraphx/op/slice.hpp>
 
 #include <migraphx/serialize.hpp>
 
@@ -840,12 +839,8 @@ TEST_CASE(slice_test)
     mm->add_literal(migraphx::literal{s0, {1, 0}});
     mm->add_literal(migraphx::literal{s0, {2, -1}});
 
-    migraphx::op::slice op;
-    op.starts = {1, 0};
-    op.ends   = {3, 10};
-    op.axes   = std::vector<int64_t>(num_axes);
-    std::iota(op.axes.begin(), op.axes.end(), 0);
-    mm->add_instruction(op, l0);
+    mm->add_instruction(
+        migraphx::make_op("slice", {{"starts", {1, 0}}, {"ends", {3, 10}}, {"axes", {0, 1}}}), l0);
     auto prog = optimize_tf("slice_test.pb", false);
 
     EXPECT(p == prog);
@@ -975,13 +970,10 @@ TEST_CASE(stridedslice_test)
     auto l0  = mm->add_parameter("0", migraphx::shape{migraphx::shape::float_type, {1, 10, 1, 1}});
     auto l1 =
         mm->add_instruction(migraphx::make_op("transpose", {{"permutation", {0, 2, 3, 1}}}), l0);
-    std::size_t num_axes = 4;
-    migraphx::op::slice op;
-    op.starts = {0, 0, 0, 0};
-    op.ends   = {1, 1, 1, 5};
-    op.axes   = std::vector<int64_t>(num_axes);
-    std::iota(op.axes.begin(), op.axes.end(), 0);
-    auto l2          = mm->add_instruction(op, l1);
+    auto l2 = mm->add_instruction(
+        migraphx::make_op(
+            "slice", {{"starts", {0, 0, 0, 0}}, {"ends", {1, 1, 1, 5}}, {"axes", {0, 1, 2, 3}}}),
+        l1);
     auto shrink_axis = 1;
     mm->add_instruction(migraphx::make_op("squeeze", {{"axes", {shrink_axis}}}), l2);
     auto prog = optimize_tf("stridedslice_test.pb", true);
@@ -995,12 +987,6 @@ TEST_CASE(stridedslice_masks_test)
 
     auto* mm = p.get_main_module();
     auto l0  = mm->add_parameter("0", migraphx::shape{migraphx::shape::float_type, {1, 10, 3, 3}});
-    std::size_t num_axes = 4;
-    migraphx::op::slice op;
-    op.starts = {0, 1, 1, 0};
-    op.ends   = {1, 3, 3, 10};
-    op.axes   = std::vector<int64_t>(num_axes);
-    std::iota(op.axes.begin(), op.axes.end(), 0);
     // add literals for starts, ends, and strides in tf (NHWC format)
     mm->add_literal(migraphx::shape{migraphx::shape::int32_type, {4}},
                     std::vector<int>{0, 1, 1, 0});
@@ -1011,7 +997,10 @@ TEST_CASE(stridedslice_masks_test)
 
     auto l1 =
         mm->add_instruction(migraphx::make_op("transpose", {{"permutation", {0, 2, 3, 1}}}), l0);
-    auto l2 = mm->add_instruction(op, l1);
+    auto l2 = mm->add_instruction(
+        migraphx::make_op(
+            "slice", {{"starts", {0, 1, 1, 0}}, {"ends", {1, 3, 3, 10}}, {"axes", {0, 1, 2, 3}}}),
+        l1);
     auto l3 =
         mm->add_instruction(migraphx::make_op("transpose", {{"permutation", {0, 3, 1, 2}}}), l2);
     mm->add_return({l3});

test/verify/CMakeLists.txt

@@ -25,14 +25,14 @@
 file(GLOB VERIFY_TESTS CONFIGURE_DEPENDS *.cpp)
 
 add_executable(test_verify ${VERIFY_TESTS})
-add_dependencies(tests test_verify)
-add_dependencies(check test_verify)
+rocm_mark_as_test(test_verify)
+rocm_install_test(TARGETS test_verify)
 target_link_libraries(test_verify migraphx migraphx_all_targets)
 target_include_directories(test_verify PUBLIC ../include)
 rocm_clang_tidy_check(test_verify)
 
 foreach(SECTION general rnn)
-    add_test_command(test_verify_${SECTION} test_verify ${SECTION})
+    rocm_add_test(NAME test_verify_${SECTION} COMMAND test_verify ${SECTION})
     set_tests_properties(test_verify_${SECTION} PROPERTIES 
         COST 100 
     )

test/verify/gemm_literal.cpp

@@ -25,7 +25,7 @@
 #include "verify_program.hpp"
 #include <migraphx/program.hpp>
 #include <migraphx/generate.hpp>
-#include <migraphx/operators.hpp>
+#include <migraphx/make_op.hpp>
 
 struct gemm_literal : verify_program<gemm_literal>
 {
@@ -38,7 +38,7 @@ struct gemm_literal : verify_program<gemm_literal>
 
         auto a = mm->add_literal(migraphx::generate_literal(a_shape));
         auto b = mm->add_parameter("b", b_shape);
-        mm->add_instruction(migraphx::op::dot{}, a, b);
+        mm->add_instruction(migraphx::make_op("dot"), a, b);
 
         return p;
     }

test/verify/run_verify.cpp

@@ -251,7 +251,8 @@ void run_verify::verify(const std::string& name,
             std::size_t num = gold.size();
             for(std::size_t i = 0; ((i < num) and passed); ++i)
             {
-                passed &= migraphx::verify_args(tname, gold[i], result[i]);
+                passed &= migraphx::verify_args_with_tolerance(
+                    tname, result[i], migraphx::verify::expected{gold[i]});
             }
 
             if(not passed or migraphx::enabled(MIGRAPHX_TRACE_TEST{}))

test/verify/test_layernorm.cpp

@@ -49,7 +49,8 @@ migraphx::instruction_ref add_layernorm(migraphx::module& m,
     auto pow            = m.add_instruction(migraphx::make_op("pow"), sub, exponent_mbcast);
     auto var            = m.add_instruction(migraphx::make_op("reduce_mean", {{"axes", {2}}}), pow);
     auto epsilon_mbcast = m.add_instruction(
-        migraphx::make_op("multibroadcast", {{"out_lens", {1, dims.at(1), 1}}}), epsilon);
+        migraphx::make_op("multibroadcast", {{"out_lens", {dims.at(0), dims.at(1), 1}}}), epsilon);
+
     auto add_epsilon = m.add_instruction(migraphx::make_op("add"), var, epsilon_mbcast);
     auto sqrt        = m.add_instruction(migraphx::make_op("sqrt"), add_epsilon);
     auto sqrt_mbcast =
@@ -57,7 +58,8 @@ migraphx::instruction_ref add_layernorm(migraphx::module& m,
     auto div = m.add_instruction(migraphx::make_op("div"), sub, sqrt_mbcast);
     auto scale_mbcast =
         m.add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", dims}}), scale);
-    auto mul = m.add_instruction(migraphx::make_op("mul"), scale_mbcast, div);
+    auto mul = m.add_instruction(migraphx::make_op("mul"), div, scale_mbcast);
+
     auto bias_mbcast =
         m.add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", dims}}), bias);
     return m.add_instruction(migraphx::make_op("add"), mul, bias_mbcast);
@@ -161,3 +163,21 @@ struct test_layernorm_triadd_large : verify_program<test_layernorm_triadd_large>
         return p;
     }
 };
+
+struct test_add_layernorm_add_gemm_nonstd : verify_program<test_add_layernorm_add_gemm_nonstd>
+{
+    migraphx::program create_program() const
+    {
+        migraphx::program p;
+        auto* mm = p.get_main_module();
+        auto s =
+            migraphx::shape::from_permutation(migraphx::shape::float_type, {8, 1, 16}, {1, 2, 0});
+        auto x = mm->add_parameter("x", s);
+        auto y = mm->add_parameter("y", s);
+        auto z = mm->add_parameter("z", migraphx::shape{migraphx::shape::float_type, {8, 16, 64}});
+        auto add           = mm->add_instruction(migraphx::make_op("add"), x, y);
+        auto layernorm_ins = add_layernorm(*mm, add, s.lens());
+        mm->add_instruction(migraphx::make_op("dot"), layernorm_ins, z);
+        return p;
+    }
+};

test/verify/test_reduce_add.cpp

+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "verify_program.hpp"
+#include <migraphx/program.hpp>
+#include <migraphx/generate.hpp>
+#include <migraphx/make_op.hpp>
+#include <migraphx/instruction.hpp>
+
+struct test_reduce_add : verify_program<test_reduce_add>
+{
+    migraphx::program create_program() const
+    {
+        migraphx::program p;
+        auto* mm = p.get_main_module();
+        migraphx::shape s{migraphx::shape::float_type, {4, 1000, 2, 2}};
+        migraphx::shape bs{migraphx::shape::half_type, {1, 32, 128}};
+        auto x = mm->add_parameter("x", s);
+        auto reduce_mean =
+            mm->add_instruction(migraphx::make_op("reduce_mean", {{"axes", {2, 3}}}), x);
+        auto reduce_max =
+            mm->add_instruction(migraphx::make_op("reduce_max", {{"axes", {2, 3}}}), x);
+        auto add = mm->add_instruction(migraphx::make_op("add"), reduce_mean, reduce_max);
+        mm->add_return({add});
+        return p;
+    };
+};

test/verify/test_reduce_noop_add.cpp

+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "verify_program.hpp"
+#include <migraphx/program.hpp>
+#include <migraphx/generate.hpp>
+#include <migraphx/make_op.hpp>
+#include <migraphx/instruction.hpp>
+
+struct test_reduce_noop_add : verify_program<test_reduce_noop_add>
+{
+    migraphx::program create_program() const
+    {
+        migraphx::program p;
+        auto* mm = p.get_main_module();
+        migraphx::shape s{migraphx::shape::float_type, {4, 1000, 1, 1}};
+        migraphx::shape bs{migraphx::shape::half_type, {1, 32, 128}};
+        auto x = mm->add_parameter("x", s);
+        auto reduce_mean =
+            mm->add_instruction(migraphx::make_op("reduce_mean", {{"axes", {2, 3}}}), x);
+        auto reduce_max =
+            mm->add_instruction(migraphx::make_op("reduce_max", {{"axes", {2, 3}}}), x);
+        auto add = mm->add_instruction(migraphx::make_op("add"), reduce_mean, reduce_max);
+        mm->add_return({add});
+        return p;
+    };
+};

tools/accuracy/accuracy_checker.py

@@ -82,6 +82,27 @@ def parse_args():
                         default=False,
                         help='Turn on ort VERBOSE logging via session options')
 
+    parser.add_argument(
+        '--disable-offload-copy',
+        dest="offload_copy",
+        action='store_false',
+        default=True,
+        help=
+        'Disable offload copying (user must handle copy to and from device)')
+
+    parser.add_argument(
+        '--disable-fast-math',
+        dest="fast_math",
+        action='store_false',
+        default=True,
+        help='Disable fast math optimizations (etc: rewrite_gelu)')
+
+    parser.add_argument('--exhaustive_tune',
+                        dest="exhaustive_tune",
+                        action='store_true',
+                        default=False,
+                        help='Enable exhaustive tuning for solutions')
+
     args = parser.parse_args()
 
     return args
@@ -177,7 +198,12 @@ def main():
         print(model)
 
     if not args.ort_run:
-        model.compile(migraphx.get_target(args.target))
+        model.compile(
+            migraphx.get_target(args.target),
+            offload_copy=args.offload_copy,
+            fast_math=args.fast_math,
+            exhaustive_tune=args.exhaustive_tune,
+        )
 
     params = {}
     test_inputs = {}

tools/build_and_test_onnxrt.sh

@@ -31,7 +31,7 @@ pip3 install -r requirements-dev.txt
 # Add newer cmake to the path
 export PATH="/opt/cmake/bin:$PATH"
 export CXXFLAGS="-D__HIP_PLATFORM_AMD__=1 -w"
-./build.sh --config Release  --cmake_extra_defines CMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ --update --build --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) --skip_tests --rocm_home /opt/rocm --use_migraphx --migraphx_home /opt/rocm --rocm_version=`cat /opt/rocm/.info/version-dev` --allow_running_as_root
+./build.sh --config Release  --cmake_extra_defines CMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ --update --build --build_wheel --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) --skip_tests --rocm_home /opt/rocm --use_migraphx --migraphx_home /opt/rocm --rocm_version=`cat /opt/rocm/.info/version-dev` --allow_running_as_root
 
 cd build/Linux/Release
 #Add test launcher for onnxrt tests

tools/docker/sles.docker

+FROM registry.suse.com/suse/sle15:15.4
+
+RUN sh -c 'echo -e "\
+[rocm]\n\
+name=rocm\n\
+baseurl=https://repo.radeon.com/rocm/zyp/5.7/main\n\
+enabled=1\n\
+gpgcheck=1\n\
+gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key\n\
+" > /etc/zypp/repos.d/rocm.repo'
+
+RUN cat /etc/zypp/repos.d/rocm.repo
+
+RUN zypper -n --gpg-auto-import-keys refresh
+
+RUN zypper install -y -t pattern devel_basis enhanced_base
+RUN zypper --gpg-auto-import-keys install -y \
+    doxygen \
+    gcc-c++ \
+    gdb \
+    git \
+    python3-pip \
+    rpm-build
+
+#addition of repos for packages
+RUN OPENSUSE_REPO=https://download.opensuse.org/repositories && \
+    zypper addrepo ${OPENSUSE_REPO}/devel:/languages:/perl/SLE_15_SP4/devel:languages:perl.repo
+
+# Workaround broken rocm packages
+RUN ln -s /opt/rocm-* /opt/rocm
+RUN echo "/opt/rocm/lib" > /etc/ld.so.conf.d/rocm.conf
+RUN echo "/opt/rocm/llvm/lib" > /etc/ld.so.conf.d/rocm-llvm.conf
+RUN ldconfig
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+# Install yapf
+RUN pip3 install yapf==0.28.0
+
+# Install doc requirements
+# ADD docs/.sphinx/requirements.txt /doc-requirements.txt
+# RUN pip3 install -r /doc-requirements.txt
+
+# Install dependencies
+ADD dev-requirements.txt /dev-requirements.txt
+ADD requirements.txt /requirements.txt
+ADD rbuild.ini /rbuild.ini
+
+COPY ./tools/install_prereqs.sh /
+RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh

tools/docker/ubuntu_2204.dockerfile

@@ -10,7 +10,7 @@ RUN apt-get update && apt-get install -y gnupg2 --no-install-recommends curl &&
     curl -fsSL http://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor -o /etc/apt/trusted.gpg.d/rocm-keyring.gpg
 
 # Add rocm repository
-RUN sh -c "echo 'deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/rocm-keyring.gpg] http://repo.radeon.com/rocm/apt/5.5 jammy main' > /etc/apt/sources.list.d/rocm.list"
+RUN sh -c "echo 'deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/rocm-keyring.gpg] http://repo.radeon.com/rocm/apt/5.7 jammy main' > /etc/apt/sources.list.d/rocm.list"
 
 # From docs.amd.com for installing rocm. Needed to install properly
 RUN sh -c "echo 'Package: *\nPin: release o=repo.radeon.com\nPin-priority: 600' > /etc/apt/preferences.d/rocm-pin-600"
@@ -87,7 +87,7 @@ RUN test -f /usr/local/hash || exit 1
 RUN pip3 install yapf==0.28.0
 
 # Install doc requirements
-ADD doc/requirements.txt /doc-requirements.txt
+ADD docs/.sphinx/requirements.txt /doc-requirements.txt
 RUN pip3 install -r /doc-requirements.txt
 
 # Download real models to run onnx unit tests

tools/download_models.sh

@@ -49,3 +49,8 @@ do
 curl https://download.onnxruntime.ai/onnx/models/$name.tar.gz --output $tmp_dir/$name.tar.gz
 tar -xzvf $tmp_dir/$name.tar.gz --directory $model_dir && rm $tmp_dir/$name.tar.gz
 done
+
+# CI jobs can run as a different user then the docker image builder.
+# Allow read/write access to the models
+chmod 777 $model_dir
+

tools/install_prereqs.sh

@@ -31,9 +31,30 @@ set -e
 export LC_ALL=C.UTF-8
 export LANG=C.UTF-8
 
+source /etc/os-release
+
+if [[ ("${ID}" == "sles") ]]; then
+  zypper -n --gpg-auto-import-keys install -y \
+    cmake \
+    miopen-hip-devel \
+    openmp-extras-devel \
+    python3-devel \
+    python3-pip \
+    rocblas-devel \
+    rocm-cmake
+else
+  # Need pip3 and Python headers to build dependencies
+  apt update && apt install -y \
+    cmake \
+    libnuma-dev \
+    miopen-hip-dev \
+    openmp-extras \
+    python3-dev \
+    python3-pip \
+    rocblas-dev \
+    rocm-cmake
+fi
 
-# Need pip3 and Python headers to build dependencies
-apt update && apt install -y python3-pip python3-dev cmake rocm-cmake rocblas miopen-hip openmp-extras
 
 # Needed for cmake to build various pip packages
 pip3 install setuptools wheel
@@ -56,9 +77,11 @@ echo "Dependencies are installed at $PREFIX"
 # Install deps with rbuild
 rbuild prepare -d $PREFIX -s develop
 
+if [[ ("${ID}" != "sles") ]]; then
 export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON"
 
-pip3 install onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0
+pip3 install onnx==1.14.1 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0
 
 # pin version of protobuf in Python for onnx runtime unit tests between dist versions
-pip3 install protobuf==3.20.0
+pip3 install protobuf==3.20.2
+fi

tools/license_stamper.py

@@ -2,7 +2,7 @@
 #####################################################################################
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+#  Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal
@@ -38,7 +38,7 @@ def getipynb_markdownBlockAsList():
         '\t\t"cell_type": "code",\n', '\t\t"execution_count": null,\n',
         '\t\t"metadata": {},\n', '\t\t"outputs": [],\n', '\t\t"source": [\n',
         '\t\t\t\"#  The MIT License (MIT)\\n\",\n', '\t\t\t\"#\\n\",\n',
-        '\t\t\t\"#  Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.\\n\",\n',
+        '\t\t\t\"#  Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.\\n\",\n',
         '\t\t\t\"#\\n\",\n',
         '\t\t\t\"#  Permission is hereby granted, free of charge, to any person obtaining a copy\\n\",\n',
         '\t\t\t\"#  of this software and associated documentation files (the \'Software\'), to deal\\n\",\n',

tools/test_runner.py

@@ -39,6 +39,15 @@ def parse_args():
                         type=str,
                         default='gpu',
                         help='Specify where the tests execute (ref, gpu)')
+    parser.add_argument('--fp16', action='store_true', help='Quantize to fp16')
+    parser.add_argument('--atol',
+                        type=float,
+                        default=1e-3,
+                        help='The absolute tolerance parameter')
+    parser.add_argument('--rtol',
+                        type=float,
+                        default=1e-3,
+                        help='The relative tolerance parameter')
     args = parser.parse_args()
 
     return args
@@ -257,6 +266,8 @@ def main():
 
     # read and compile model
     model = migraphx.parse_onnx(model_path_name, map_input_dims=param_shapes)
+    if args.fp16:
+        migraphx.quantize_fp16(model)
     model.compile(migraphx.get_target(target))
 
     # get test cases
@@ -279,7 +290,10 @@ def main():
         output_data = run_one_case(model, input_data)
 
         # check output correctness
-        ret = check_correctness(gold_outputs, output_data)
+        ret = check_correctness(gold_outputs,
+                                output_data,
+                                atol=args.atol,
+                                rtol=args.rtol)
         if ret:
             correct_num += 1