Merge remote-tracking branch 'origin/develop' into ck-flash-attn

src/include/migraphx/op/roialign.hpp

@@ -33,6 +33,7 @@
 #include <migraphx/dfor.hpp>
 #include <migraphx/ranges.hpp>
 #include <migraphx/shape_for_each.hpp>
+#include <array>
 #include <cmath>
 #include <numeric>
 #include <utility>

src/include/migraphx/optional.hpp

@@ -29,6 +29,17 @@
 #if defined(CPPCHECK)
 #define MIGRAPHX_HAS_OPTIONAL 1
 #define MIGRAPHX_HAS_OPTIONAL_TS 1
+#elif defined(_WIN32)
+#if _MSC_VER >= 1920
+#define MIGRAPHX_HAS_OPTIONAL 1
+#define MIGRAPHX_HAS_OPTIONAL_TS 0
+#elif _MSC_VER >= 1900
+#define MIGRAPHX_HAS_OPTIONAL 0
+#define MIGRAPHX_HAS_OPTIONAL_TS 1
+#else
+#define MIGRAPHX_HAS_OPTIONAL 0
+#define MIGRAPHX_HAS_OPTIONAL_TS 0
+#endif
 #elif defined(__has_include)
 #if __has_include(<optional>) && __cplusplus >= 201703L
 #define MIGRAPHX_HAS_OPTIONAL 1

src/include/migraphx/pad_calc.hpp

@@ -64,6 +64,7 @@ shape compute_padded_shape(const shape& input,
 
 // Used for dynamic auto padding of pooling operators where padding needs to be computed at
 // evaulation time.
+MIGRAPHX_EXPORT
 shape compute_padded_pool_shape(const shape& input,
                                 const shape& kernel,
                                 const std::vector<std::size_t>& padding,

src/include/migraphx/run_loop.hpp

@@ -31,6 +31,7 @@
 #include <migraphx/module.hpp>
 #include <migraphx/config.hpp>
 #include <migraphx/ranges.hpp>
+#include <array>
 #include <string>
 
 namespace migraphx {

src/include/migraphx/source_location.hpp

@@ -24,6 +24,7 @@
 #ifndef MIGRAPHX_GUARD_MIGRAPHX_SOURCE_LOCATION_HPP
 #define MIGRAPHX_GUARD_MIGRAPHX_SOURCE_LOCATION_HPP
 
+#include <cstdint>
 #include <migraphx/config.hpp>
 
 #if defined(CPPCHECK)

src/include/migraphx/type_name.hpp

@@ -34,7 +34,7 @@ template <class PrivateMigraphTypeNameProbe>
 std::string compute_type_name()
 {
     std::string name;
-#ifdef _MSC_VER
+#if defined(_MSC_VER) && !defined(__clang__)
     name = typeid(PrivateMigraphTypeNameProbe).name();
     name = name.substr(7);
 #else

src/onnx/broadcast_qdq.cpp

+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <migraphx/onnx/broadcast_qdq.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace onnx {
+
+// This method is to prep for quantizelinear or dequantizelinear operation for
+// either the broadcasting of weight-scale or zero-points of qlinearadd operator
+// outputs: operator op (inputs x, broadcasted: scale (float) & zero_pt (8-bit))
+instruction_ref bcast_qdq_instr(const std::string& op_name,
+                                instruction_ref x_in,
+                                instruction_ref arg_fscale,
+                                instruction_ref arg_z_pt,
+                                const onnx_parser::node_info& info)
+{
+    auto in_lens = x_in->get_shape().lens();
+
+    // prep 1: broadcast scale. it can come as a scalar or a 1-D tensor.
+    instruction_ref bcast_scale;
+    if(arg_fscale->get_shape().elements() > 1)
+        bcast_scale = info.add_instruction(
+            migraphx::make_op("broadcast", {{"axis", 0}, {"out_lens", in_lens}}), arg_fscale);
+    else
+        bcast_scale = info.add_instruction(
+            migraphx::make_op("multibroadcast", {{"out_lens", in_lens}}), arg_fscale);
+
+    // prep 2: broadcast zero point. it can come as a scalar or a 1-D tensor.
+    instruction_ref bcast_zero_pt;
+    if(arg_z_pt->get_shape().elements() > 1)
+        bcast_zero_pt = info.add_instruction(
+            migraphx::make_op("broadcast", {{"axis", 0}, {"out_lens", in_lens}}), arg_z_pt);
+    else
+        bcast_zero_pt = info.add_instruction(
+            migraphx::make_op("multibroadcast", {{"out_lens", in_lens}}), arg_z_pt);
+
+    // op_name is either quantizelinear or dequantizelinear:
+    return info.add_instruction(migraphx::make_op(op_name), x_in, bcast_scale, bcast_zero_pt);
+}
+
+// Multibroadcast a scaler..
+instruction_ref bcast_scalar_instr(const migraphx::shape& shape_out,
+                                   instruction_ref arg_in,
+                                   const onnx_parser::node_info& info)
+{
+    auto bcast_instr_out = info.add_instruction(
+        migraphx::make_op("multibroadcast", {{"out_lens", shape_out.lens()}}), arg_in);
+    return bcast_instr_out;
+}
+
+} // namespace onnx
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx

src/onnx/include/migraphx/onnx/broadcast_qdq.hpp

+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_BROADCAST_QDQ_HPP
+#define MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_BROADCAST_QDQ_HPP
+
+#include <string>
+
+#include <migraphx/onnx/op_parser.hpp>
+#include <migraphx/make_op.hpp>
+#include <migraphx/instruction.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace onnx {
+
+// This method is to prep for quantizelinear or dequantizelinear operation for
+// either the broadcasting of weight-scale or zero-points of qlinearadd operator
+// outputs: operator op (inputs x, broadcasted: scale (float) & zero_pt (8-bit))
+instruction_ref bcast_qdq_instr(const std::string& op_name,
+                                instruction_ref x_in,
+                                instruction_ref arg_fscale,
+                                instruction_ref arg_z_pt,
+                                const onnx_parser::node_info& info);
+
+// Multibroadcast a scaler..
+instruction_ref bcast_scalar_instr(const migraphx::shape& shape_out,
+                                   instruction_ref arg_in,
+                                   const onnx_parser::node_info& info);
+
+} // namespace onnx
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/onnx/onnx_parser.cpp

@@ -244,7 +244,7 @@ void onnx_parser::parse_from(std::istream& is, std::string name)
     this->filename   = std::move(name);
     auto parent_path = fs::path(this->filename).parent_path();
     if(not parent_path.empty())
-        this->path = parent_path;
+        this->path = parent_path.string();
 
     onnx::ModelProto model;
     if(model.ParseFromIstream(&is))

src/onnx/parse_depthtospace.cpp

@@ -87,8 +87,7 @@ struct parse_depthtospace : op_parser<parse_depthtospace>
 
         auto temp1 = info.add_instruction(make_op("reshape", {{"dims", lens1}}), args[0]);
         auto temp2 = info.add_instruction(make_op("transpose", {{"permutation", perm}}), temp1);
-        return info.add_instruction(make_op("reshape", {{"dims", lens2}}),
-                                    info.make_contiguous(temp2));
+        return info.add_instruction(make_op("reshape", {{"dims", lens2}}), temp2);
     }
 };
 

src/onnx/parse_qlinearadd.cpp

+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <migraphx/onnx/op_parser.hpp>
+#include <migraphx/ranges.hpp>
+#include <migraphx/common.hpp>
+#include <migraphx/make_op.hpp>
+#include <migraphx/onnx/checks.hpp>
+#include <migraphx/onnx/broadcast_qdq.hpp>
+#include <migraphx/instruction.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace onnx {
+
+/*
+ *********************************************************************************
+ *  Reference: see QLinearAdd in                                                 *
+ *  https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md  *
+ *********************************************************************************
+
+  com.microsoft.QLinearAdd
+  Performs element-wise binary addition on 8 bit data types (with Numpy-style broadcasting support).
+
+  C = (A_scale * (A - A_zero_point) + B_scale * (B - B_zero_point))/C_scale + C_zero_point
+
+  Version
+  This version of the operator has been available since version 1 of the 'com.microsoft' operator
+  set.
+
+  Inputs (7 - 8)
+  A : T
+  First operand.
+
+  A_scale : tensor(float)
+  Input A's scale. It's a scalar, which means a per-tensor/layer quantization.
+
+  A_zero_point (optional) : T
+  Input A zero point. Default value is 0 if it's not specified. It's a scalar, which means a
+  per-tensor/layer quantization.
+
+  B : T
+  Second operand.
+
+  B_scale : tensor(float)
+  Input B's scale. It's a scalar, which means a per-tensor/layer quantization.
+
+  B_zero_point (optional) : T
+  Input B zero point. Default value is 0 if it's not specified. It's a scalar, which means a
+  per-tensor/layer quantization.
+
+  C_scale : tensor(float)
+  Output scale. It's a scalar, which means a per-tensor/layer quantization.
+
+  C_zero_point (optional) : T
+
+  Output zero point. Default value is 0 if it's not specified. It's a scalar, which means a
+  per-tensor/layer quantization.
+
+  Outputs
+  C : T
+  Result, has same element type as two inputs
+
+  Type Constraints
+  T : tensor(uint8), tensor(int8)
+  Constrain input and output types to 8 bit signed and unsigned tensors.
+
+*/
+
+struct parse_qlinearadd : op_parser<parse_qlinearadd>
+{
+    std::vector<op_desc> operators() const { return {{"QLinearAdd"}}; }
+
+    // basic type checking for QLinearAdd Operator
+    void check_inputs(const std::vector<instruction_ref>& args) const
+    {
+        if(args.size() < 7)
+            MIGRAPHX_THROW("QLINEARADD: missing inputs");
+
+        const auto& in_a = args[0];
+        const auto& in_b = args[3];
+
+        auto sh_a = in_a->get_shape();
+        auto sh_b = in_b->get_shape();
+
+        auto type_a = sh_a.type();
+        auto type_b = sh_b.type();
+        if(type_a != migraphx::shape::int8_type and type_a != migraphx::shape::uint8_type)
+            MIGRAPHX_THROW("QLINEARADD: unsupported input type");
+        if(type_b != migraphx::shape::int8_type and type_b != migraphx::shape::uint8_type)
+            MIGRAPHX_THROW("QLINEARADD: unsupported input type");
+        if(type_a != type_b)
+            MIGRAPHX_THROW("QLINEARADD: mismatched input types");
+    }
+
+    instruction_ref parse(const op_desc& /* opd */,
+                          const onnx_parser& /*parser*/,
+                          const onnx_parser::node_info& info,
+                          const std::vector<instruction_ref>& args) const
+    {
+        check_inputs(args);
+
+        // A
+        const auto& in_a         = args[0];
+        const auto& in_scale_a   = args[1];
+        const auto& in_zero_pt_a = args[2];
+
+        auto dquant_a = bcast_qdq_instr("dequantizelinear", in_a, in_scale_a, in_zero_pt_a, info);
+
+        // B
+        const auto& in_b         = args[3];
+        const auto& in_scale_b   = args[4];
+        const auto& in_zero_pt_b = args[5];
+        auto dquant_b = bcast_qdq_instr("dequantizelinear", in_b, in_scale_b, in_zero_pt_b, info);
+
+        // C = A + B
+        auto out_c = info.add_common_op("add", dquant_a, dquant_b);
+
+        const auto& in_scale_c = args[6];
+
+        // zero_pt for C is supplied as the last optional argument..
+        if(args.size() == 8)
+            return (bcast_qdq_instr("quantizelinear", out_c, in_scale_c, args[7], info));
+
+        // if no zero_pt: just broadcast the scale..
+        auto bcast_scale_c = bcast_scalar_instr(out_c->get_shape(), in_scale_c, info);
+        return (info.add_instruction(migraphx::make_op("quantizelinear"), out_c, bcast_scale_c));
+    }
+};
+
+} // namespace onnx
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx

src/onnx/parse_reshape.cpp

@@ -53,8 +53,7 @@ struct parse_reshape : op_parser<parse_reshape>
             s.visit([&](auto v) { copy(v, std::back_inserter(dims)); });
         }
 
-        auto cont = info.add_instruction(make_op("contiguous"), args[0]);
-        return info.add_instruction(make_op("reshape", {{"dims", dims}}), cont);
+        return info.add_instruction(make_op("reshape", {{"dims", dims}}), args[0]);
     }
 };
 

src/onnx/parse_spacetodepth.cpp

@@ -73,8 +73,7 @@ struct parse_spacetodepth : op_parser<parse_spacetodepth>
         std::vector<int64_t> perm = {0, 3, 5, 1, 2, 4};
         auto temp1 = info.add_instruction(make_op("reshape", {{"dims", trans_lens}}), args[0]);
         auto temp2 = info.add_instruction(make_op("transpose", {{"permutation", perm}}), temp1);
-        return info.add_instruction(make_op("reshape", {{"dims", res_lens}}),
-                                    info.make_contiguous(temp2));
+        return info.add_instruction(make_op("reshape", {{"dims", res_lens}}), temp2);
     }
 };
 

src/process.cpp

@@ -26,13 +26,23 @@
 #include <migraphx/env.hpp>
 #include <functional>
 #include <iostream>
+#include <optional>
+
+#ifdef _WIN32
+// cppcheck-suppress definePrefix
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#else
 #include <unistd.h>
+#endif
 
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 
 MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_CMD_EXECUTE)
 
+#ifndef _WIN32
+
 std::function<void(const char*)> redirect_to(std::ostream& os)
 {
     return [&](const char* x) { os << x; };
@@ -74,6 +84,155 @@ int exec(const std::string& cmd, std::function<void(process::writer)> std_in)
     });
 }
 
+#else
+
+constexpr std::size_t MIGRAPHX_PROCESS_BUFSIZE = 4096;
+
+class pipe
+{
+    public:
+    explicit pipe(bool inherit_handle = true)
+    {
+        SECURITY_ATTRIBUTES attrs;
+        attrs.nLength              = sizeof(SECURITY_ATTRIBUTES);
+        attrs.bInheritHandle       = inherit_handle ? TRUE : FALSE;
+        attrs.lpSecurityDescriptor = nullptr;
+
+        if(CreatePipe(&m_read, &m_write, &attrs, 0) == FALSE)
+            throw GetLastError();
+
+        if(SetHandleInformation(&m_read, HANDLE_FLAG_INHERIT, 0) == FALSE)
+            throw GetLastError();
+    }
+
+    pipe(const pipe&)            = delete;
+    pipe& operator=(const pipe&) = delete;
+
+    pipe(pipe&&) = default;
+
+    ~pipe()
+    {
+        CloseHandle(m_read);
+        m_read = nullptr;
+        CloseHandle(m_write);
+        m_write = nullptr;
+    }
+
+    std::optional<std::pair<bool, DWORD>> read(LPVOID buffer, DWORD length) const
+    {
+        DWORD bytes_read;
+        if(ReadFile(m_read, buffer, length, &bytes_read, nullptr) == FALSE)
+        {
+            DWORD error{GetLastError()};
+            if(error != ERROR_MORE_DATA)
+            {
+                return std::nullopt;
+            }
+            return {{true, bytes_read}};
+        }
+        return {{false, bytes_read}};
+    }
+
+    HANDLE get_read_handle() const { return m_read; }
+
+    bool write(LPCVOID buffer, DWORD length) const
+    {
+        DWORD bytes_written;
+        return WriteFile(m_write, buffer, length, &bytes_written, nullptr) == TRUE;
+    }
+
+    HANDLE get_write_handle() const { return m_write; }
+
+    private:
+    HANDLE m_write = nullptr, m_read = nullptr;
+};
+
+template <typename F>
+int exec(const std::string& cmd, F f)
+{
+    try
+    {
+        if(enabled(MIGRAPHX_TRACE_CMD_EXECUTE{}))
+            std::cout << cmd << std::endl;
+
+        STARTUPINFO info;
+        PROCESS_INFORMATION process_info;
+
+        pipe in{}, out{};
+
+        ZeroMemory(&info, sizeof(STARTUPINFO));
+        info.cb         = sizeof(STARTUPINFO);
+        info.hStdError  = out.get_write_handle();
+        info.hStdOutput = out.get_write_handle();
+        info.hStdInput  = in.get_read_handle();
+        info.dwFlags |= STARTF_USESTDHANDLES;
+
+        ZeroMemory(&process_info, sizeof(process_info));
+
+        if(CreateProcess(nullptr,
+                         const_cast<LPSTR>(cmd.c_str()),
+                         nullptr,
+                         nullptr,
+                         TRUE,
+                         0,
+                         nullptr,
+                         nullptr,
+                         &info,
+                         &process_info) == FALSE)
+        {
+            return GetLastError();
+        }
+
+        f(in, out);
+
+        WaitForSingleObject(process_info.hProcess, INFINITE);
+
+        DWORD status{};
+        GetExitCodeProcess(process_info.hProcess, &status);
+
+        CloseHandle(process_info.hProcess);
+        CloseHandle(process_info.hThread);
+
+        return static_cast<int>(status);
+    }
+    // cppcheck-suppress catchExceptionByValue
+    catch(DWORD last_error)
+    {
+        return last_error;
+    }
+}
+
+int exec(const std::string& cmd)
+{
+    TCHAR buffer[MIGRAPHX_PROCESS_BUFSIZE];
+    HANDLE std_out{GetStdHandle(STD_OUTPUT_HANDLE)};
+    return (std_out == nullptr or std_out == INVALID_HANDLE_VALUE)
+               ? GetLastError()
+               : exec(cmd, [&](const pipe&, const pipe& out) {
+                     for(;;)
+                     {
+                         if(auto result = out.read(buffer, MIGRAPHX_PROCESS_BUFSIZE))
+                         {
+                             auto [more_data, bytes_read] = *result;
+                             if(not more_data or bytes_read == 0)
+                                 break;
+                             DWORD written;
+                             if(WriteFile(std_out, buffer, bytes_read, &written, nullptr) == FALSE)
+                                 break;
+                         }
+                     }
+                 });
+}
+
+int exec(const std::string& cmd, std::function<void(process::writer)> std_in)
+{
+    return exec(cmd, [&](const pipe& in, const pipe&) {
+        std_in([&](const char* buffer, std::size_t n) { in.write(buffer, n); });
+    });
+}
+
+#endif
+
 struct process_impl
 {
     std::string command{};
@@ -119,7 +278,14 @@ process& process::cwd(const fs::path& p)
     return *this;
 }
 
-void process::exec() { impl->check_exec(impl->get_command(), redirect_to(std::cout)); }
+void process::exec()
+{
+#ifndef _WIN32
+    impl->check_exec(impl->get_command(), redirect_to(std::cout));
+#else
+    impl->check_exec(impl->get_command());
+#endif
+}
 
 void process::write(std::function<void(process::writer)> pipe_in)
 {

src/py/CMakeLists.txt

@@ -22,27 +22,24 @@
 # THE SOFTWARE.
 #####################################################################################
 
-option(MIGRAPHX_ENABLE_PYTHON "Enable python bindings" ON)
 add_library(migraphx_py py_loader.cpp)
 migraphx_generate_export_header(migraphx_py)
 target_include_directories(migraphx_py PRIVATE include)
 target_link_libraries(migraphx_py PUBLIC migraphx)
 rocm_install_targets(TARGETS migraphx_py INCLUDE include)
-if(MIGRAPHX_ENABLE_PYTHON)
-    include(PythonModules)
 
+include(PythonModules)
 
-    foreach(PYTHON_VERSION ${PYTHON_VERSIONS})
-        py_add_module(migraphx_pybind_${PYTHON_VERSION} migraphx_py.cpp PYTHON_VERSION ${PYTHON_VERSION} PYTHON_MODULE migraphx)
-        target_link_libraries(migraphx_pybind_${PYTHON_VERSION} PRIVATE migraphx migraphx_tf migraphx_onnx migraphx_all_targets)
-        rocm_install_targets(TARGETS migraphx_pybind_${PYTHON_VERSION})
-        add_dependencies(migraphx_py migraphx_pybind_${PYTHON_VERSION})
-        
-        add_library(migraphx_py_${PYTHON_VERSION} py.cpp)
-        target_include_directories(migraphx_py_${PYTHON_VERSION} PRIVATE include)
-        target_link_libraries(migraphx_py_${PYTHON_VERSION} PUBLIC migraphx)
-        target_link_libraries(migraphx_py_${PYTHON_VERSION} PRIVATE pybind11::pybind11 python${PYTHON_VERSION}::runtime)
-        rocm_install_targets(TARGETS migraphx_py_${PYTHON_VERSION})
-        add_dependencies(migraphx_py migraphx_py_${PYTHON_VERSION})
-    endforeach()
-endif()
+foreach(PYTHON_VERSION ${PYTHON_VERSIONS})
+    py_add_module(migraphx_pybind_${PYTHON_VERSION} migraphx_py.cpp PYTHON_VERSION ${PYTHON_VERSION} PYTHON_MODULE migraphx)
+    target_link_libraries(migraphx_pybind_${PYTHON_VERSION} PRIVATE migraphx migraphx_tf migraphx_onnx migraphx_all_targets)
+    rocm_install_targets(TARGETS migraphx_pybind_${PYTHON_VERSION})
+    add_dependencies(migraphx_py migraphx_pybind_${PYTHON_VERSION})
+
+    add_library(migraphx_py_${PYTHON_VERSION} py.cpp)
+    target_include_directories(migraphx_py_${PYTHON_VERSION} PRIVATE include)
+    target_link_libraries(migraphx_py_${PYTHON_VERSION} PUBLIC migraphx)
+    target_link_libraries(migraphx_py_${PYTHON_VERSION} PRIVATE pybind11::pybind11 python${PYTHON_VERSION}::runtime)
+    rocm_install_targets(TARGETS migraphx_py_${PYTHON_VERSION})
+    add_dependencies(migraphx_py migraphx_py_${PYTHON_VERSION})
+endforeach()

src/quantization.cpp

@@ -70,6 +70,10 @@ void quantize_int8(program& prog,
         MIGRAPHX_THROW("QUANTIZE_INT8: only support DOT and CONVOLUTION operation");
     }
 
+    // Run optimize_module() before converting to int8 to const eval and fold in FP32 to
+    // avoid loss of precision.
+    run_passes(prog, {optimize_module{}});
+
     std::shared_ptr<std::vector<std::pair<float, float>>> int8_quant_params =
         std::make_shared<std::vector<std::pair<float, float>>>();
     std::shared_ptr<std::vector<float>> max_abs_vals = std::make_shared<std::vector<float>>();
@@ -143,10 +147,7 @@ void quantize_int8(program& prog,
 
     run_passes(prog,
                {quantize_int8_pass{ins_names, *int8_quant_params},
-                eliminate_common_subexpression{},
-                dead_code_elimination{},
-                simplify_reshapes{},
-                dead_code_elimination{},
+                optimize_module{},
                 simplify_qdq{},
                 dead_code_elimination{}});
 }

src/targets/cpu/include/migraphx/cpu/dnnl.hpp

@@ -91,6 +91,19 @@ struct post_op : reflect_equality<post_op>, reflect_stream<post_op>
     }
 };
 
+template <class F>
+struct execute_wrapper
+{
+    F f;
+    argument operator()(context&, const std::vector<argument>& args) const { return f(args); }
+};
+
+template <class F>
+execute_wrapper<F> make_execute_wrapper(F f)
+{
+    return {std::move(f)};
+}
+
 template <class Derived, class Primitive>
 struct dnnl_op : auto_register_op<Derived>
 {
@@ -308,7 +321,7 @@ struct dnnl_op : auto_register_op<Derived>
 #ifndef NDEBUG
         auto prim_attr = get_primitive_attr(md);
 #endif
-        execute = [=](context&, const std::vector<argument>& args) {
+        execute = make_execute_wrapper([=](const std::vector<argument>& args) {
 #ifndef NDEBUG
             // Check that the memory descriptors have not changed
             auto debug_args = args;
@@ -379,7 +392,7 @@ struct dnnl_op : auto_register_op<Derived>
                 m[arg_lookup[i]] = to_dnnl_memory(md.at(arg_lookup[i]), args[i]);
             prim.execute(get_dnnl_context().stream, m);
             return args.back();
-        };
+        });
     }
     std::vector<shape> trim_post_op_inputs(const std::vector<shape>& inputs) const
     {

src/targets/cpu/include/migraphx/cpu/pointwise.hpp

@@ -24,6 +24,7 @@
 #ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_CPU_POINTWISE_HPP
 #define MIGRAPHX_GUARD_AMDMIGRAPHX_CPU_POINTWISE_HPP
 
+#include <array>
 #include <migraphx/config.hpp>
 #include <migraphx/context.hpp>
 #include <migraphx/check_shapes.hpp>

src/targets/gpu/device/targets.hpp.in

@@ -24,7 +24,7 @@
 #ifndef MIGRAPHX_GUARD_DEVICE_TARGETS_CPP
 #define MIGRAPHX_GUARD_DEVICE_TARGETS_CPP
 
-#include <migraphx/config.hpp>
+#include <migraphx/gpu/device/config.hpp>
 #include <string>
 #include <vector>
 
@@ -34,9 +34,13 @@ namespace gpu {
 namespace device {
 #define MIGRAPHX_GPU_TARGETS "@GPU_TARGETS@" // NOLINT
 
+MIGRAPHX_DEVICE_EXPORT
 const std::vector<std::string>& get_targets();
+
+MIGRAPHX_DEVICE_EXPORT
 std::string get_targets_as_string();
 
+MIGRAPHX_DEVICE_EXPORT
 std::string get_device_name();
 
 } // namespace device

src/targets/gpu/hiprtc/main.cpp

@@ -27,6 +27,7 @@
 #include <migraphx/msgpack.hpp>
 #include <migraphx/file_buffer.hpp>
 #include <migraphx/ranges.hpp>
+#include <array>
 #include <iostream>
 #include <cstring>