Merge branch 'develop' into tests

src/targets/gpu/include/migraphx/gpu/miopen.hpp

@@ -34,11 +34,11 @@ Result make_obj(F f, Ts... xs)
     auto status                = f(&x, xs...);
     Result r{x};
     if(status != miopenStatusSuccess)
-        MIGRAPHX_THROW("MIOpen call failed");
+        MIGRAPHX_THROW("MAKE_OBJ: MIOpen call failed");
     return r;
 }
 
-inline tensor_descriptor make_tensor(const migraphx::shape& s)
+inline tensor_descriptor make_tensor(const migraphx::shape& s, bool pack = false)
 {
     auto t = make_obj<tensor_descriptor>(&miopenCreateTensorDescriptor);
     // Convert to ints
@@ -49,13 +49,33 @@ inline tensor_descriptor make_tensor(const migraphx::shape& s)
         d = miopenFloat;
     else if(s.type() == shape::half_type)
         d = miopenHalf;
+    else if(s.type() == shape::int32_type)
+        d = miopenInt32;
+    else if(s.type() == shape::int8_type)
+    {
+        if(pack)
+        {
+            // update the lens and corresponding strides
+            d          = miopenInt8x4;
+            lens[1]    = ((lens[1] + 3) / 4) * 4;
+            strides[0] = strides[1] * lens[1];
+        }
+        else
+        {
+            d = miopenInt8;
+        }
+    }
     else
-        MIGRAPHX_THROW("Unsupported type");
+    {
+        MIGRAPHX_THROW("MAKE_TENSOR: unsupported type");
+    }
     miopenSetTensorDescriptor(t.get(), d, s.lens().size(), lens.data(), strides.data());
+
     return t;
 }
 
-inline convolution_descriptor make_conv(const migraphx::op::convolution& op)
+template <class T>
+inline convolution_descriptor make_conv(const T& op)
 {
     auto c = make_obj<convolution_descriptor>(&miopenCreateConvolutionDescriptor);
     miopenConvolutionMode_t c_mode = miopenConvolution;

src/targets/gpu/include/migraphx/gpu/name.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_OP_NAME_HPP
+#define MIGRAPHX_GUARD_RTGLIB_OP_NAME_HPP
+
+#include <migraphx/config.hpp>
+#include <migraphx/type_name.hpp>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+template <class Derived>
+struct oper
+{
+    // function to extract the name part of an operator. For example, we have
+    // a operation "sin", then the get_type_name() will return
+    // "migraphx::version_1::gpu::hip_sin", this functin will return the name
+    // "gpu::sin" as the operator name
+    std::string name() const
+    {
+        const std::string& name = get_type_name<Derived>();
+        // search the namespace gpu (::gpu::)
+        auto pos_ns = name.find("::gpu::");
+        if(pos_ns != std::string::npos)
+        {
+            auto pos_name = name.find("hip_", pos_ns + std::string("::gpu::").length());
+            if(pos_name != std::string::npos)
+            {
+                return std::string("gpu::") + name.substr(pos_name + 4);
+            }
+            else
+            {
+                return name.substr(pos_ns + 2);
+            }
+        }
+
+        return "unknown_operator_name";
+    }
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/oper.hpp

 #ifndef MIGRAPHX_GUARD_RTGLIB_UNARY_HPP
 #define MIGRAPHX_GUARD_RTGLIB_UNARY_HPP
 
+#include <migraphx/gpu/name.hpp>
 #include <migraphx/gpu/hip.hpp>
 #include <migraphx/gpu/context.hpp>
 #include <migraphx/shape.hpp>
@@ -14,31 +15,6 @@ namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 namespace gpu {
 
-template <class Derived>
-struct oper
-{
-    std::string name() const
-    {
-        const std::string& name = get_type_name<Derived>();
-        // search the namespace gpu (::gpu::)
-        auto pos_ns = name.find("::gpu::");
-        if(pos_ns != std::string::npos)
-        {
-            auto pos_name = name.find("hip_", pos_ns + std::string("::gpu::").length());
-            if(pos_name != std::string::npos)
-            {
-                return std::string("gpu::") + name.substr(pos_name + 4);
-            }
-            else
-            {
-                return name.substr(pos_ns + 2);
-            }
-        }
-
-        return "unknown";
-    }
-};
-
 template <class Derived, void (*F)(hipStream_t, const argument&, const argument&)>
 struct unary_device : oper<Derived>
 {
@@ -88,7 +64,7 @@ struct binary_device : oper<Derived>
 
     argument compute(context& ctx, const shape&, const std::vector<argument>& args) const
     {
-        F(ctx.get_stream().get(), args[2], args[1], args[0]);
+        F(ctx.get_stream().get(), args[2], args[0], args[1]);
         return args[2];
     }
 
@@ -98,6 +74,39 @@ struct binary_device : oper<Derived>
     }
 };
 
+template <class Derived,
+          void (*F)(
+              hipStream_t, const argument&, const argument&, const argument&, const argument&)>
+struct ternary_device : oper<Derived>
+{
+    shape compute_shape(const std::vector<shape>& inputs) const
+    {
+        check_shapes{inputs, *this}.has(4);
+        auto s0 = inputs.at(0);
+        auto s1 = inputs.at(1);
+        auto s2 = inputs.at(2);
+        if(s0 == s1 and s1 == s2 and s0.packed())
+        {
+            return s0;
+        }
+        else
+        {
+            return {s0.type(), s0.lens()};
+        }
+    }
+
+    argument compute(context& ctx, const shape&, const std::vector<argument>& args) const
+    {
+        F(ctx.get_stream().get(), args[3], args[0], args[1], args[2]);
+        return args[3];
+    }
+
+    std::ptrdiff_t output_alias(const std::vector<shape>& shapes) const
+    {
+        return shapes.size() - 1;
+    }
+};
+
 } // namespace gpu
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx

src/targets/gpu/include/migraphx/gpu/pack_int8_args.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_PACK_INT8_ARGS_HPP
+#define MIGRAPHX_GUARD_RTGLIB_PACK_INT8_ARGS_HPP
+
+#include <migraphx/program.hpp>
+#include <migraphx/config.hpp>
+#include <migraphx/gpu/context.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+namespace gpu {
+
+struct pack_int8_args
+{
+    std::string name() const { return "gpu::pack_int8_args"; }
+    void apply(program& p) const;
+    shape pack_int8_shape(const shape& s) const;
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/pow.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_POW_HPP
+#define MIGRAPHX_GUARD_RTGLIB_POW_HPP
+
+#include <migraphx/gpu/oper.hpp>
+#include <migraphx/gpu/device/pow.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct hip_pow : binary_device<hip_pow, device::pow>
+{
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/quant_convolution.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_QUANT_CONVOLUTION_HPP
+#define MIGRAPHX_GUARD_RTGLIB_QUANT_CONVOLUTION_HPP
+
+#include <migraphx/shape.hpp>
+#include <migraphx/op/quant_convolution.hpp>
+#include <migraphx/gpu/miopen.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct context;
+
+struct miopen_quant_convolution
+{
+    op::quant_convolution op;
+    shared<convolution_descriptor> cd;
+    miopenConvFwdAlgorithm_t algo{};
+    miopenHandle_t handle = nullptr;
+
+    template <class Self, class F>
+    static auto reflect(Self& self, F f)
+    {
+        // TODO: Add algo
+        return op::quant_convolution::reflect(self.op, f);
+    }
+
+    std::string name() const { return "gpu::quant_convolution"; }
+    shape compute_shape(const std::vector<shape>& inputs) const;
+    argument
+    compute(context& ctx, const shape& output_shape, const std::vector<argument>& args) const;
+    shape compile(context& ctx, const shape& output_shape, std::vector<shape> inputs);
+    void finalize(context& ctx, const shape& output_shape, std::vector<shape> inputs);
+    std::ptrdiff_t output_alias(const std::vector<shape>& shapes) const
+    {
+        return shapes.size() - 1;
+    }
+
+    private:
+    shape pack_int8_shape(const shape& s) const;
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/reduce_mean.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_REDUCE_MEAN_HPP
+#define MIGRAPHX_GUARD_RTGLIB_REDUCE_MEAN_HPP
+
+#include <migraphx/op/reduce_mean.hpp>
+#include <migraphx/gpu/reduce_op.hpp>
+#include <migraphx/gpu/device/reduce_mean.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct context;
+
+struct hip_reduce_mean : reduce_op<hip_reduce_mean, op::reduce_mean, device::reduce_mean>
+{
+    hip_reduce_mean() {}
+    hip_reduce_mean(const op::reduce_mean& op_ref) : reduce_op(op_ref) {}
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/reduce_op.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_REDUCE_OP_HPP
+#define MIGRAPHX_GUARD_RTGLIB_REDUCE_OP_HPP
+
+#include <migraphx/gpu/name.hpp>
+#include <migraphx/gpu/hip.hpp>
+#include <migraphx/gpu/context.hpp>
+#include <migraphx/shape.hpp>
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <migraphx/type_name.hpp>
+#include <utility>
+#include <iostream>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct context;
+
+template <class Derived, class Op, void (*F)(hipStream_t, const argument&, const argument&)>
+struct reduce_op : oper<Derived>
+{
+    Op op;
+
+    template <class Self, class T>
+    static auto reflect(Self& self, T f)
+    {
+        return migraphx::reflect(self.op, f);
+    }
+
+    shape compute_shape(const std::vector<shape>& inputs) const
+    {
+        std::vector<shape> in_shapes{inputs};
+        in_shapes.pop_back();
+        return op.compute_shape(in_shapes);
+    }
+
+    argument compute(context& ctx, const shape&, const std::vector<argument>& args) const
+    {
+        F(ctx.get_stream().get(), args[1], args[0]);
+        return args[1];
+    }
+
+    std::ptrdiff_t output_alias(const std::vector<shape>& shapes) const
+    {
+        return shapes.size() - 1;
+    }
+
+    reduce_op() {}
+    reduce_op(const Op& op_ref) : op(op_ref) {}
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/reduce_sum.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_REDUCE_SUM_HPP
+#define MIGRAPHX_GUARD_RTGLIB_REDUCE_SUM_HPP
+
+#include <migraphx/op/reduce_sum.hpp>
+#include <migraphx/gpu/reduce_op.hpp>
+#include <migraphx/gpu/device/reduce_sum.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct context;
+
+struct hip_reduce_sum : reduce_op<hip_reduce_sum, op::reduce_sum, device::reduce_sum>
+{
+    hip_reduce_sum() {}
+    hip_reduce_sum(const op::reduce_sum& op_ref) : reduce_op(op_ref) {}
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/relu.hpp

 #ifndef MIGRAPHX_GUARD_RTGLIB_RELU_HPP
 #define MIGRAPHX_GUARD_RTGLIB_RELU_HPP
 
-#include <migraphx/shape.hpp>
-#include <migraphx/gpu/miopen.hpp>
+#include <migraphx/gpu/oper.hpp>
+#include <migraphx/gpu/device/relu.hpp>
 
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
@@ -10,24 +10,8 @@ namespace gpu {
 
 struct context;
 
-struct miopen_relu
+struct hip_relu : unary_device<hip_relu, device::relu>
 {
-    shared<activation_descriptor> ad;
-
-    template <class Self, class F>
-    static auto reflect(Self& self, F f)
-    {
-        return gpu::reflect(self.ad.get(), f);
-    }
-
-    std::string name() const { return "gpu::relu"; }
-    shape compute_shape(const std::vector<shape>& inputs) const;
-    argument
-    compute(context& ctx, const shape& output_shape, const std::vector<argument>& args) const;
-    std::ptrdiff_t output_alias(const std::vector<shape>& shapes) const
-    {
-        return shapes.size() - 1;
-    }
 };
 
 } // namespace gpu

src/targets/gpu/include/migraphx/gpu/round.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_ROUND_HPP
+#define MIGRAPHX_GUARD_RTGLIB_ROUND_HPP
+
+#include <migraphx/gpu/oper.hpp>
+#include <migraphx/gpu/device/round.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct hip_round : unary_device<hip_round, device::round>
+{
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/rsqrt.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_RSQRT_HPP
+#define MIGRAPHX_GUARD_RTGLIB_RSQRT_HPP
+
+#include <migraphx/gpu/oper.hpp>
+#include <migraphx/gpu/device/rsqrt.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct hip_rsqrt : unary_device<hip_rsqrt, device::rsqrt>
+{
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/sigmoid.hpp

 #ifndef MIGRAPHX_GUARD_RTGLIB_SIGMOID_HPP
 #define MIGRAPHX_GUARD_RTGLIB_SIGMOID_HPP
 
-#include <migraphx/shape.hpp>
-#include <migraphx/gpu/miopen.hpp>
+#include <migraphx/gpu/oper.hpp>
+#include <migraphx/gpu/device/sigmoid.hpp>
 
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 namespace gpu {
 
-struct context;
-
-struct miopen_sigmoid
+struct hip_sigmoid : unary_device<hip_sigmoid, device::sigmoid>
 {
-    shared<activation_descriptor> ad;
-
-    template <class Self, class F>
-    static auto reflect(Self& self, F f)
-    {
-        return gpu::reflect(self.ad.get(), f);
-    }
-
-    std::string name() const { return "gpu::sigmoid"; }
-    shape compute_shape(const std::vector<shape>& inputs) const;
-    argument
-    compute(context& ctx, const shape& output_shape, const std::vector<argument>& args) const;
-    std::ptrdiff_t output_alias(const std::vector<shape>& shapes) const
-    {
-        return shapes.size() - 1;
-    }
 };
 
 } // namespace gpu

src/targets/gpu/include/migraphx/gpu/sign.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_SIGN_HPP
+#define MIGRAPHX_GUARD_RTGLIB_SIGN_HPP
+
+#include <migraphx/gpu/oper.hpp>
+#include <migraphx/gpu/device/sign.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct hip_sign : unary_device<hip_sign, device::sign>
+{
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/softmax.hpp

 #ifndef MIGRAPHX_GUARD_RTGLIB_SOFTMAX_HPP
 #define MIGRAPHX_GUARD_RTGLIB_SOFTMAX_HPP
 
-#include <migraphx/shape.hpp>
+#include <migraphx/gpu/lowering.hpp>
+#include <migraphx/manage_ptr.hpp>
+#include <migraphx/instruction.hpp>
 #include <migraphx/op/softmax.hpp>
+#include <migraphx/generate.hpp>
+#include <migraphx/shape_for_each.hpp>
+#include <migraphx/config.hpp>
+#include <migraphx/gpu/miopen.hpp>
+#include <migraphx/gpu/hip.hpp>
+#include <migraphx/dfor.hpp>
+#include <migraphx/gpu/device/contiguous.hpp>
+#include <migraphx/gpu/device/add.hpp>
+#include <migraphx/iterator_for.hpp>
+#include <migraphx/gpu/rocblas.hpp>
+#include <migraphx/gpu/context.hpp>
+#include <utility>
 
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
@@ -20,6 +34,26 @@ struct miopen_softmax
         return migraphx::reflect(self.op, f);
     }
 
+    std::string name() const { return "miopen::softmax"; }
+    shape compute_shape(const std::vector<shape>& inputs) const;
+    argument
+    compute(context& ctx, const shape& output_shape, const std::vector<argument>& args) const;
+    std::ptrdiff_t output_alias(const std::vector<shape>& shapes) const
+    {
+        return shapes.size() - 1;
+    }
+};
+
+struct hip_softmax
+{
+    op::softmax op;
+
+    template <class Self, class F>
+    static auto reflect(Self& self, F f)
+    {
+        return migraphx::reflect(self.op, f);
+    }
+
     std::string name() const { return "gpu::softmax"; }
     shape compute_shape(const std::vector<shape>& inputs) const;
     argument

src/targets/gpu/include/migraphx/gpu/sqdiff.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_SQDIFF_HPP
+#define MIGRAPHX_GUARD_RTGLIB_SQDIFF_HPP
+
+#include <migraphx/gpu/oper.hpp>
+#include <migraphx/gpu/device/sqdiff.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct hip_sqdiff : binary_device<hip_sqdiff, device::sqdiff>
+{
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/sqrt.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_SQRT_HPP
+#define MIGRAPHX_GUARD_RTGLIB_SQRT_HPP
+
+#include <migraphx/gpu/oper.hpp>
+#include <migraphx/gpu/device/sqrt.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct hip_sqrt : unary_device<hip_sqrt, device::sqrt>
+{
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/tanh.hpp

 #ifndef MIGRAPHX_GUARD_RTGLIB_TANH_HPP
 #define MIGRAPHX_GUARD_RTGLIB_TANH_HPP
 
-#include <migraphx/shape.hpp>
-#include <migraphx/gpu/miopen.hpp>
+#include <migraphx/gpu/oper.hpp>
+#include <migraphx/gpu/device/tanh.hpp>
 
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 namespace gpu {
 
-struct context;
-
-struct miopen_tanh
+struct hip_tanh : unary_device<hip_tanh, device::tanh>
 {
-    shared<activation_descriptor> ad;
-
-    template <class Self, class F>
-    static auto reflect(Self& self, F f)
-    {
-        return gpu::reflect(self.ad.get(), f);
-    }
-
-    std::string name() const { return "gpu::tanh"; }
-    shape compute_shape(const std::vector<shape>& inputs) const;
-    argument
-    compute(context& ctx, const shape& output_shape, const std::vector<argument>& args) const;
-    std::ptrdiff_t output_alias(const std::vector<shape>& shapes) const
-    {
-        return shapes.size() - 1;
-    }
 };
 
 } // namespace gpu

src/targets/gpu/include/migraphx/gpu/target.hpp

@@ -13,6 +13,10 @@ struct target
     std::string name() const;
     std::vector<pass> get_passes(migraphx::context& gctx) const;
     migraphx::context get_context() const;
+
+    argument copy_to(const argument& arg) const;
+    argument copy_from(const argument& arg) const;
+    argument allocate(const shape& s) const;
 };
 
 } // namespace gpu

src/targets/gpu/int8_conv_pack.cpp

+#include <migraphx/gpu/int8_conv_pack.hpp>
+#include <migraphx/gpu/context.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+shape miopen_int8_conv_pack::compute_shape(const std::vector<shape>& inputs) const
+{
+    check_shapes{{inputs.at(0)}, *this}.has(1).standard();
+    return inputs.at(0);
+}
+
+argument
+miopen_int8_conv_pack::compute(context& ctx, const shape&, const std::vector<argument>& args) const
+{
+    auto arg_desc      = make_tensor(args[0].get_shape());
+    auto arg_desc_vec4 = make_tensor(args[0].get_shape(), true);
+
+    float alpha = 1;
+    float beta  = 0;
+    // pack input to vec4 format
+    auto status = miopenTransformTensor(ctx.get_stream().get_miopen(),
+                                        &alpha,
+                                        arg_desc.get(),
+                                        args[0].implicit(),
+                                        &beta,
+                                        arg_desc_vec4.get(),
+                                        args[1].implicit());
+    if(status != miopenStatusSuccess)
+    {
+        MIGRAPHX_THROW("INT8_CONV_PACK: transform input tensor failed");
+    }
+
+    return args[1];
+}
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx