Merge branch 'develop' into tests

src/targets/gpu/include/migraphx/gpu/device/mul_add.hpp

+
+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_MUL_ADD_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_MUL_ADD_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void mul_add(hipStream_t stream,
+             const argument& result,
+             const argument& arg1,
+             const argument& arg2,
+             const argument& arg3);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/device/pow.hpp

+
+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_POW_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_POW_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void pow(hipStream_t stream, const argument& result, const argument& arg1, const argument& arg2);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/device/reduce_mean.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_REDUCE_MEAN_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_REDUCE_MEAN_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void reduce_mean(hipStream_t stream, const argument& result, const argument& arg);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/device/reduce_sum.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_REDUCE_SUM_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_REDUCE_SUM_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void reduce_sum(hipStream_t stream, const argument& result, const argument& arg);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/device/relu.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_RELU_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_RELU_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void relu(hipStream_t stream, const argument& result, const argument& arg);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/device/round.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_ROUND_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_ROUND_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void round(hipStream_t stream, const argument& result, const argument& arg);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/device/rsqrt.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_RSQRT_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_RSQRT_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void rsqrt(hipStream_t stream, const argument& result, const argument& arg);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/device/sigmoid.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_SIGMOID_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_SIGMOID_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void sigmoid(hipStream_t stream, const argument& result, const argument& arg);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/device/sign.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_SIGN_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_SIGN_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void sign(hipStream_t stream, const argument& result, const argument& arg);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/device/softmax.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_SOFTMAX_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_SOFTMAX_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void softmax(hipStream_t stream, const argument& result, const argument& arg, int axis);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/device/sqdiff.hpp

+
+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_SQDIFF_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_SQDIFF_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void sqdiff(hipStream_t stream, const argument& result, const argument& arg1, const argument& arg2);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/device/sqrt.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_SQRT_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_SQRT_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void sqrt(hipStream_t stream, const argument& result, const argument& arg);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/device/tanh.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_TANH_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DEVICE_TANH_HPP
+
+#include <migraphx/argument.hpp>
+#include <migraphx/config.hpp>
+#include <hip/hip_runtime_api.h>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+namespace device {
+
+void tanh(hipStream_t stream, const argument& result, const argument& arg);
+
+} // namespace device
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/div.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_DIV_HPP
+#define MIGRAPHX_GUARD_RTGLIB_DIV_HPP
+
+#include <migraphx/gpu/oper.hpp>
+#include <migraphx/gpu/device/div.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct hip_div : binary_device<hip_div, device::div>
+{
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/erf.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_ERF_HPP
+#define MIGRAPHX_GUARD_RTGLIB_ERF_HPP
+
+#include <migraphx/gpu/oper.hpp>
+#include <migraphx/gpu/device/erf.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct hip_erf : unary_device<hip_erf, device::erf>
+{
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/gemm.hpp

@@ -2,7 +2,11 @@
 #define MIGRAPHX_GUARD_RTGLIB_GEMM_HPP
 
 #include <migraphx/shape.hpp>
+#include <migraphx/gpu/context.hpp>
+#include <migraphx/gpu/gemm_impl.hpp>
+#include <migraphx/op/quant_dot.hpp>
 #include <migraphx/op/dot.hpp>
+#include <migraphx/ranges.hpp>
 
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
@@ -10,9 +14,10 @@ namespace gpu {
 
 struct context;
 
-struct miopen_gemm
+template <class Op>
+struct rocblas_gemm
 {
-    op::dot op;
+    Op op;
 
     template <class Self, class F>
     static auto reflect(Self& self, F f)
@@ -20,10 +25,49 @@ struct miopen_gemm
         return migraphx::reflect(self.op, f);
     }
 
-    std::string name() const { return "gpu::gemm"; }
-    shape compute_shape(const std::vector<shape>& inputs) const;
+    std::string name() const
+    {
+        if(contains(op.name(), "quant_"))
+        {
+            return "gpu::quant_gemm";
+        }
+        return "gpu::gemm";
+    }
+
+    shape compute_shape(const std::vector<shape>& inputs) const
+    {
+        std::vector<shape> in_shapes(inputs);
+        in_shapes.pop_back();
+        check_shapes{in_shapes}.not_broadcasted();
+        batch_not_transposed(inputs[0].strides());
+        batch_not_transposed(inputs[1].strides());
+
+        return op.compute_shape(in_shapes);
+    }
+
     argument
-    compute(context& ctx, const shape& output_shape, const std::vector<argument>& args) const;
+    compute(context& ctx, const shape& output_shape, const std::vector<argument>& args) const
+    {
+        gemm(ctx, output_shape, args, op.alpha, op.beta);
+        return args.back();
+    }
+
+    void batch_not_transposed(const std::vector<std::size_t>& strides) const
+    {
+        if(strides.size() <= 2)
+            return;
+        auto dim_0       = strides.size() - 2;
+        auto matrix_size = std::max(strides[dim_0], strides[dim_0 + 1]);
+        std::vector<std::size_t> batch(strides.begin(), strides.begin() + dim_0);
+        if(std::adjacent_find(batch.begin(), batch.end(), [&](auto i, auto j) {
+               return (i < j or i < matrix_size or j < matrix_size);
+           }) != batch.end())
+        {
+            MIGRAPHX_THROW("GPU_GEMM: batch size {" + to_string_range(strides) +
+                           "} is transposed!");
+        }
+    }
+
     std::ptrdiff_t output_alias(const std::vector<shape>& shapes) const
     {
         return shapes.size() - 1;

src/targets/gpu/include/migraphx/gpu/gemm_impl.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_GEMM_IMPL_HPP
+#define MIGRAPHX_GUARD_RTGLIB_GEMM_IMPL_HPP
+
+#include <migraphx/shape.hpp>
+#include <migraphx/argument.hpp>
+#include <migraphx/gpu/context.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+void gemm(context& ctx,
+          const shape& output_shape,
+          const std::vector<argument>& args,
+          float alpha,
+          float beta);
+void gemm(context& ctx,
+          const shape& output_shape,
+          const std::vector<argument>& args,
+          int32_t alpha,
+          int32_t beta);
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/hip.hpp

@@ -24,6 +24,8 @@ void gpu_sync();
 
 void copy_to_gpu(const argument& src, const argument& dst);
 
+void gpu_copy(context& ctx, const argument& src, const argument& dst);
+
 struct hip_allocate
 {
     shape s;
@@ -90,9 +92,9 @@ struct hip_write
     std::ptrdiff_t output_alias(const std::vector<shape>&) const { return 0; }
 };
 
-struct hip_copy
+struct hip_copy_to_gpu
 {
-    std::string name() const { return "hip_copy"; }
+    std::string name() const { return "hip_copy_to_gpu"; }
     shape compute_shape(std::vector<shape> inputs) const
     {
         check_shapes{inputs}.has(2);
@@ -106,6 +108,22 @@ struct hip_copy
     std::ptrdiff_t output_alias(const std::vector<shape>&) const { return 1; }
 };
 
+struct hip_copy
+{
+    std::string name() const { return "hip_copy"; }
+    shape compute_shape(std::vector<shape> inputs) const
+    {
+        check_shapes{inputs}.has(2).standard();
+        return inputs.at(1);
+    }
+    argument compute(context& ctx, const shape&, std::vector<argument> args) const
+    {
+        gpu_copy(ctx, args[0], args[1]);
+        return args[1];
+    }
+    std::ptrdiff_t output_alias(const std::vector<shape>&) const { return 1; }
+};
+
 } // namespace gpu
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx

src/targets/gpu/include/migraphx/gpu/int8_conv_pack.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_INT8_CONV_PACK_HPP
+#define MIGRAPHX_GUARD_RTGLIB_INT8_CONV_PACK_HPP
+
+#include <migraphx/op/quant_dot.hpp>
+#include <migraphx/config.hpp>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct context;
+
+struct miopen_int8_conv_pack
+{
+    std::string name() const { return "gpu::int8_conv_pack"; }
+    shape compute_shape(const std::vector<shape>& inputs) const;
+    argument compute(context& ctx, const shape&, const std::vector<argument>& args) const;
+    std::ptrdiff_t output_alias(const std::vector<shape>& shapes) const
+    {
+        return shapes.size() - 1;
+    }
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/targets/gpu/include/migraphx/gpu/int8_gemm_pack.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_INT8_GEMM_PACK_HPP
+#define MIGRAPHX_GUARD_RTGLIB_INT8_GEMM_PACK_HPP
+
+#include <migraphx/op/quant_dot.hpp>
+#include <migraphx/config.hpp>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace gpu {
+
+struct context;
+
+struct hip_int8_gemm_pack_a
+{
+    std::string name() const { return "gpu::int8_gemm_pack_a"; }
+    shape compute_shape(const std::vector<shape>& inputs) const;
+    argument compute(context& ctx, const shape&, const std::vector<argument>& args) const;
+    std::ptrdiff_t output_alias(const std::vector<shape>& shapes) const
+    {
+        return shapes.size() - 1;
+    }
+};
+
+struct hip_int8_gemm_pack_b
+{
+    std::string name() const { return "gpu::int8_gemm_pack_b"; }
+    shape compute_shape(const std::vector<shape>& inputs) const;
+    argument compute(context& ctx, const shape&, const std::vector<argument>& args) const;
+    std::ptrdiff_t output_alias(const std::vector<shape>& shapes) const
+    {
+        return shapes.size() - 1;
+    }
+};
+
+} // namespace gpu
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif