Merge branch 'develop' into batch-concat

src/include/migraphx/concat_opt.hpp

@@ -9,7 +9,7 @@
 #include <utility>
 
 #include <migraphx/operation.hpp>
-#include <migraphx/operators.hpp>
+#include <migraphx/op/concat.hpp>
 #include <migraphx/config.hpp>
 
 namespace migraphx {

src/include/migraphx/eliminate_identity.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_ELIMINATE_IDENTITY_HPP
+#define MIGRAPHX_GUARD_RTGLIB_ELIMINATE_IDENTITY_HPP
+
+#include <string>
+#include <migraphx/instruction_ref.hpp>
+#include <migraphx/config.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+struct program;
+
+/**
+ * Remove identity instructions. Currently when used as the last pass, it will
+ * preserve the semantics of previous program state, therefore dead code elimination
+ * should not be used afterwards.
+ */
+struct eliminate_identity
+{
+    std::string name() const { return "eliminate_identity"; }
+    void apply(program& p) const;
+};
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/eliminate_pad.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_ELIMINATE_PAD_HPP
+#define MIGRAPHX_GUARD_RTGLIB_ELIMINATE_PAD_HPP
+
+#include <string>
+#include <vector>
+#include <array>
+#include <migraphx/instruction_ref.hpp>
+#include <migraphx/config.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+struct program;
+
+/**
+ * Remove pads if they can be written as an
+ * attribute to another op (im2col, convolution, pooling)
+ */
+struct eliminate_pad
+{
+    std::string name() const { return "eliminate_pad"; }
+    void apply(program& p) const;
+    template <class T>
+    void update_op(T, const instruction_ref& input, const instruction_ref& ins, program& p) const;
+};
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/env.hpp

@@ -19,6 +19,8 @@ bool enabled(const char* name);
 bool disabled(const char* name);
 std::vector<std::string> env(const char* name);
 
+std::size_t value_of(const char* name);
+
 template <class T>
 bool enabled(T)
 {
@@ -33,6 +35,13 @@ bool disabled(T)
     return result;
 }
 
+template <class T>
+std::size_t value_of(T)
+{
+    static const std::size_t result = value_of(T::value());
+    return result;
+}
+
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx
 

src/include/migraphx/generate.hpp

@@ -17,7 +17,7 @@ constexpr T normalize(unsigned long z)
         return T(0);
     const auto max     = 32;
     const double range = max / 2; // NOLINT
-    double result      = (z % max) / range;
+    double result      = double(z % max) / range;
     result -= 1;
     return T(result);
 }

src/include/migraphx/instruction.hpp

@@ -24,7 +24,7 @@ struct instruction
 
     instruction(literal l);
 
-    void replace(const shape& r);
+    void replace(operation o);
 
     void recompute_shape();
 
@@ -72,7 +72,9 @@ struct instruction
     static void
     replace(instruction_ref ins, operation o, const shape& r, std::vector<instruction_ref> args);
 
-    argument eval() const;
+    bool can_eval() const;
+
+    argument eval(bool check_eval = true) const;
 
     void finalize(context& ctx);
 
@@ -88,7 +90,8 @@ struct instruction
     // internal
     void replace_argument(instruction_ref old, instruction_ref new_ins);
 
-    private:
+    void replace(const shape& r);
+
     operation op;
     shape result;
     std::vector<instruction_ref> output;

src/include/migraphx/int_divide.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_INT_DIVIDE_HPP
+#define MIGRAPHX_GUARD_RTGLIB_INT_DIVIDE_HPP
+
+#include <migraphx/config.hpp>
+#include <cmath>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+template <class R, class T, class U>
+R floor_divide(T x, U y)
+{
+    return R(std::floor(double(x) / double(y)));
+}
+
+template <class R, class T, class U>
+R ceil_divide(T x, U y)
+{
+    return R(std::ceil(double(x) / double(y)));
+}
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/make_signed.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_MAKE_SIGNED_HPP
+#define MIGRAPHX_GUARD_RTGLIB_MAKE_SIGNED_HPP
+
+#include <migraphx/config.hpp>
+#include <type_traits>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+template <class T>
+typename std::conditional_t<std::is_integral<T>{}, std::make_signed<T>, std::enable_if<true, T>>::
+    type
+    make_signed(T x)
+{
+    return x;
+}
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/abnormal_ops.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_ABNORMAL_OPS_HPP
+#define MIGRAPHX_GUARD_OPERATORS_ABNORMAL_OPS_HPP
+
+#include <array>
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/stringutils.hpp>
+#include <migraphx/streamutils.hpp>
+#include <migraphx/literal.hpp>
+#include <migraphx/shape_for_each.hpp>
+#include <migraphx/config.hpp>
+#include <cmath>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+struct not_computable
+{
+    argument compute(const shape&, const std::vector<argument>&) const
+    {
+        MIGRAPHX_THROW("not computable");
+    }
+};
+
+struct undefined
+{
+    std::string name() const { return "undefined"; }
+    shape compute_shape(const std::vector<shape>& inputs) const
+    {
+        check_shapes{inputs, *this}.has(0);
+        return {};
+    }
+
+    argument compute(const shape&, const std::vector<argument>&) const { return {{}, nullptr}; }
+};
+
+struct unknown
+{
+    std::string op;
+    template <class Self, class F>
+    static auto reflect(Self& self, F f)
+    {
+        return pack(f(self.op, "op"));
+    }
+    std::string name() const { return "unknown:" + op; }
+    shape compute_shape(std::vector<shape> input) const
+    {
+        if(input.empty())
+            return {};
+        else
+            return input.front();
+    }
+
+    friend std::ostream& operator<<(std::ostream& os, const unknown& x)
+    {
+        os << x.name();
+        return os;
+    }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/abs.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_ABS_HPP
+#define MIGRAPHX_GUARD_OPERATORS_ABS_HPP
+
+#include <array>
+#include <migraphx/op/unary.hpp>
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/stringutils.hpp>
+#include <migraphx/streamutils.hpp>
+#include <migraphx/literal.hpp>
+#include <migraphx/shape_for_each.hpp>
+#include <migraphx/config.hpp>
+#include <migraphx/make_signed.hpp>
+#include <cmath>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+struct abs : unary<abs>
+{
+    auto apply() const
+    {
+        return [](auto x) { return std::abs(make_signed(x)); };
+    }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/acos.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_ACOS_HPP
+#define MIGRAPHX_GUARD_OPERATORS_ACOS_HPP
+
+#include <array>
+#include <migraphx/op/unary.hpp>
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/stringutils.hpp>
+#include <migraphx/streamutils.hpp>
+#include <migraphx/literal.hpp>
+#include <migraphx/shape_for_each.hpp>
+#include <migraphx/config.hpp>
+#include <cmath>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+struct acos : unary<acos>
+{
+    auto apply() const
+    {
+        return [](auto x) { return std::acos(x); };
+    }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/add.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_ADD_HPP
+#define MIGRAPHX_GUARD_OPERATORS_ADD_HPP
+
+#include <array>
+#include <migraphx/op/binary.hpp>
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/stringutils.hpp>
+#include <migraphx/streamutils.hpp>
+#include <migraphx/literal.hpp>
+#include <migraphx/shape_for_each.hpp>
+#include <migraphx/config.hpp>
+#include <cmath>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+struct add : binary<add>
+{
+    auto apply() const
+    {
+        return [](auto x, auto y) { return x + y; };
+    }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/as_shape.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_AS_SHAPE_HPP
+#define MIGRAPHX_GUARD_OPERATORS_AS_SHAPE_HPP
+
+#include <array>
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/stringutils.hpp>
+#include <migraphx/streamutils.hpp>
+#include <migraphx/literal.hpp>
+#include <migraphx/shape_for_each.hpp>
+#include <migraphx/config.hpp>
+#include <cmath>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+struct as_shape
+{
+    shape s;
+    template <class Self, class F>
+    static auto reflect(Self& self, F f)
+    {
+        return pack(f(self.s, "shape"));
+    }
+
+    std::string name() const { return "as_shape"; }
+    shape compute_shape(const std::vector<shape>& inputs) const
+    {
+        check_shapes{inputs, *this}.has(1).standard();
+        assert(inputs.front().elements() == s.elements());
+        return s;
+    }
+    argument compute(shape output_shape, std::vector<argument> args) const
+    {
+        return {std::move(output_shape), std::move(args.front().data)};
+    }
+    std::ptrdiff_t output_alias(const std::vector<shape>&) const { return 0; }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/asin.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_ASIN_HPP
+#define MIGRAPHX_GUARD_OPERATORS_ASIN_HPP
+
+#include <array>
+#include <migraphx/op/unary.hpp>
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/stringutils.hpp>
+#include <migraphx/streamutils.hpp>
+#include <migraphx/literal.hpp>
+#include <migraphx/shape_for_each.hpp>
+#include <migraphx/config.hpp>
+#include <cmath>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+struct asin : unary<asin>
+{
+    auto apply() const
+    {
+        return [](auto x) { return std::asin(x); };
+    }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/atan.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_ATAN_HPP
+#define MIGRAPHX_GUARD_OPERATORS_ATAN_HPP
+
+#include <array>
+#include <migraphx/op/unary.hpp>
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/stringutils.hpp>
+#include <migraphx/streamutils.hpp>
+#include <migraphx/literal.hpp>
+#include <migraphx/shape_for_each.hpp>
+#include <migraphx/config.hpp>
+#include <cmath>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+struct atan : unary<atan>
+{
+    auto apply() const
+    {
+        return [](auto x) { return std::atan(x); };
+    }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/batch_norm.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_BATCH_NORM_HPP
+#define MIGRAPHX_GUARD_OPERATORS_BATCH_NORM_HPP
+
+#include <array>
+#include <migraphx/op/common.hpp>
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/stringutils.hpp>
+#include <migraphx/streamutils.hpp>
+#include <migraphx/literal.hpp>
+#include <migraphx/shape_for_each.hpp>
+#include <migraphx/config.hpp>
+#include <cmath>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+struct batch_norm_inference
+{
+    float epsilon  = 1.0e-6f;
+    float momentum = 0.9f;
+
+    std::string name() const { return "batch_norm_inference"; }
+
+    enum bn_infer_mode_t
+    {
+        per_activation,
+        spatial,
+    };
+
+    bn_infer_mode_t bn_mode = spatial;
+
+    template <class Self, class F>
+    static auto reflect(Self& self, F f)
+    {
+        return pack(
+            f(self.epsilon, "epsilon"), f(self.momentum, "momentum"), f(self.bn_mode, "bn_mode"));
+    }
+
+    shape compute_shape(std::vector<shape> inputs) const
+    {
+        check_shapes{inputs, *this}.has(5);
+        check_shapes{inputs.data(), inputs.data() + 1, *this}.only_dims(4);
+        check_shapes{inputs.data() + 1, inputs.data() + inputs.size(), *this}.same_shape().elements(
+            inputs.front().lens()[1]);
+        return inputs.front();
+    }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/binary.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_BINARY_HPP
+#define MIGRAPHX_GUARD_OPERATORS_BINARY_HPP
+
+#include <migraphx/op/name.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+template <class Derived>
+struct binary : op_name<Derived>
+{
+    shape compute_shape(std::vector<shape> inputs) const
+    {
+        check_shapes{inputs}.has(2).same_type().same_dims();
+        auto s0 = inputs.at(0);
+        auto s1 = inputs.at(1);
+        if(s0 == s1 and s0.packed())
+        {
+            return s0;
+        }
+        else
+        {
+            return {s0.type(), s0.lens()};
+        }
+    }
+
+    argument compute(const shape& output_shape, std::vector<argument> args) const
+    {
+        argument result{output_shape};
+        visit_all(result, args[0], args[1])([&](auto output, auto input1, auto input2) {
+            if(input1.get_shape().packed() and input2.get_shape().packed())
+            {
+                std::transform(input1.begin(),
+                               input1.end(),
+                               input2.begin(),
+                               output.begin(),
+                               static_cast<const Derived&>(*this).apply());
+            }
+            else
+            {
+                shape_for_each(output.get_shape(), [&](const auto& idx) {
+                    output(idx.begin(), idx.end()) = static_cast<const Derived&>(*this).apply()(
+                        input1(idx.begin(), idx.end()), input2(idx.begin(), idx.end()));
+                });
+            }
+        });
+
+        return result;
+    }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/broadcast.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_BROADCAST_HPP
+#define MIGRAPHX_GUARD_OPERATORS_BROADCAST_HPP
+
+#include <array>
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/stringutils.hpp>
+#include <migraphx/streamutils.hpp>
+#include <migraphx/literal.hpp>
+#include <migraphx/shape_for_each.hpp>
+#include <migraphx/config.hpp>
+#include <cmath>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+/// The broadcast operator performs the numpy-style broadcasting of an axis of a given tensor. This
+/// is achieved primarily by setting the stride of the broadcasted axis to zero. Linear indicies are
+/// computed from multi-indicies by computing the inner product on the multi-index with the strides.
+/// For example, if we have a tensor A(2,3) it has lengths of (2,3) and strides of (3,1). If we want
+/// to compute the linear offset that corresponds to the element on the 2nd row (i = 1) and 3rd
+/// column (j = 2), we compute the following inner product (1,2) dot (3, 1) = 1*3 + 2*1 = 5. It is
+/// obvious from there that we can negate the effects of a given axis by setting the stride of that
+/// axis to zero.
+struct broadcast
+{
+    uint64_t axis = 0;
+    std::vector<std::size_t> broadcast_lens;
+
+    template <class Self, class F>
+    static auto reflect(Self& self, F f)
+    {
+        return pack(f(self.axis, "axis"), f(self.broadcast_lens, "dims"));
+    }
+
+    std::string name() const { return "broadcast"; }
+    shape compute_shape(std::vector<shape> inputs) const
+    {
+        auto t     = inputs.at(0).type();
+        auto input = inputs.at(0);
+
+        std::vector<size_t> bcast_strides(broadcast_lens.size(), 0);
+
+        if(std::all_of(
+               broadcast_lens.cbegin(), broadcast_lens.cend(), [&](auto x) { return x == 1; }))
+        {
+            if(axis != 0)
+                MIGRAPHX_THROW("BROADCAST: when broadcasting tensor of size 1, axis should be 0");
+            return {t, broadcast_lens, std::move(bcast_strides)};
+        }
+        else
+        {
+            assert(broadcast_lens.size() - axis >= input.lens().size());
+            if(!std::equal(input.lens().begin(), input.lens().end(), broadcast_lens.begin() + axis))
+                MIGRAPHX_THROW("BROADCAST: when broadcasting success sizes must match");
+            std::copy(input.strides().begin(), input.strides().end(), bcast_strides.begin() + axis);
+            return {t, broadcast_lens, std::move(bcast_strides)};
+        }
+    }
+    argument compute(shape output_shape, std::vector<argument> args) const
+    {
+        return {std::move(output_shape), std::move(args.at(0).data)};
+    }
+    std::ptrdiff_t output_alias(const std::vector<shape>&) const { return 0; }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/clip.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_CLIP_HPP
+#define MIGRAPHX_GUARD_OPERATORS_CLIP_HPP
+
+#include <array>
+#include <migraphx/op/unary.hpp>
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/stringutils.hpp>
+#include <migraphx/streamutils.hpp>
+#include <migraphx/literal.hpp>
+#include <migraphx/shape_for_each.hpp>
+#include <migraphx/config.hpp>
+#include <cmath>
+#include <utility>
+#include <limits>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+struct clip : unary<clip>
+{
+    float max_val = std::numeric_limits<float>::max();
+    float min_val = std::numeric_limits<float>::min();
+
+    clip() {}
+
+    clip(float max, float min) : max_val(max), min_val(min) {}
+
+    auto apply() const
+    {
+        auto max = max_val;
+        auto min = min_val;
+        return [max, min](auto x) {
+            using type = decltype(x);
+            return std::min(std::max(type(min), x), type(max));
+        };
+    }
+
+    template <class Self, class F>
+    static auto reflect(Self& self, F f)
+    {
+        return pack(f(self.max_val, "max"), f(self.min_val, "min"));
+    }
+};
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/op/common.hpp

+#ifndef MIGRAPHX_GUARD_OPERATORS_COMMON_HPP
+#define MIGRAPHX_GUARD_OPERATORS_COMMON_HPP
+
+#include <array>
+#include <migraphx/operation.hpp>
+#include <migraphx/check_shapes.hpp>
+#include <migraphx/stringutils.hpp>
+#include <migraphx/streamutils.hpp>
+#include <migraphx/literal.hpp>
+#include <migraphx/shape_for_each.hpp>
+#include <migraphx/config.hpp>
+#include <cmath>
+#include <utility>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+namespace op {
+
+enum padding_mode_t
+{
+    default_, // NOLINT
+    same,
+    valid
+};
+
+// indicate rnn computation direction
+enum class rnn_direction
+{
+    forward,
+    reverse,
+    bidirectional,
+};
+
+std::ostream& operator<<(std::ostream& os, rnn_direction v);
+
+} // namespace op
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif