Merge branch 'develop' of https://github.com/ROCmSoftwarePlatform/AMDMIGraphX into layernorm_eps

src/targets/gpu/pack_int8_args.cpp

@@ -154,7 +154,7 @@ void pack_int8_args::apply(module& m) const
 
             bool transa = inputs[0]->get_shape().transposed();
             bool transb = inputs[1]->get_shape().transposed();
-            if(!transb)
+            if(not transb)
             {
                 auto packed_b = m.insert_instruction(
                     ins, make_op("hip::allocate", {{"shape", to_value(inputs[1]->get_shape())}}));

src/targets/gpu/target.cpp

@@ -42,6 +42,7 @@
 #include <migraphx/register_target.hpp>
 #include <migraphx/replace_allocate.hpp>
 #include <migraphx/rewrite_batchnorm.hpp>
+#include <migraphx/rewrite_gelu.hpp>
 #include <migraphx/rewrite_pooling.hpp>
 #include <migraphx/rewrite_quantization.hpp>
 #include <migraphx/rewrite_rnn.hpp>
@@ -116,6 +117,8 @@ std::vector<pass> target::get_passes(migraphx::context& gctx, const compile_opti
         inline_module{},
         rewrite_pooling{},
         dead_code_elimination{},
+        rewrite_gelu{},
+        dead_code_elimination{},
         eliminate_common_subexpression{},
         dead_code_elimination{},
         simplify_algebra{},

src/tf/parse_conv.cpp

@@ -100,7 +100,7 @@ struct parse_conv : op_parser<parse_conv>
                 {
                     MIGRAPHX_THROW("padding should have 4 values");
                 }
-                if(padding[0] != padding[2] || padding[1] != padding[3])
+                if(padding[0] != padding[2] or padding[1] != padding[3])
                 {
                     MIGRAPHX_THROW("migraphx does not support asymetric padding");
                 }

src/tf/parse_depthwiseconv.cpp

@@ -90,7 +90,7 @@ struct parse_depthwiseconv : op_parser<parse_depthwiseconv>
                 calculate_padding(0, pads, input_dims[2], op.stride[0], op.dilation[0], weight_h);
                 calculate_padding(1, pads, input_dims[3], op.stride[1], op.dilation[1], weight_w);
 
-                if(pads[0] != pads[2] || pads[1] != pads[3])
+                if(pads[0] != pads[2] or pads[1] != pads[3])
                 {
                     std::vector<int64_t> padding = {0, 0, pads[0], pads[1], 0, 0, pads[2], pads[3]};
                     l0 = info.add_instruction(migraphx::make_op("pad", {{"pads", padding}}), l0);

src/tf/parse_pooling.cpp

@@ -42,7 +42,7 @@ struct parse_pooling : op_parser<parse_pooling>
                           tf_parser::node_info info,
                           std::vector<instruction_ref> args) const
     {
-        if(!starts_with(opd.tf_name, "Max") && !starts_with(opd.tf_name, "Av"))
+        if(not starts_with(opd.tf_name, "Max") and not starts_with(opd.tf_name, "Av"))
         {
             MIGRAPHX_THROW("tf pooling mode must be Max or Average");
         }

src/tf/tf_parser.cpp

@@ -371,7 +371,7 @@ void tf_parser::parse_node(const std::string& name)
         {
             result = ops[node.op()](*this, {get_attributes(node), node.op(), mm}, args);
         }
-        assert(!result.empty());
+        assert(not result.empty());
         // First output has no ":" delimiter
         instructions[name] = result.front();
         for(size_t i = 1; i < result.size(); i++)
@@ -458,7 +458,7 @@ literal tf_parser::parse_tensor(const tensorflow::TensorProto& t) const
 {
     std::vector<size_t> dims = parse_dims(t.tensor_shape());
     size_t shape_size = std::accumulate(dims.begin(), dims.end(), 1, std::multiplies<size_t>());
-    if(!t.tensor_content().empty()) // has raw data
+    if(not t.tensor_content().empty()) // has raw data
     {
         const std::string& s = t.tensor_content();
         switch(t.dtype())

src/tmp_dir.cpp

@@ -78,7 +78,7 @@ void tmp_dir::execute(const std::string& exe, const std::string& args) const
 
 tmp_dir::~tmp_dir()
 {
-    if(!enabled(MIGRAPHX_DEBUG_SAVE_TEMP_DIR{}))
+    if(not enabled(MIGRAPHX_DEBUG_SAVE_TEMP_DIR{}))
     {
         fs::remove_all(this->path);
     }

src/value.cpp

@@ -400,7 +400,7 @@ std::pair<value*, bool> value::insert(const value& v)
 {
     if(v.key.empty())
     {
-        if(!x)
+        if(not x)
             x = std::make_shared<array_value_holder>();
         get_array_impl(x).push_back(v);
         assert(this->if_array());
@@ -408,7 +408,7 @@ std::pair<value*, bool> value::insert(const value& v)
     }
     else
     {
-        if(!x)
+        if(not x)
             x = std::make_shared<object_value_holder>();
         auto p = x->if_object()->emplace(v.key, get_array_impl(x).size());
         if(p.second)
@@ -420,7 +420,7 @@ std::pair<value*, bool> value::insert(const value& v)
 value* value::insert(const value* pos, const value& v)
 {
     assert(v.key.empty());
-    if(!x)
+    if(not x)
         x = std::make_shared<array_value_holder>();
     auto&& a = get_array_impl(x);
     auto it  = a.insert(a.begin() + (pos - begin()), v);
@@ -466,7 +466,7 @@ bool compare(const value& x, const value& y, F f)
 
 value::type_t value::get_type() const
 {
-    if(!x)
+    if(not x)
         return null_type;
     return x->get_type();
 }

test/api/test_custom_op_gpu.cpp

@@ -55,7 +55,7 @@ struct simple_custom_op final : migraphx::experimental_custom_op_base
 
     virtual migraphx::shape compute_shape(migraphx::shapes inputs) const override
     {
-        if(!inputs[0].standard())
+        if(not inputs[0].standard())
         {
             throw std::runtime_error("first arg must be standard shaped");
         }

test/check_shapes_test.cpp

@@ -49,6 +49,6 @@ bool create_shapes(bool dynamic_allowed)
 
 TEST_CASE(allow_dynamic_shape) { EXPECT(create_shapes(true)); }
 
-TEST_CASE(fail_dynamic_shape) { EXPECT(!create_shapes(false)); }
+TEST_CASE(fail_dynamic_shape) { EXPECT(not create_shapes(false)); }
 
 int main(int argc, const char* argv[]) { test::run(argc, argv); }

test/eval_test.cpp

@@ -187,7 +187,7 @@ TEST_CASE(print_test)
     std::stringstream ss;
     ss << p;
     std::string s = ss.str();
-    EXPECT(!s.empty());
+    EXPECT(not s.empty());
 }
 
 TEST_CASE(param_test)

test/fpga/get_target_assignments.cpp

@@ -47,7 +47,7 @@ TEST_CASE(is_supported)
 {
     auto p       = create_program();
     auto targets = migraphx::get_targets();
-    EXPECT(!targets.empty());
+    EXPECT(not targets.empty());
     auto t = migraphx::make_target("fpga");
 
     const auto assignments = p.get_target_assignments({t});

test/include/basic_ops.hpp

@@ -112,12 +112,12 @@ struct mod_pass_op
     migraphx::shape compute_shape(std::vector<migraphx::shape> inputs,
                                   std::vector<migraphx::module_ref> mods) const
     {
-        if(!mods.empty())
+        if(not mods.empty())
         {
             auto out_shapes = mods[0]->get_output_shapes();
             return out_shapes[0];
         }
-        if(!inputs.empty())
+        if(not inputs.empty())
         {
             return inputs.front();
         }

test/include/test.hpp

@@ -345,7 +345,7 @@ inline std::ostream& operator<<(std::ostream& os, const color& c)
 template <class T, class F>
 void failed(T x, const char* msg, const char* func, const char* file, int line, F f)
 {
-    if(!bool(x.value()))
+    if(not bool(x.value()))
     {
         std::cout << func << std::endl;
         std::cout << file << ":" << line << ":" << std::endl;

test/literal_test.cpp

@@ -39,8 +39,8 @@ TEST_CASE(literal_test)
     migraphx::literal l2 = l1; // NOLINT
     EXPECT(l1 == l2);
     EXPECT(l1.at<int>(0) == 1);
-    EXPECT(!l1.empty());
-    EXPECT(!l2.empty());
+    EXPECT(not l1.empty());
+    EXPECT(not l2.empty());
 
     migraphx::literal l3{};
     migraphx::literal l4{};

test/ref_ops_test.cpp

@@ -3988,7 +3988,8 @@ TEST_CASE(not_test)
         std::vector<char> results_vector;
         result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
         std::vector<bool> gold(data.size());
-        std::transform(data.begin(), data.end(), gold.begin(), [](bool n) -> bool { return !n; });
+        std::transform(
+            data.begin(), data.end(), gold.begin(), [](bool n) -> bool { return not n; });
         EXPECT(migraphx::verify_range(results_vector, gold));
     }
 }

test/rewrite_gelu_test.cpp

+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include <migraphx/rewrite_gelu.hpp>
+#include <migraphx/dead_code_elimination.hpp>
+#include <migraphx/program.hpp>
+#include <migraphx/ref/target.hpp>
+#include <migraphx/op/convolution.hpp>
+#include <migraphx/op/reshape.hpp>
+#include <migraphx/instruction.hpp>
+#include <migraphx/generate.hpp>
+#include <migraphx/ranges.hpp>
+#include <test.hpp>
+#include <migraphx/make_op.hpp>
+#include <migraphx/common.hpp>
+
+#include <migraphx/serialize.hpp>
+
+#include <migraphx/verify.hpp>
+
+TEST_CASE(bias_gelu)
+{
+    migraphx::shape s1{migraphx::shape::half_type, {2, 4, 8}};
+    migraphx::shape s2{migraphx::shape::half_type};
+    migraphx::module m1;
+    {
+        auto a    = m1.add_parameter("a", s1);
+        auto b    = m1.add_parameter("b", s1);
+        auto add1 = m1.add_instruction(migraphx::make_op("add"), a, b);
+        auto l1   = m1.add_literal(migraphx::literal{s2, {1.4140625f}});
+        auto div  = add_common_op(m1, migraphx::make_op("div"), {add1, l1});
+        auto erf  = m1.add_instruction(migraphx::make_op("erf"), div);
+        auto l2   = m1.add_literal(migraphx::literal{s2, {1.0f}});
+        auto add2 = add_common_op(m1, migraphx::make_op("add"), {erf, l2});
+        auto mul  = m1.add_instruction(migraphx::make_op("mul"), add1, add2);
+        auto l3   = m1.add_literal(migraphx::literal{s2, {0.5f}});
+        mul       = add_common_op(m1, migraphx::make_op("mul"), {mul, l3});
+        m1.add_return({mul});
+    }
+    migraphx::rewrite_gelu pass;
+    pass.apply(m1);
+    migraphx::dead_code_elimination dce;
+    dce.apply(m1);
+
+    migraphx::module m2;
+    {
+        auto a   = m2.add_parameter("a", s1);
+        auto b   = m2.add_parameter("b", s1);
+        auto add = m2.add_instruction(migraphx::make_op("add"), a, b);
+        auto l1  = m2.add_literal(migraphx::literal{s2, {1.702f}});
+        auto mul = add_common_op(m2, migraphx::make_op("mul"), {add, l1});
+        auto sig = m2.add_instruction(migraphx::make_op("neg"), mul);
+        sig      = m2.add_instruction(migraphx::make_op("exp"), sig);
+        auto l2  = m2.add_literal(migraphx::literal{s2, {1.0f}});
+        sig      = add_common_op(m2, migraphx::make_op("add"), {sig, l2});
+        sig      = m2.add_instruction(migraphx::make_op("div"), add, sig);
+        m2.add_return({sig});
+    }
+
+    EXPECT(m1 == m2);
+}
+
+TEST_CASE(non_bias_gelu)
+{
+    migraphx::shape s1{migraphx::shape::half_type, {2, 4, 8}};
+    migraphx::shape s2{migraphx::shape::half_type};
+    migraphx::module m1;
+    {
+        auto a    = m1.add_parameter("a", s1);
+        auto b    = m1.add_parameter("b", s1);
+        auto sub  = m1.add_instruction(migraphx::make_op("sub"), a, b);
+        auto l1   = m1.add_literal(migraphx::literal{s2, {1.4140625f}});
+        auto div  = add_common_op(m1, migraphx::make_op("div"), {sub, l1});
+        auto erf  = m1.add_instruction(migraphx::make_op("erf"), div);
+        auto l2   = m1.add_literal(migraphx::literal{s2, {1.0f}});
+        auto add2 = add_common_op(m1, migraphx::make_op("add"), {erf, l2});
+        auto mul  = m1.add_instruction(migraphx::make_op("mul"), sub, add2);
+        auto l3   = m1.add_literal(migraphx::literal{s2, {0.5f}});
+        mul       = add_common_op(m1, migraphx::make_op("mul"), {mul, l3});
+        m1.add_return({mul});
+    }
+    migraphx::rewrite_gelu pass;
+    pass.apply(m1);
+    migraphx::dead_code_elimination dce;
+    dce.apply(m1);
+
+    migraphx::module m2;
+    {
+        auto a   = m2.add_parameter("a", s1);
+        auto b   = m2.add_parameter("b", s1);
+        auto sub = m2.add_instruction(migraphx::make_op("sub"), a, b);
+        auto l1  = m2.add_literal(migraphx::literal{s2, {1.702f}});
+        auto mul = add_common_op(m2, migraphx::make_op("mul"), {sub, l1});
+        auto sig = m2.add_instruction(migraphx::make_op("neg"), mul);
+        sig      = m2.add_instruction(migraphx::make_op("exp"), sig);
+        auto l2  = m2.add_literal(migraphx::literal{s2, {1.0f}});
+        sig      = add_common_op(m2, migraphx::make_op("add"), {sig, l2});
+        sig      = m2.add_instruction(migraphx::make_op("div"), sub, sig);
+        m2.add_return({sig});
+    }
+
+    EXPECT(m1 == m2);
+}
+
+int main(int argc, const char* argv[]) { test::run(argc, argv); }

test/shape_test.cpp

@@ -43,7 +43,7 @@ TEST_CASE(test_shape_assign)
     migraphx::shape s1{migraphx::shape::float_type, {100, 32, 8, 8}};
     migraphx::shape s2 = s1; // NOLINT
     EXPECT(s1 == s2);
-    EXPECT(!(s1 != s2));
+    EXPECT(not(s1 != s2));
 }
 
 TEST_CASE(test_shape_packed_default)
@@ -325,7 +325,7 @@ TEST_CASE(test_shape_default_copy)
     migraphx::shape s1{};
     migraphx::shape s2{};
     EXPECT(s1 == s2);
-    EXPECT(!(s1 != s2));
+    EXPECT(not(s1 != s2));
 }
 
 TEST_CASE(test_shape_normalize_standard1)

test/verify/run_verify.cpp

@@ -169,7 +169,7 @@ void run_verify::verify(const std::string& name, const migraphx::program& p) con
     for(const auto& tname : migraphx::get_targets())
     {
         // TODO(varunsh): once verify tests can run, remove fpga
-        if(tname == "ref" || tname == "fpga")
+        if(tname == "ref" or tname == "fpga")
             continue;
 
         // if tests disabled, skip running it

test/verify/test_add_gelu_half.cpp

+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "verify_program.hpp"
+#include <migraphx/program.hpp>
+#include <migraphx/generate.hpp>
+#include <migraphx/make_op.hpp>
+
+struct test_add_gelu_half : verify_program<test_add_gelu_half>
+{
+    migraphx::program create_program() const
+    {
+        migraphx::program p;
+        auto* mm = p.get_main_module();
+        std::vector<size_t> input_lens{1, 1, 5};
+        auto x     = mm->add_parameter("x", {migraphx::shape::half_type, input_lens});
+        auto y     = mm->add_parameter("y", {migraphx::shape::half_type, input_lens});
+        auto half  = mm->add_literal(migraphx::literal{{migraphx::shape::half_type}, {0.5f}});
+        auto one   = mm->add_literal(migraphx::literal{{migraphx::shape::half_type}, {1.0f}});
+        auto sqrt2 = mm->add_literal(migraphx::literal{{migraphx::shape::half_type}, {M_SQRT2}});
+        auto add   = mm->add_instruction(migraphx::make_op("add"), x, y);
+        auto half_mbcast = mm->add_instruction(
+            migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), half);
+        auto mul_half     = mm->add_instruction(migraphx::make_op("mul"), add, half_mbcast);
+        auto sqrt2_mbcast = mm->add_instruction(
+            migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), sqrt2);
+        auto div        = mm->add_instruction(migraphx::make_op("div"), add, sqrt2_mbcast);
+        auto erf        = mm->add_instruction(migraphx::make_op("erf"), div);
+        auto one_mbcast = mm->add_instruction(
+            migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), one);
+        auto add_one = mm->add_instruction(migraphx::make_op("add"), erf, one_mbcast);
+        mm->add_instruction(migraphx::make_op("mul"), mul_half, add_one);
+        return p;
+    }
+};