Fix convert operation for NaNs (#1840)

* Fix convert for the NaNs

* NaNs can't be compared, use std::isnan()

* formatting

* formatting

* formatting

* add extra tests

src/include/migraphx/op/convert.hpp

@@ -66,7 +66,17 @@ struct convert : unary<convert>
         auto type = target_type;
         return [type](auto x) {
             auto y = x;
-            shape::visit(type, [&](auto as) { y = std::min(std::max(as(x), as.min()), as.max()); });
+            shape::visit(type, [&](auto as) {
+                // clamping value between target_type's max and min doesn't work for NaNs,
+                if(std::isnan(x))
+                {
+                    y = as.nan();
+                }
+                else
+                {
+                    y = std::min(std::max(as(x), as.min()), as.max());
+                }
+            });
             return y;
         };
     }

src/include/migraphx/shape.hpp

@@ -299,6 +299,8 @@ struct shape
 
         type min() const { return std::numeric_limits<type>::lowest(); }
 
+        type nan() const { return std::numeric_limits<type>::quiet_NaN(); }
+
         template <class U>
         type operator()(U u) const
         {

test/ref_ops_test.cpp

@@ -1849,6 +1849,80 @@ TEST_CASE(cosh_dyn_test)
     EXPECT(migraphx::verify_range(results_vector, gold));
 }
 
+TEST_CASE(convert_nan_upcast_test)
+{
+    migraphx::program p;
+    auto* mm = p.get_main_module();
+    migraphx::shape s{migraphx::shape::half_type, {2, 2}};
+    std::vector<migraphx::half> data(4, std::numeric_limits<migraphx::half>::quiet_NaN());
+    auto l = mm->add_literal(migraphx::literal{s, data});
+    mm->add_instruction(
+        migraphx::make_op("convert", {{"target_type", migraphx::shape::float_type}}), l);
+    p.compile(migraphx::make_target("ref"));
+    auto result = p.eval({}).back();
+    std::vector<float> results_vector(4, -1);
+    result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
+    EXPECT(std::all_of(
+        results_vector.begin(), results_vector.end(), [](const auto& x) { return std::isnan(x); }));
+}
+
+TEST_CASE(convert_nan_downcast_test)
+{
+    migraphx::program p;
+    auto* mm = p.get_main_module();
+    migraphx::shape s{migraphx::shape::double_type, {2, 2}};
+    std::vector<double> data(4, std::numeric_limits<double>::quiet_NaN());
+    auto l = mm->add_literal(migraphx::literal{s, data});
+    mm->add_instruction(
+        migraphx::make_op("convert", {{"target_type", migraphx::shape::float_type}}), l);
+    p.compile(migraphx::make_target("ref"));
+    auto result = p.eval({}).back();
+    std::vector<float> results_vector(4, -1);
+    result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
+    EXPECT(std::all_of(
+        results_vector.begin(), results_vector.end(), [](const auto& x) { return std::isnan(x); }));
+}
+
+TEST_CASE(convert_nan_double_convert_test)
+{
+    migraphx::program p;
+    auto* mm = p.get_main_module();
+    migraphx::shape s{migraphx::shape::double_type, {2, 2}};
+    std::vector<double> data(4, std::numeric_limits<double>::quiet_NaN());
+    auto l   = mm->add_literal(migraphx::literal{s, data});
+    auto f_l = mm->add_instruction(
+        migraphx::make_op("convert", {{"target_type", migraphx::shape::float_type}}), l);
+    mm->add_instruction(migraphx::make_op("convert", {{"target_type", migraphx::shape::half_type}}),
+                        f_l);
+    p.compile(migraphx::make_target("ref"));
+    auto result = p.eval({}).back();
+    std::vector<migraphx::half> results_vector(4);
+    result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
+    EXPECT(std::all_of(
+        results_vector.begin(), results_vector.end(), [](const auto& x) { return std::isnan(x); }));
+}
+
+TEST_CASE(convert_nan_convert_updown_test)
+{
+    migraphx::program p;
+    auto* mm = p.get_main_module();
+    migraphx::shape s{migraphx::shape::float_type, {2, 2}};
+    std::vector<float> data(4, std::numeric_limits<float>::quiet_NaN());
+    auto l   = mm->add_literal(migraphx::literal{s, data});
+    auto f_l = mm->add_instruction(
+        migraphx::make_op("convert", {{"target_type", migraphx::shape::float_type}}), l);
+    auto h_l = mm->add_instruction(
+        migraphx::make_op("convert", {{"target_type", migraphx::shape::half_type}}), f_l);
+    mm->add_instruction(
+        migraphx::make_op("convert", {{"target_type", migraphx::shape::float_type}}), h_l);
+    p.compile(migraphx::make_target("ref"));
+    auto result = p.eval({}).back();
+    std::vector<float> results_vector(4);
+    result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
+    EXPECT(std::all_of(
+        results_vector.begin(), results_vector.end(), [](const auto& x) { return std::isnan(x); }));
+}
+
 TEST_CASE(deconv_1d_test)
 {
     migraphx::shape s{migraphx::shape::float_type, {1, 1, 3}};