Convert value to json (#592)

* Add initial serialization

* Formatting

* Add unit tests

* Formatting

* Add tests for serialization

* Formatting

* Use or not and

* Add value test

* Formatting

* Add more tests

* Add shape serialization

* Formatting

* Add serializtion for literal and argument

* Formatting

* Serialize empty types

* Formatting

* Tidy fixes

* Formatting

* Fix tidy issues

* Formatting

* Reformat value type macro

* Formatting

* Handle enum types

* Formatting

* Add float_equal

* Fix tidy issue

* Use declval for better sfinae

* Formatting

* backup code change

* Fix maro name

* Add more test cases to improve coverage

* Formatting

* Add more tests

* Formatting

* Fix assertion

* code backup

* clang format

* Fix bug with keyless assignment

* Formatting

* add implementation of converting json to migraphx::value

* clang format

* code backup

* clang format

* code backup

* clang format

* code backup

* code refactor

* add unit tests

* separate json conversion from serialization

* clang format

* add the missing files

* refine code

* clang format

* fix cppcheck error

* clang format

* fix review comments

* clang format

* code backup

* fix cppcheck error

* refine an implementation

* clang format

* fix review comments

* clang format

* fix review comments and add more unit tests

* clang format

* code backup to try

* fix review comments and add more unit tests

* clang format

* some refinement

* clang format

* fix review comments

* remove unnecessary code

* add assignment of null
Co-authored-by: Paul <pfultz2@yahoo.com>
Co-authored-by: mvermeulen <5479696+mvermeulen@users.noreply.github.com>

CMakeLists.txt

@@ -19,6 +19,7 @@ endif()
 
 project(migraphx)
 find_package(ROCM REQUIRED)
+find_package(nlohmann_json 3.8.0 REQUIRED)
 
 include(ROCMSetupVersion)
 

requirements.txt

@@ -2,6 +2,7 @@ google/protobuf@v3.11.0 -DCMAKE_POSITION_INDEPENDENT_CODE=On -X subdir -Dprotobu
 RadeonOpenCompute/rocm-cmake@b29ff83 --build
 ROCmSoftwarePlatform/rocBLAS@abd98a2b48b29326ebaef471630786a548622c06
 ROCmSoftwarePlatform/MIOpen@2.4.0
+nlohmann/json@v3.8.0
 blaze,https://bitbucket.org/blaze-lib/blaze/get/f0755dea0e03.tar.gz -X header -DHEADER_DIR=blaze
 half,https://github.com/pfultz2/half/archive/1.12.0.tar.gz -X header -H sha256:0a08660b68abb176ebc2a0cdf8de46e3182a7f46c66443bb80dbfaaec98cf969
 pybind/pybind11@v2.2.4 -DPYBIND11_TEST=Off --build

src/CMakeLists.txt

@@ -29,6 +29,7 @@ add_library(migraphx
     simplify_algebra.cpp
     simplify_reshapes.cpp
     value.cpp
+    json.cpp
     opt/memory_coloring.cpp
     opt/memory_coloring_impl.cpp
 )

src/include/migraphx/json.hpp

+#ifndef MIGRAPHX_GUARD_RTGLIB_JSON_HPP
+#define MIGRAPHX_GUARD_RTGLIB_JSON_HPP
+
+#include <migraphx/config.hpp>
+#include <migraphx/value.hpp>
+#include <string>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+std::string to_json_string(const value& val);
+value from_json_string(const std::string& str);
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+#endif

src/include/migraphx/value.hpp

@@ -134,6 +134,7 @@ struct value
     value(const std::string& pkey, const std::vector<value>& v, bool array_on_empty = true);
     value(const std::string& pkey, const std::unordered_map<std::string, value>& m);
     value(const std::string& pkey, std::nullptr_t);
+    value(std::nullptr_t);
 
     value(const char* i);
 
@@ -176,6 +177,8 @@ struct value
         return *this = pick<T>{rhs}; // NOLINT
     }
 
+    value& operator=(std::nullptr_t);
+
     bool is_array() const;
     const std::vector<value>& get_array() const;
     const std::vector<value>* if_array() const;

src/json.cpp

+#include <migraphx/serialize.hpp>
+#include <migraphx/argument.hpp>
+#include <migraphx/literal.hpp>
+#include <nlohmann/json.hpp>
+#include <migraphx/json.hpp>
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+using json = nlohmann::json;
+
+void value_to_json(const value& val, json& j);
+migraphx::value value_from_json(const json& j);
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx
+
+namespace nlohmann {
+template <>
+struct adl_serializer<migraphx::value>
+{
+    static void to_json(json& j, const migraphx::value& val) { migraphx::value_to_json(val, j); }
+
+    static void from_json(const json& j, migraphx::value& val)
+    {
+        val = migraphx::value_from_json(j);
+    }
+};
+} // namespace nlohmann
+
+namespace migraphx {
+inline namespace MIGRAPHX_INLINE_NS {
+
+using json = nlohmann::json;
+
+template <class T>
+void value_to_json(const T& x, json& j)
+{
+    j = x;
+}
+
+void value_to_json(const std::vector<value>& x, json& j)
+{
+    for(auto& v : x)
+    {
+        if(v.get_key().empty())
+        {
+            j.push_back(v);
+        }
+        else
+        {
+            j[v.get_key()] = v.without_key();
+        }
+    }
+}
+
+void value_to_json(std::nullptr_t&, json& j) { j = {}; }
+
+void value_to_json(const value& val, json& j)
+{
+    if(val.is_array())
+    {
+        j = json::array();
+    }
+
+    if(val.is_object())
+    {
+        j = json::object();
+    }
+
+    val.visit([&](auto v) { value_to_json(v, j); });
+}
+
+migraphx::value value_from_json(const json& j)
+{
+    migraphx::value val;
+    json::value_t type = j.type();
+    switch(type)
+    {
+    case json::value_t::null: val = nullptr; break;
+
+    case json::value_t::boolean: val = j.get<bool>(); break;
+
+    case json::value_t::number_float: val = j.get<double>(); break;
+
+    case json::value_t::number_integer: val = j.get<int64_t>(); break;
+
+    case json::value_t::number_unsigned: val = j.get<uint64_t>(); break;
+
+    case json::value_t::string: val = j.get<std::string>(); break;
+
+    case json::value_t::array:
+        val = migraphx::value::array{};
+        std::transform(j.begin(), j.end(), std::back_inserter(val), [&](const json& jj) {
+            return jj.get<value>();
+        });
+        break;
+
+    case json::value_t::object:
+        val = migraphx::value::object{};
+        for(const auto& item : j.items())
+        {
+            const auto& key = item.key();
+            const json& jv  = item.value();
+            val[key]        = jv.get<value>();
+        }
+        break;
+
+    case json::value_t::binary: MIGRAPHX_THROW("Convert JSON to Value: binary type not supported!");
+    case json::value_t::discarded:
+        MIGRAPHX_THROW("Convert JSON to Value: discarded type not supported!");
+    }
+
+    return val;
+}
+
+std::string to_json_string(const value& val)
+{
+    json j = val;
+    return j.dump();
+}
+
+migraphx::value from_json_string(const std::string& str)
+{
+    json j = json::parse(str);
+    return j.get<value>();
+}
+
+} // namespace MIGRAPHX_INLINE_NS
+} // namespace migraphx

src/value.cpp

@@ -131,6 +131,8 @@ value::value(const std::string& pkey, const std::unordered_map<std::string, valu
 
 value::value(const std::string& pkey, std::nullptr_t) : x(nullptr), key(pkey) {}
 
+value::value(std::nullptr_t) : x(nullptr) {}
+
 value::value(const std::string& pkey, const value& rhs)
     : x(rhs.x ? rhs.x->clone() : nullptr), key(pkey)
 {
@@ -159,6 +161,12 @@ value::value(const char* i) : value(std::string(i)) {}
     const cpp_type* value::if_##vt() const { return x ? x->if_##vt() : nullptr; }
 MIGRAPHX_VISIT_VALUE_TYPES(MIGRAPHX_VALUE_GENERATE_DEFINE_METHODS)
 
+value& value::operator=(std::nullptr_t)
+{
+    x = nullptr;
+    return *this;
+}
+
 bool value::is_array() const { return x ? x->get_type() == array_type : false; }
 const std::vector<value>& value::value::get_array() const
 {

test/json_test.cpp

+#include <migraphx/shape.hpp>
+#include <migraphx/argument.hpp>
+#include <migraphx/literal.hpp>
+#include <migraphx/serialize.hpp>
+#include <migraphx/functional.hpp>
+#include <migraphx/json.hpp>
+#include <test.hpp>
+
+TEST_CASE(null_value)
+{
+    migraphx::value v;
+    auto json_str = migraphx::to_json_string(v);
+    EXPECT(json_str == "null");
+}
+
+TEST_CASE(null_value_rev)
+{
+    std::string json_str = "null";
+    migraphx::value v    = migraphx::from_json_string(json_str);
+    migraphx::value ev;
+    EXPECT(v == ev);
+}
+
+TEST_CASE(null_array)
+{
+    migraphx::value v;
+    migraphx::value arr = {v, v};
+    auto json_str       = migraphx::to_json_string(arr);
+    EXPECT(json_str == "[null,null]");
+}
+
+TEST_CASE(null_array_rev)
+{
+    std::string json_str = "[null,null]";
+    migraphx::value v    = migraphx::from_json_string(json_str);
+    migraphx::value e;
+    migraphx::value ev = {e, e};
+    EXPECT(ev == v);
+}
+
+TEST_CASE(empty_object1)
+{
+    migraphx::value val = migraphx::from_json_string("{}");
+    EXPECT(val == migraphx::value::object{});
+    EXPECT(migraphx::to_json_string(migraphx::value::object{}) == "{}");
+}
+
+TEST_CASE(empty_array1)
+{
+    migraphx::value val = migraphx::from_json_string("[]");
+    EXPECT(val == migraphx::value::array{});
+    EXPECT(migraphx::to_json_string(migraphx::value::array{}) == "[]");
+}
+
+TEST_CASE(int_value)
+{
+    migraphx::value v    = -1;
+    std::string json_str = migraphx::to_json_string(v);
+    EXPECT(json_str == "-1");
+}
+
+TEST_CASE(int_value_rev)
+{
+    std::string json_str = "-1";
+    migraphx::value v    = migraphx::from_json_string(json_str);
+    migraphx::value ev   = -1;
+    EXPECT(v == ev);
+}
+
+TEST_CASE(unsigned_value)
+{
+    migraphx::value v    = 1;
+    std::string json_str = migraphx::to_json_string(v);
+    EXPECT(json_str == "1");
+}
+
+TEST_CASE(unsigned_value_rev)
+{
+    std::string json_str = "1";
+    migraphx::value v    = migraphx::from_json_string(json_str);
+    EXPECT(v.is_uint64());
+    EXPECT(v.get_uint64() == 1);
+}
+
+TEST_CASE(float_value)
+{
+    migraphx::value v    = 1.5;
+    std::string json_str = migraphx::to_json_string(v);
+    EXPECT(json_str == "1.5");
+}
+
+TEST_CASE(float_value_rev)
+{
+    std::string json_str = "1.5";
+    migraphx::value v    = migraphx::from_json_string(json_str);
+    migraphx::value ev   = 1.5;
+    EXPECT(v == ev);
+}
+
+TEST_CASE(array_value)
+{
+    migraphx::value v    = {1, 2};
+    std::string json_str = migraphx::to_json_string(v);
+    EXPECT(json_str == "[1,2]");
+}
+
+TEST_CASE(array_value_rev)
+{
+    std::string json_str = "[1,2]";
+    migraphx::value v    = migraphx::from_json_string(json_str);
+    EXPECT(v.is_array());
+    EXPECT(v.size() == 2);
+    EXPECT(v[0].get_uint64() == 1);
+    EXPECT(v[1].get_uint64() == 2);
+}
+
+TEST_CASE(object_value)
+{
+    migraphx::value v    = {{"a", 1.2}, {"b", true}};
+    std::string json_str = migraphx::to_json_string(v);
+    EXPECT(json_str == "{\"a\":1.2,\"b\":true}");
+}
+
+TEST_CASE(object_value_rev)
+{
+    std::string json_str = R"({"a":1.2,"b":true})";
+    migraphx::value v    = migraphx::from_json_string(json_str);
+    migraphx::value ev   = {{"a", 1.2}, {"b", true}};
+    EXPECT(v == ev);
+}
+
+TEST_CASE(null_object)
+{
+    migraphx::value v;
+    migraphx::value v1   = {{"a", v}};
+    std::string json_str = migraphx::to_json_string(v1);
+    EXPECT(json_str == "{\"a\":null}");
+}
+
+TEST_CASE(null_object_rev)
+{
+    std::string json_str = R"({"a":null})";
+    migraphx::value eo   = migraphx::from_json_string(json_str);
+    migraphx::value v;
+    migraphx::value ev = {{"a", v}};
+    EXPECT(eo == ev);
+}
+
+TEST_CASE(string_value)
+{
+    migraphx::value v    = "string_test";
+    std::string json_str = migraphx::to_json_string(v);
+    EXPECT(json_str == "\"string_test\"");
+}
+
+TEST_CASE(string_value_rev)
+{
+    std::string json_str = "\"string_test\"";
+    migraphx::value v    = migraphx::from_json_string(json_str);
+    migraphx::value ev   = "string_test";
+    EXPECT(v == ev);
+}
+
+TEST_CASE(array_of_objects)
+{
+    migraphx::value obj1 = {"key1", uint64_t{1}};
+    migraphx::value obj2 = {"key2", uint64_t{2}};
+    migraphx::value arr  = {obj1, obj2};
+    std::string json_str = migraphx::to_json_string(arr);
+    EXPECT(json_str == "{\"key1\":1,\"key2\":2}");
+}
+
+TEST_CASE(array_of_objects_rev)
+{
+    std::string json_str = R"({"key1":1,"key2":2})";
+    migraphx::value v    = migraphx::from_json_string(json_str);
+    migraphx::value obj1 = {"key1", uint64_t{1}};
+    migraphx::value obj2 = {"key2", uint64_t{2}};
+    migraphx::value arr  = {obj1, obj2};
+    EXPECT(arr == v);
+}
+
+TEST_CASE(object_of_array)
+{
+    migraphx::value obj1 = {"key1", 1};
+    migraphx::value obj2 = {"key2", 2};
+    migraphx::value obj;
+    obj["key"]           = {obj1, obj2};
+    std::string json_str = migraphx::to_json_string(obj);
+    EXPECT(json_str == "{\"key\":{\"key1\":1,\"key2\":2}}");
+}
+
+TEST_CASE(object_of_array_rev)
+{
+    std::string json_str = R"({"key":{"key1":1,"key2":2}})";
+    migraphx::value v    = migraphx::from_json_string(json_str);
+    migraphx::value obj1 = {"key1", uint64_t{1}};
+    migraphx::value obj2 = {"key2", uint64_t{2}};
+    migraphx::value obj;
+    obj["key"] = {obj1, obj2};
+    EXPECT(v == obj);
+}
+
+TEST_CASE(shape_value)
+{
+    migraphx::shape s{migraphx::shape::int32_type, {2, 3, 4, 5}};
+    migraphx::value val     = migraphx::to_value(s);
+    std::string json_str    = migraphx::to_json_string(val);
+    migraphx::value val_rev = migraphx::from_json_string(json_str);
+    migraphx::shape s_rev;
+    migraphx::from_value(val_rev, s_rev);
+
+    EXPECT(s == s_rev);
+}
+
+TEST_CASE(argument_value)
+{
+    migraphx::shape s{migraphx::shape::int32_type, {2, 3, 4, 5}};
+    std::vector<int> data(s.elements());
+    std::iota(data.begin(), data.end(), 1);
+    migraphx::argument argu = migraphx::argument(s, data.data());
+
+    migraphx::value val     = migraphx::to_value(argu);
+    std::string json_str    = migraphx::to_json_string(val);
+    migraphx::value val_rev = migraphx::from_json_string(json_str);
+    migraphx::argument argu_rev;
+    migraphx::from_value(val_rev, argu_rev);
+
+    EXPECT(argu == argu_rev);
+}
+
+TEST_CASE(literal_value)
+{
+    migraphx::shape s{migraphx::shape::int32_type, {2, 3, 4, 5}};
+    std::vector<int> data(s.elements());
+    std::iota(data.begin(), data.end(), 1);
+    migraphx::literal l = migraphx::literal(s, data);
+
+    migraphx::value val     = migraphx::to_value(l);
+    std::string json_str    = migraphx::to_json_string(val);
+    migraphx::value val_rev = migraphx::from_json_string(json_str);
+    migraphx::literal l_rev;
+    migraphx::from_value(val_rev, l_rev);
+
+    EXPECT(l == l_rev);
+}
+
+int main(int argc, const char* argv[]) { test::run(argc, argv); }