Serialization to and from vector<int8_t> and vector<uint8_t> (#2301)

* [SERIALIZATION]	- vectorstream can now be used with vector<int8_t> and vector<uint8_t>

* [SERIALIZATION]	- update proxy_serialize and proxy_deserialize to work with vector<int8_t> and vector<uint8_t>

* [SERIALIZATION]	- updated vectorstream tests

* [SERIALIZATION]	- updated serialize tests. check you can go to and from any of vector<char>, vector<int8_t> and vector<uint8_t>

* [SERIALIZATION]	- updated matrix tests. check you can go to and from any of vector<char>, vector<int8_t> and vector<uint8_t>

* [SERIALIZATION]	- updated dnn tests. check you can go to and from any of vector<char>, vector<int8_t> and vector<uint8_t>

* [SERIALIZATION] improved and possibly safer

* [SERIALIZATION] use placement new. best of all worlds i think. we have least object overhead. but code looks a tad uglier. oh well, user doesn't have to care

* [SERIALIZATION] i hope this is easier on the eyes.
Co-authored-by: pf <pf@pf-ubuntu-dev>

dlib/serialize.h

@@ -2269,6 +2269,20 @@ namespace dlib
         {
         }
         
+        explicit proxy_serialize (
+            std::vector<int8_t>& buf
+        ) : fout_optional_owning_ptr(new vectorstream(buf)),
+            fout(*fout_optional_owning_ptr)
+        {
+        }
+        
+        explicit proxy_serialize (
+            std::vector<uint8_t>& buf
+        ) : fout_optional_owning_ptr(new vectorstream(buf)),
+            fout(*fout_optional_owning_ptr)
+        {
+        }
+        
         explicit proxy_serialize (
             std::ostream& ss
         ) : fout_optional_owning_ptr(nullptr),
@@ -2309,6 +2323,22 @@ namespace dlib
             init();
         }
         
+        explicit proxy_deserialize (
+            std::vector<int8_t>& buf
+        ) : fin_optional_owning_ptr(new vectorstream(buf)),
+            fin(*fin_optional_owning_ptr)   
+        {
+            init();
+        }
+         
+        explicit proxy_deserialize (
+            std::vector<uint8_t>& buf
+        ) : fin_optional_owning_ptr(new vectorstream(buf)),
+            fin(*fin_optional_owning_ptr)   
+        {
+            init();
+        }
+        
         explicit proxy_deserialize (
             std::istream& ss
         ) : fin_optional_owning_ptr(nullptr),
@@ -2417,12 +2447,20 @@ namespace dlib
     { return proxy_serialize(ss); }
     inline proxy_serialize serialize(std::vector<char>& buf)
     { return proxy_serialize(buf); }
+    inline proxy_serialize serialize(std::vector<int8_t>& buf)
+    { return proxy_serialize(buf); }
+    inline proxy_serialize serialize(std::vector<uint8_t>& buf)
+    { return proxy_serialize(buf); }
     inline proxy_deserialize deserialize(const std::string& filename)
     { return proxy_deserialize(filename); }
     inline proxy_deserialize deserialize(std::istream& ss)
     { return proxy_deserialize(ss); }
     inline proxy_deserialize deserialize(std::vector<char>& buf)
     { return proxy_deserialize(buf); }
+    inline proxy_deserialize deserialize(std::vector<int8_t>& buf)
+    { return proxy_deserialize(buf); }
+    inline proxy_deserialize deserialize(std::vector<uint8_t>& buf)
+    { return proxy_deserialize(buf); }
 
 // ----------------------------------------------------------------------------------------
 

dlib/test/dnn.cpp

@@ -3753,6 +3753,11 @@ namespace
         const std::string serialized = out.str();
         std::istringstream in(serialized);
         dlib::deserialize(net2, in);
+        
+        std::vector<char> buf1;
+        dlib::serialize(buf1) << net;
+        std::vector<uint8_t> buf2(buf1.begin(), buf1.end());
+        dlib::deserialize(buf2) >> net2;
     }
 
 // ----------------------------------------------------------------------------------------

dlib/test/matrix.cpp

@@ -314,6 +314,9 @@ namespace
             serialize(c2,sout);
             serialize(d1,sout);
             serialize(d2,sout);
+            
+            std::vector<int8_t> buf1;
+            dlib::serialize(buf1) << a1 << a2 << b1 << b2 << c1 << c2 << d1 << d2;
 
             DLIB_TEST(a1 == orig1);
             DLIB_TEST(a2 == orig2);
@@ -361,8 +364,27 @@ namespace
             DLIB_TEST(c2 == orig2);
             DLIB_TEST(d1 == orig1);
             DLIB_TEST(d2 == orig2);
-
-
+            
+            set_all_elements(a1,99);
+            set_all_elements(a2,99);
+            set_all_elements(b1,99);
+            set_all_elements(b2,99);
+            set_all_elements(c1,99);
+            set_all_elements(c2,99);
+            set_all_elements(d1,99);
+            set_all_elements(d2,99);
+            
+            std::vector<uint8_t> buf2(buf1.begin(), buf1.end());
+            dlib::deserialize(buf2) >> a1 >> a2 >> b1 >> b2 >> c1 >> c2 >> d1 >> d2;
+            
+            DLIB_TEST(a1 == orig1);
+            DLIB_TEST(a2 == orig2);
+            DLIB_TEST(b1 == orig1);
+            DLIB_TEST(b2 == orig2);
+            DLIB_TEST(c1 == orig1);
+            DLIB_TEST(c2 == orig2);
+            DLIB_TEST(d1 == orig1);
+            DLIB_TEST(d2 == orig2);
         }
 
         {

dlib/test/serialize.cpp

@@ -1180,6 +1180,69 @@ namespace
             DLIB_TEST(pointers_values_equal(uptr1, uptr3));
             DLIB_TEST(pointers_values_equal(uptr2, uptr4));
         }
+        
+        {
+            std::vector<int8_t> buf;
+            dlib::serialize(buf) << t1 << t2 << v1 << uptr1 << uptr2;
+            dlib::deserialize(buf) >> t3 >> t4 >> v2 >> uptr3 >> uptr4;
+
+            DLIB_TEST(t1 == t3);
+            DLIB_TEST(t2 == t4);
+            DLIB_TEST(v1 == v2);
+            DLIB_TEST(pointers_values_equal(uptr1, uptr3));
+            DLIB_TEST(pointers_values_equal(uptr2, uptr4));
+        }
+        
+        {
+            std::vector<uint8_t> buf;
+            dlib::serialize(buf) << t1 << t2 << v1 << uptr1 << uptr2;
+            dlib::deserialize(buf) >> t3 >> t4 >> v2 >> uptr3 >> uptr4;
+
+            DLIB_TEST(t1 == t3);
+            DLIB_TEST(t2 == t4);
+            DLIB_TEST(v1 == v2);
+            DLIB_TEST(pointers_values_equal(uptr1, uptr3));
+            DLIB_TEST(pointers_values_equal(uptr2, uptr4));
+        }
+        
+        {
+            std::vector<char> buf1;
+            dlib::serialize(buf1) << t1 << t2 << v1 << uptr1 << uptr2;
+            std::vector<int8_t> buf2(buf1.begin(), buf1.end());
+            dlib::deserialize(buf2) >> t3 >> t4 >> v2 >> uptr3 >> uptr4;
+
+            DLIB_TEST(t1 == t3);
+            DLIB_TEST(t2 == t4);
+            DLIB_TEST(v1 == v2);
+            DLIB_TEST(pointers_values_equal(uptr1, uptr3));
+            DLIB_TEST(pointers_values_equal(uptr2, uptr4));
+        }
+        
+        {
+            std::vector<char> buf1;
+            dlib::serialize(buf1) << t1 << t2 << v1 << uptr1 << uptr2;
+            std::vector<uint8_t> buf2(buf1.begin(), buf1.end());
+            dlib::deserialize(buf2) >> t3 >> t4 >> v2 >> uptr3 >> uptr4;
+
+            DLIB_TEST(t1 == t3);
+            DLIB_TEST(t2 == t4);
+            DLIB_TEST(v1 == v2);
+            DLIB_TEST(pointers_values_equal(uptr1, uptr3));
+            DLIB_TEST(pointers_values_equal(uptr2, uptr4));
+        }
+        
+         {
+            std::vector<int8_t> buf1;
+            dlib::serialize(buf1) << t1 << t2 << v1 << uptr1 << uptr2;
+            std::vector<uint8_t> buf2(buf1.begin(), buf1.end());
+            dlib::deserialize(buf2) >> t3 >> t4 >> v2 >> uptr3 >> uptr4;
+
+            DLIB_TEST(t1 == t3);
+            DLIB_TEST(t2 == t4);
+            DLIB_TEST(v1 == v2);
+            DLIB_TEST(pointers_values_equal(uptr1, uptr3));
+            DLIB_TEST(pointers_values_equal(uptr2, uptr4));
+        }
     }
 
 // ----------------------------------------------------------------------------------------

dlib/test/vectorstream.cpp

@@ -22,8 +22,8 @@ namespace
 
     logger dlog("test.vectorstream");
           
-    template <typename stream>
-    void test1_variant(std::vector<char>& buf, stream& s)
+    template <typename CharType, typename stream>
+    void test1_variant(std::vector<CharType>& buf, stream& s)
     {
         for (int i = -1000; i <= 1000; ++i)
         {
@@ -37,7 +37,7 @@ namespace
         for (unsigned long i = 0; i < buf.size(); ++i)
         {
             char ch = cnt;
-            DLIB_TEST(buf[i] == ch);
+            DLIB_TEST((char)buf[i] == ch);
             ++cnt;
         }
 
@@ -175,7 +175,33 @@ namespace
             dlib::vectorstream s1(buf);
             std::iostream& s2 = s1;
             test1_variant(buf, s2);
-        }        
+        }    
+        
+        {
+            std::vector<int8_t> buf;
+            vectorstream s1(buf);
+            test1_variant(buf, s1);
+        }
+        
+        {
+            vector<int8_t> buf;
+            dlib::vectorstream s1(buf);
+            std::iostream& s2 = s1;
+            test1_variant(buf, s2);
+        }   
+        
+        {
+            std::vector<uint8_t> buf;
+            vectorstream s1(buf);
+            test1_variant(buf, s1);
+        }
+        
+        {
+            vector<uint8_t> buf;
+            dlib::vectorstream s1(buf);
+            std::iostream& s2 = s1;
+            test1_variant(buf, s2);
+        }   
     }
 
 // ----------------------------------------------------------------------------------------

dlib/vectorstream/vectorstream.h

@@ -24,15 +24,16 @@ namespace dlib
 {
     class vectorstream : public std::iostream
     {
+        template<typename CharType>
         class vector_streambuf : public std::streambuf
         {
-            typedef std::vector<char>::size_type size_type;
-            size_type read_pos; // buffer[read_pos] == next byte to read from buffer
+            typedef typename std::vector<CharType>::size_type size_type;
+            size_type read_pos = 0; // buffer[read_pos] == next byte to read from buffer
         public:
-            std::vector<char>& buffer;
+            std::vector<CharType>& buffer;
 
             vector_streambuf(
-                std::vector<char>& buffer_
+                std::vector<CharType>& buffer_
             ) :
                 read_pos(0),
                 buffer(buffer_) 
@@ -142,16 +143,44 @@ namespace dlib
 
         vectorstream (
             std::vector<char>& buffer
-        ) :
-            std::iostream(&buf),
-            buf(buffer)
-        {}
+        ) : std::iostream(0),
+            buf1(buffer),
+            buf2(dummy2),
+            buf3(dummy3)
+        {
+            rdbuf(&buf1);
+        }
+        
+        vectorstream (
+            std::vector<int8_t>& buffer
+        ) : std::iostream(0),
+            buf1(dummy1),
+            buf2(buffer),
+            buf3(dummy3)
+        {
+            rdbuf(&buf2);
+        }
+        
+        vectorstream (
+            std::vector<uint8_t>& buffer
+        ) : std::iostream(0),
+            buf1(dummy1),
+            buf2(dummy2),
+            buf3(buffer)
+        {
+            rdbuf(&buf3);
+        }
             
         vectorstream(const vectorstream& ori) = delete;
         vectorstream(vectorstream&& item) = delete;
-            
+                
     private:
-        vector_streambuf buf;
+        std::vector<char>           dummy1;
+        std::vector<int8_t>         dummy2;
+        std::vector<uint8_t>        dummy3;
+        vector_streambuf<char>      buf1;
+        vector_streambuf<int8_t>    buf2;
+        vector_streambuf<uint8_t>   buf3;
     };
 }