Code cleanup

dlib/dnn/cpu_dlib.cpp

@@ -164,7 +164,7 @@ namespace dlib
             }
         }
 
-        void batch_normalize_gradient (
+        void batch_normalize_gradient::operator() (
             const tensor& gradient_input,
             const tensor& means,
             const tensor& invstds,
@@ -192,7 +192,6 @@ namespace dlib
             const auto p_invstds = invstds.host();
             const auto p_means = means.host();
 
-            resizable_tensor dvars, dmeans;
             dvars.copy_size(invstds);
             dmeans.copy_size(means);
             dvars = 0;
@@ -343,7 +342,7 @@ namespace dlib
             }
         }
 
-        void batch_normalize_conv_gradient (
+        void batch_normalize_conv_gradient::operator() (
             const tensor& gradient_input,
             const tensor& means,
             const tensor& invstds,
@@ -371,7 +370,6 @@ namespace dlib
             const auto p_invstds = invstds.host();
             const auto p_means = means.host();
 
-            resizable_tensor dvars, dmeans;
             dvars.copy_size(invstds);
             dmeans.copy_size(means);
             dvars = 0;

dlib/dnn/cpu_dlib.h

@@ -49,7 +49,10 @@ namespace dlib
             const tensor& beta 
         );
 
-        void batch_normalize_gradient (
+        class batch_normalize_gradient
+        {
+        public:
+            void operator() (
                 const tensor& gradient_input,
                 const tensor& means,
                 const tensor& invstds,
@@ -59,6 +62,9 @@ namespace dlib
                 tensor& gamma_grad, 
                 tensor& beta_grad 
             );
+        private:
+            resizable_tensor dvars, dmeans;
+        };
 
         void batch_normalize_conv (
             resizable_tensor& dest,
@@ -69,7 +75,10 @@ namespace dlib
             const tensor& beta 
         );
 
-        void batch_normalize_conv_gradient (
+        class batch_normalize_conv_gradient
+        {
+        public:
+            void operator() (
                 const tensor& gradient_input,
                 const tensor& means,
                 const tensor& invstds,
@@ -79,6 +88,9 @@ namespace dlib
                 tensor& gamma_grad, 
                 tensor& beta_grad 
             );
+        private:
+            resizable_tensor dvars, dmeans;
+        };
 
     // -----------------------------------------------------------------------------------
 

dlib/dnn/cuda_dlib.h

@@ -18,14 +18,6 @@ namespace dlib
             tensor& dest,
             const tensor& src
         );
-        /*!
-            requires
-                - have_same_dimensions(dest,src) == true
-            ensures
-                - #dest == dest*src 
-                  That is, for all valid i:
-                    #dest.host()[i] == dest.host()[i]*src.host()[i]
-        !*/
 
     // -----------------------------------------------------------------------------------
 
@@ -35,11 +27,6 @@ namespace dlib
             const float A,
             const float B
         );
-        /*!
-            ensures
-                - have_same_dimensions(#dest,src) == true
-                - #dest == A*src + B
-        !*/
 
     // -----------------------------------------------------------------------------------
 
@@ -49,25 +36,6 @@ namespace dlib
             const tensor& A,
             const tensor& B
         );
-        /*!
-            requires
-                - if (A.num_samples() == 1) then
-                    - B.num_samples() == 1
-                - else
-                    - A.num_samples() == src.num_samples()
-                    - B.num_samples() == src.num_samples()
-                - A.nr() == B.nr() == src.nr()
-                - A.nc() == B.nc() == src.nc()
-                - A.k()  == B.k()  == src.k()
-            ensures
-                - have_same_dimensions(#dest,src) == true
-                - if (A.num_samples() == 1) then
-                    - #dest == A*src + B
-                      (done for each sample in src)
-                - else
-                    - for all valid i:
-                        - #dest.host()[i] == A.host()[i]*src.host()[i] + B.host()[i]  
-        !*/
 
     // -----------------------------------------------------------------------------------
 
@@ -79,27 +47,11 @@ namespace dlib
             const tensor& gamma, 
             const tensor& beta 
         );
-        /*!
-            requires
-                - src.num_samples() > 1
-                - gamma.num_samples() == 1
-                - beta.num_samples() == 1
-                - gamma.nr() == beta.nr() == src.nr()
-                - gamma.nc() == beta.nc() == src.nc()
-                - gamma.k()  == beta.k()  == src.k()
-            ensures
-                - have_same_dimensions(#dest, src) == true
-                - #means.num_samples() == 1
-                - #invstds.num_samples() == 1
-                - means.nr() == invstds.nr() == src.nr()
-                - means.nc() == invstds.nc() == src.nc()
-                - means.k()  == invstds.k()  == src.k()
-                - #src == the batch normalized version of src.
-                - #means == the mean values of the contents of src.
-                - #invstds == 1/(the standard deviation values of the contents of src).
-        !*/
-
-        void batch_normalize_gradient (
+
+        class batch_normalize_gradient
+        {
+        public:
+            void operator() (
                 const tensor& gradient_input,
                 const tensor& means,
                 const tensor& invstds,
@@ -109,28 +61,9 @@ namespace dlib
                 tensor& gamma_grad, 
                 tensor& beta_grad 
             );
-        /*!
-            requires
-                - invstds and means should be the output of a call to
-                  batch_normalize(dest,means,invstds,src,gamma,beta)
-                - have_same_dimensions(gradient_input, src) == true
-                - have_same_dimensions(src, src_grad) == true
-                - src.num_samples() > 1
-                - gamma.num_samples() == 1
-                - have_same_dimensions(gamma, gamma_grad) == true
-                - have_same_dimensions(gamma, beta_grad) == true
-                - gamma.nr() == src.nr()
-                - gamma.nc() == src.nc()
-                - gamma.k()  == src.k()
-                - have_same_dimensions(means, gamma) == true
-                - have_same_dimensions(invstds, gamma) == true
-            ensures
-                - Let f(src,gamma,beta) == dot(gradient_input, dest output of
-                  batch_normalize(dest,means,invstds,src,gamma,beta))
-                - Adds the gradient of f() with respect to src to #src_grad.
-                - Adds the gradient of f() with respect to gamma to #gamma_grad.
-                - Adds the gradient of f() with respect to beta to #beta_grad.
-        !*/
+        private:
+            resizable_tensor dvars, dmeans;
+        };
 
         void batch_normalize_conv (
             resizable_tensor& dest,
@@ -140,23 +73,11 @@ namespace dlib
             const tensor& gamma, 
             const tensor& beta 
         );
-        /*!
-            requires
-                - src.num_samples() > 1
-                - gamma.num_samples()==gamma.nr()==gamma.nc() == 1
-                - beta.num_samples() ==beta.nr() ==gamma.nc() == 1
-                - gamma.k()  == beta.k()  == src.k()
-            ensures
-                - have_same_dimensions(#dest, src) == true
-                - #means.num_samples()==means.nr()==means.nc() == 1
-                - #invstds.num_samples() ==invstds.nr() ==invstds.nc() == 1
-                - means.k()  == invstds.k()  == src.k()
-                - #src == the batch normalized version of src.
-                - #means == the mean values of the contents of src.
-                - #invstds == 1/(the standard deviation values of the contents of src).
-        !*/
-
-        void batch_normalize_conv_gradient (
+
+        class batch_normalize_conv_gradient
+        {
+        public:
+            void operator() (
                 const tensor& gradient_input,
                 const tensor& means,
                 const tensor& invstds,
@@ -166,26 +87,9 @@ namespace dlib
                 tensor& gamma_grad, 
                 tensor& beta_grad 
             );
-        /*!
-            requires
-                - invstds and means should be the output of a call to
-                  batch_normalize_conv(dest,means,invstds,src,gamma,beta)
-                - have_same_dimensions(gradient_input, src) == true
-                - have_same_dimensions(src, src_grad) == true
-                - src.num_samples() > 1
-                - gamma.num_samples()==gamma.nr()==gamma.nc() == 1
-                - have_same_dimensions(gamma, gamma_grad) == true
-                - have_same_dimensions(gamma, beta_grad) == true
-                - gamma.k()  == src.k()
-                - have_same_dimensions(means, gamma) == true
-                - have_same_dimensions(invstds, gamma) == true
-            ensures
-                - Let f(src,gamma,beta) == dot(gradient_input, dest output of
-                  batch_normalize_conv(dest,means,invstds,src,gamma,beta))
-                - Adds the gradient of f() with respect to src to #src_grad.
-                - Adds the gradient of f() with respect to gamma to #gamma_grad.
-                - Adds the gradient of f() with respect to beta to #beta_grad.
-        !*/
+        private:
+            resizable_tensor dvars, dmeans;
+        };
 
     // -----------------------------------------------------------------------------------
 
@@ -193,12 +97,6 @@ namespace dlib
             tensor& data,
             float thresh
         );
-        /*!
-            ensures
-                - Sets all elements of data to 1 or 0 depending on if they are above or
-                  below the given threshold.  Specifically, for all valid i:
-                    - #data.host()[i] == data.host()[i]>thresh ? 1 : 0
-        !*/
 
     // ------------------------------------------------------------------------------------
 

dlib/dnn/tensor_tools.cpp

@@ -4,8 +4,6 @@
 #define DLIB_TeNSOR_TOOLS_CPP_
 
 #include "tensor_tools.h"
-#include "cpu_dlib.h"
-#include "cuda_dlib.h"
 
 namespace dlib { namespace tt
 {
@@ -143,30 +141,6 @@ namespace dlib { namespace tt
 #endif
     }
 
-// ----------------------------------------------------------------------------------------
-
-    void batch_normalize_gradient (
-        const tensor& gradient_input,
-        const tensor& means,
-        const tensor& vars,
-        const tensor& src,
-        const tensor& gamma,
-        tensor& src_grad,
-        tensor& gamma_grad, 
-        tensor& beta_grad 
-    )
-    {
-#ifdef DLIB_USE_CUDA
-        /*
-        cuda::batch_normalize_gradient(gradient_input,means,vars,src,gamma,
-                                       src_grad,gamma_grad,beta_grad);
-        */
-#else
-        cpu::batch_normalize_gradient(gradient_input,means,vars,src,gamma,
-                                       src_grad,gamma_grad,beta_grad);
-#endif
-    }
-
 // ----------------------------------------------------------------------------------------
 
     void batch_normalize_conv (
@@ -185,30 +159,6 @@ namespace dlib { namespace tt
 #endif
     }
 
-// ----------------------------------------------------------------------------------------
-
-    void batch_normalize_conv_gradient (
-        const tensor& gradient_input,
-        const tensor& means,
-        const tensor& vars,
-        const tensor& src,
-        const tensor& gamma,
-        tensor& src_grad,
-        tensor& gamma_grad, 
-        tensor& beta_grad 
-    )
-    {
-#ifdef DLIB_USE_CUDA
-        /*
-        cuda::batch_normalize_conv_gradient(gradient_input,means,vars,src,gamma,
-                                       src_grad,gamma_grad,beta_grad);
-        */
-#else
-        cpu::batch_normalize_conv_gradient(gradient_input,means,vars,src,gamma,
-                                       src_grad,gamma_grad,beta_grad);
-#endif
-    }
-
 // ----------------------------------------------------------------------------------------
 
     void threshold (

dlib/dnn/tensor_tools.h

@@ -7,6 +7,8 @@
 #include "cudnn_dlibapi.h"
 #include "cublas_dlibapi.h"
 #include "curand_dlibapi.h"
+#include "cpu_dlib.h"
+#include "cuda_dlib.h"
 #include "../rand.h"
 
 namespace dlib { namespace tt
@@ -175,9 +177,10 @@ namespace dlib { namespace tt
             - #invstds == 1/(the standard deviation values of the contents of src).
     !*/
 
-// ----------------------------------------------------------------------------------------
-
-    void batch_normalize_gradient (
+    class batch_normalize_gradient
+    {
+    public:
+        void operator() (
             const tensor& gradient_input,
             const tensor& means,
             const tensor& invstds,
@@ -186,7 +189,7 @@ namespace dlib { namespace tt
             tensor& src_grad,
             tensor& gamma_grad, 
             tensor& beta_grad 
-    );
+        ){impl(gradient_input,means,invstds,src,gamma,src_grad,gamma_grad,beta_grad);}
         /*!
             requires
                 - invstds and means should be the output of a call to
@@ -209,6 +212,15 @@ namespace dlib { namespace tt
                 - Adds the gradient of f() with respect to gamma to #gamma_grad.
                 - Adds the gradient of f() with respect to beta to #beta_grad.
         !*/
+    private:
+#ifdef DLIB_USE_CUDA
+        cuda::batch_normalize_conv_gradient impl;
+#else
+        cpu::batch_normalize_conv_gradient impl;
+#endif
+    };
+
+// ----------------------------------------------------------------------------------------
 
     void batch_normalize_conv (
         resizable_tensor& dest,
@@ -234,7 +246,10 @@ namespace dlib { namespace tt
             - #invstds == 1/(the standard deviation values of the contents of src).
     !*/
 
-    void batch_normalize_conv_gradient (
+    class batch_normalize_conv_gradient
+    {
+    public:
+        void operator() (
             const tensor& gradient_input,
             const tensor& means,
             const tensor& invstds,
@@ -243,7 +258,7 @@ namespace dlib { namespace tt
             tensor& src_grad,
             tensor& gamma_grad, 
             tensor& beta_grad 
-    );
+        ){impl(gradient_input,means,invstds,src,gamma,src_grad,gamma_grad,beta_grad);}
         /*!
             requires
                 - invstds and means should be the output of a call to
@@ -264,8 +279,15 @@ namespace dlib { namespace tt
                 - Adds the gradient of f() with respect to gamma to #gamma_grad.
                 - Adds the gradient of f() with respect to beta to #beta_grad.
         !*/
+    private:
+#ifdef DLIB_USE_CUDA
+        cuda::batch_normalize_conv_gradient impl;
+#else
+        cpu::batch_normalize_conv_gradient impl;
+#endif
+    };
 
-    // -----------------------------------------------------------------------------------
+// -----------------------------------------------------------------------------------
 
     void threshold (
         tensor& data,

dlib/test/dnn.cpp

@@ -17,7 +17,7 @@ namespace
 
     using namespace test;
     using namespace dlib;
-    using namespace dlib::cpu;
+    using namespace dlib::tt;
     using namespace std;
 
     logger dlog("test.dnn");
@@ -99,7 +99,8 @@ namespace
         gamma_grad = 0;
         beta_grad = 0;
 
-        batch_normalize_gradient(gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad);
+        batch_normalize_gradient bng;
+        bng(gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad);
 
         auto grad_error = compare_gradients(src_grad, grad_src);
         dlog << LINFO << "src error: " << grad_error;
@@ -175,7 +176,8 @@ namespace
         gamma_grad = 0;
         beta_grad = 0;
 
-        batch_normalize_conv_gradient(gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad);
+        batch_normalize_conv_gradient bng;
+        bng(gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad);
 
 
         auto grad_error = compare_gradients(src_grad, grad_src);