Make ELU an inplace layer and fix Clipped ReLU implementation (#2346)

* Make ELU an inplace layer

* Fix CUDA implementation of clipped_relu and update tests

dlib/cuda/cpu_dlib.cpp

@@ -1940,7 +1940,7 @@ namespace dlib
                     if (in[i] > 0)
                         out[i] = gi[i];
                     else
-                        out[i] = alpha * std::exp(in[i]) * gi[i];
+                        out[i] = (alpha + in[i]) * gi[i];
                 }
             }
             else
@@ -1950,7 +1950,7 @@ namespace dlib
                     if (in[i] > 0)
                         out[i] += gi[i];
                     else
-                        out[i] += alpha * std::exp(in[i]) * gi[i];
+                        out[i] += (alpha + in[i]) * gi[i];
                 }
             }
         }

dlib/cuda/cuda_dlib.cu

@@ -1486,10 +1486,12 @@ namespace dlib
         {
             for (auto i : grid_stride_range(0, n))
             {
-                if (s[i] > 0 && s[i] < alpha)
-                    d[i] = s[i];
+                if (s[i] < 0)
+                    d[i] = 0;
+                else if (s[i] > alpha)
+                    d[i] = alpha;
                 else
-                    d[i] = 0.f;
+                    d[i] = s[i];
             }
         }
 
@@ -1571,7 +1573,7 @@ namespace dlib
                 if (s[i] > 0)
                     out[i] = gi[i];
                 else
-                    out[i] = alpha * std::exp(s[i]) * gi[i];
+                    out[i] = (alpha + s[i]) * gi[i];
             }
         }
 
@@ -1582,7 +1584,7 @@ namespace dlib
                 if (s[i] > 0)
                     out[i] += gi[i];
                 else
-                    out[i] += alpha * std::exp(s[i]) * gi[i];
+                    out[i] += (alpha + s[i]) * gi[i];
             }
         }
 

dlib/dnn/layers.h

@@ -3558,24 +3558,19 @@ namespace dlib
         {
         }
 
-        template <typename SUBNET>
-        void forward(
-            SUBNET& sub,
-            resizable_tensor& data_output
-        )
+        void forward_inplace(const tensor& input, tensor& output)
         {
-            data_output.copy_size(sub.get_output());
-            tt::elu(data_output, sub.get_output(), alpha);
+            tt::elu(output, input, alpha);
         }
 
-        template <typename SUBNET>
-        void backward(
+        void backward_inplace(
+            const tensor& computed_output,
             const tensor& gradient_input,
-            SUBNET& sub,
+            tensor& data_grad,
             tensor&
         )
         {
-            tt::elu_gradient(sub.get_gradient_input(), sub.get_output(), gradient_input, alpha);
+            tt::elu_gradient(data_grad, computed_output, gradient_input, alpha);
         }
 
         inline dpoint map_input_to_output (const dpoint& p) const { return p; }

dlib/dnn/layers_abstract.h

@@ -2537,8 +2537,8 @@ namespace dlib
                 - returns the alpha parameter of the elu
         !*/
         template <typename SUBNET> void setup (const SUBNET& sub);
-        template <typename SUBNET> void forward(const SUBNET& sub, resizable_tensor& data_output);
-        template <typename SUBNET> void backward(const tensor& gradient_input, SUBNET& sub, tensor&);
+        void forward_inplace(const tensor& input, tensor& output);
+        void backward_inplace(const tensor& computed_output, const tensor& gradient_input, tensor& data_grad, tensor& params_grad);
         dpoint map_input_to_output(dpoint p) const;
         dpoint map_output_to_input(dpoint p) const;
         const tensor& get_layer_params() const;

dlib/test/dnn.cpp

@@ -274,14 +274,14 @@ namespace
 #ifdef DLIB_USE_CUDA
         using namespace dlib::tt;
         print_spinner();
-        const long n = 5;
+        const long n = 4;
         const long k = 5;
         const long nr = 3;
         const long nc = 3;
         const float ceiling = 6.0f;
         resizable_tensor src(n, k, nr, nc);
         tt::tensor_rand rnd;
-        rnd.fill_uniform(src);
+        rnd.fill_gaussian(src, 0, 3);
         resizable_tensor dest_cuda, dest_cpu;
         dest_cuda.copy_size(src);
         dest_cpu.copy_size(src);
@@ -313,14 +313,14 @@ namespace
 #ifdef DLIB_USE_CUDA
         using namespace dlib::tt;
         print_spinner();
-        const long n = 5;
+        const long n = 4;
         const long k = 5;
         const long nr = 3;
         const long nc = 3;
         const float alpha = 1.0f;
         resizable_tensor src(n, k, nr, nc);
         tt::tensor_rand rnd;
-        rnd.fill_uniform(src);
+        rnd.fill_gaussian(src);
         resizable_tensor dest_cuda, dest_cpu;
         dest_cuda.copy_size(src);
         dest_cpu.copy_size(src);
@@ -336,13 +336,13 @@ namespace
         grad_cuda.copy_size(src);
         grad_cpu.copy_size(src);
         grad_input.copy_size(src);
-        rnd.fill_uniform(grad_input);
+        rnd.fill_gaussian(grad_input);
         grad_cuda = 0;
         grad_cpu = 0;
         cuda::elu_gradient(grad_cuda, dest_cuda, grad_input, alpha);
         cpu::elu_gradient(grad_cpu, dest_cpu, grad_input, alpha);
         error = max(abs(mat(grad_cuda) - mat(grad_cpu)));
-        DLIB_TEST_MSG(error < 1e-7, "error: " << error);
+        DLIB_TEST_MSG(error < 1e-6, "error: " << error);
 #endif // DLIB_USE_CUDA
     }