Added clamp_function() and backtracking_line_search()

dlib/optimization/optimization.h

@@ -144,6 +144,41 @@ namespace dlib
     template <typename funct>
     const negate_function_object<funct> negate_function(const funct& f) { return negate_function_object<funct>(f); }
 
+// ----------------------------------------------------------------------------------------
+
+    template <typename funct, typename EXP1, typename EXP2>
+    struct clamped_function_object
+    {
+        clamped_function_object(
+            const funct& f_,
+            const matrix_exp<EXP1>& x_lower_,
+            const matrix_exp<EXP2>& x_upper_ 
+        ) : f(f_), x_lower(x_lower_), x_upper(x_upper_)
+        {
+        }
+
+        template <typename T>
+        double operator() (
+            const T& x
+        ) const
+        {
+            return f(clamp(x,x_lower,x_upper));
+        }
+        
+        const funct& f;
+        const matrix_exp<EXP1>& x_lower;
+        const matrix_exp<EXP2>& x_upper; 
+    };
+
+    template <typename funct, typename EXP1, typename EXP2>
+    clamped_function_object<funct,EXP1,EXP2> clamp_function(
+        const funct& f,
+        const matrix_exp<EXP1>& x_lower,
+        const matrix_exp<EXP2>& x_upper 
+    ) { return clamped_function_object<funct,EXP1,EXP2>(f,x_lower,x_upper); }
+
+// ----------------------------------------------------------------------------------------
+
 // ----------------------------------------------------------------------------------------
 // ----------------------------------------------------------------------------------------
 //                    Functions that perform unconstrained optimization 

dlib/optimization/optimization_abstract.h

@@ -70,15 +70,6 @@ namespace dlib
 
 // ----------------------------------------------------------------------------------------
 
-    template <
-        typename funct
-        >
-    class negate_function_object;
-    /*!
-        This is a function object that represents the negative of some other
-        function.   
-    !*/
-
     template <
         typename funct
         >
@@ -93,6 +84,32 @@ namespace dlib
               the returned function object represents g(x) == -f(x)
     !*/
 
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename funct, 
+        typename EXP1, 
+        typename EXP2
+        >
+    clamped_function_object<funct,EXP1,EXP2> clamp_function (
+        const funct& f,
+        const matrix_exp<EXP1>& x_lower,
+        const matrix_exp<EXP2>& x_upper 
+    );
+    /*!
+        requires
+            - f == a function that takes a matrix and returns a scalar value.  Moreover, f
+              must be capable of taking in matrices with the same dimensions as x_lower and
+              x_upper.  So f(x_lower) must be a valid expression that evaluates to a scalar
+              value.
+            - x_lower.nr() == x_upper.nr() && x_lower.nc() == x_upper.nc()
+              (i.e. x_lower and x_upper must have the same dimensions)
+            - x_lower and x_upper must contain the same type of elements.
+        ensures
+            - returns a function object that represents the function g(x) where
+              g(x) == f(clamp(x,x_lower,x_upper))
+    !*/
+
 // ----------------------------------------------------------------------------------------
 // ----------------------------------------------------------------------------------------
 //                    Functions that perform unconstrained optimization 

dlib/optimization/optimization_line_search.h

@@ -461,6 +461,56 @@ namespace dlib
         }
     }
 
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename funct
+        >
+    double backtracking_line_search (
+        const funct& f, 
+        double f0,
+        double d0,
+        double alpha,
+        double rho, 
+        unsigned long max_iter 
+    )
+    {
+        DLIB_ASSERT (
+            0 < rho && rho < 1 && max_iter > 0,
+            "\tdouble backtracking_line_search()"
+            << "\n\tYou have given invalid arguments to this function"
+            << "\n\t rho:      " << rho 
+            << "\n\t max_iter: " << max_iter 
+        );
+
+        // If the gradient is telling us we need to search backwards then that is what we
+        // will do.
+        if (d0 > 0 && alpha > 0)
+            alpha *= -1;
+
+        for (unsigned long iter = 0; iter < max_iter; ++iter)
+        {
+            const double val = f(alpha);
+            if (val <= f0 + alpha*rho*d0)
+            {
+                return alpha;
+            }
+            else
+            {
+                // Interpolate a new alpha.  We also make sure the step by which we
+                // reduce alpha is not super small.
+                double step;
+                if (d0 < 0)
+                    step = put_in_range(0.1,0.9, poly_min_extrap(f0, d0, val));
+                else
+                    step = put_in_range(0.1,0.9, poly_min_extrap(f0, -d0, val));
+
+                alpha *= step;
+            }
+        }
+        return alpha;
+    }
+
 // ----------------------------------------------------------------------------------------
 
     class optimize_single_variable_failure : public error {

dlib/optimization/optimization_line_search_abstract.h

@@ -191,6 +191,38 @@ namespace dlib
         and also in the more recent book Numerical Optimization by Nocedal and Wright.
     */
 
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename funct
+        >
+    double backtracking_line_search (
+        const funct& f, 
+        double f0,
+        double d0,
+        double alpha,
+        double rho, 
+        unsigned long max_iter 
+    );
+    /*!
+        requires
+            - 0 < rho < 1
+            - f is a scalar function of scalars 
+              (e.g. a line_search_funct object)
+            - f0 == f(0)
+            - d0 == the derivative of f() at f(0). 
+            - max_iter > 0
+        ensures
+            - Performs a backtracking line search and uses the Armijo sufficient decrease
+              rule to decide when the search can stop.
+                - rho == the parameter of the sufficient decrease condition. 
+                - max_iter == the maximum number of iterations allowable.  After this many
+                  evaluations of f() line_search() is guaranteed to terminate.
+            - The line search starts with the input alpha value and then backtracks until
+              it finds a good enough alpha value.  Once found, it returns the alpha value
+              such that f(alpha) is significantly closer to the minimum of f than f(0).
+    !*/
+
 // ----------------------------------------------------------------------------------------
 
     class optimize_single_variable_failure : public error;