Added tools for doing global optimization. The main new tools here are

find_global_maximum() and global_function_search.

dlib/CMakeLists.txt

@@ -189,6 +189,7 @@ if (NOT TARGET dlib)
          data_io/mnist.cpp
          dnn/cpu_dlib.cpp
          dnn/tensor_tools.cpp
+         global_optimization/global_function_search.cpp
    )
 
 

dlib/all/source.cpp

@@ -87,6 +87,7 @@
 // Stuff that requires C++11 (and some threading stuff)
 #include "../dnn/cpu_dlib.cpp"
 #include "../dnn/tensor_tools.cpp"
+#include "../global_optimization/global_function_search.cpp"
 
 
 #define DLIB_ALL_SOURCE_END

dlib/global_optimization.h

@@ -4,6 +4,8 @@
 #define DLIB_GLOBAL_OPTIMIZATIOn_HEADER
 
 #include "global_optimization/upper_bound_function.h"
+#include "global_optimization/global_function_search.h"
+#include "global_optimization/find_global_maximum.h"
 
 #endif // DLIB_GLOBAL_OPTIMIZATIOn_HEADER
 

dlib/global_optimization/find_global_maximum.h

+// Copyright (C) 2017  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_FiND_GLOBAL_MAXIMUM_hH_
+#define DLIB_FiND_GLOBAL_MAXIMUM_hH_
+
+#include "global_function_search.h"
+
+// TODO, move ct_make_integer_range into some other file so we don't have to include the
+// dnn header.  That thing is huge.
+#include <dlib/dnn.h>
+#include <utility>
+
+namespace dlib
+{
+    namespace gopt_impl
+    {
+    // ----------------------------------------------------------------------------------------
+
+        class disable_decay_to_scalar 
+        {
+            const matrix<double,0,1>& a;
+        public:
+            disable_decay_to_scalar(const matrix<double,0,1>& a) : a(a){}
+            operator const matrix<double,0,1>&() const { return a;}
+        };
+
+
+        template <typename T, size_t... indices> 
+        auto _cwv (
+            T&& f, 
+            const matrix<double,0,1>& a, 
+            impl::ct_integers_list<indices...>
+        ) -> decltype(f(a(indices-1)...)) 
+        {
+            DLIB_CASSERT(a.size() == sizeof...(indices), "You invoked dlib::call_with_vect(f,a) but the number of arguments expected by f() doesn't match the size of 'a'. "
+                << "Expected " << sizeof...(indices) << " arguments but got " << a.size() << "."
+            );  
+            return f(a(indices-1)...); 
+        }
+
+
+        template <size_t max_unpack>
+        struct call_with_vect
+        {
+            template <typename T>
+            static auto go(T&& f, const matrix<double,0,1>& a) -> decltype(_cwv(std::forward<T>(f),a,typename impl::ct_make_integer_range<max_unpack>::type()))
+            {
+                return _cwv(std::forward<T>(f),a,typename impl::ct_make_integer_range<max_unpack>::type());
+            }
+
+            template <typename T>
+            static auto go(T&& f, const matrix<double,0,1>& a) -> decltype(call_with_vect<max_unpack-1>::template go(std::forward<T>(f),a))
+            {
+                return call_with_vect<max_unpack-1>::go(std::forward<T>(f),a);
+            }
+        };
+
+        template <>
+        struct call_with_vect<0>
+        {
+            template <typename T>
+            static auto go(T&& f, const matrix<double,0,1>& a) -> decltype(f(disable_decay_to_scalar(a)))
+            {
+                return f(disable_decay_to_scalar(a));
+            }
+        };
+    }
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <typename T> 
+    auto call_with_vect(
+        T&& f, 
+        const matrix<double,0,1>& a
+    ) -> decltype(gopt_impl::call_with_vect<40>::go(f,a))
+    {
+        // unpack up to 40 parameters when calling f()
+        return gopt_impl::call_with_vect<40>::go(std::forward<T>(f),a);
+    }
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    struct max_function_calls
+    {
+        max_function_calls() = default;
+        explicit max_function_calls(size_t max_calls) : max_calls(max_calls) {}
+        size_t max_calls = std::numeric_limits<size_t>::max();
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename funct
+        >
+    std::pair<size_t,function_evaluation> find_global_maximum (
+        std::vector<funct>& functions,
+        const std::vector<function_spec>& specs,
+        const max_function_calls num,
+        const std::chrono::nanoseconds max_runtime,
+        double solver_epsilon = 1e-11
+    ) 
+    {
+        global_function_search opt(specs);
+        opt.set_solver_epsilon(solver_epsilon);
+
+        const auto time_to_stop = std::chrono::steady_clock::now() + max_runtime;
+
+        for (size_t i = 0; i < num.max_calls && std::chrono::steady_clock::now() < time_to_stop; ++i)
+        {
+            auto next = opt.get_next_x();
+            double y = call_with_vect(functions[next.function_idx()], next.x());
+            next.set(y);
+
+
+
+            // TODO, remove this funky test code
+            matrix<double,0,1> x;
+            size_t function_idx;
+            opt.get_best_function_eval(x,y,function_idx);
+            using namespace std;
+            cout << "\ni: "<< i << endl;
+            cout << "best eval x: "<< trans(x);
+            cout << "best eval y: "<< y << endl;
+            cout << "best eval function index: "<< function_idx << endl;
+            if (std::abs(y  - 21.9210397) < 0.0001)
+            {
+                cout << "DONE!" << endl;
+                //cin.get();
+                break;
+            }
+        }
+
+
+        matrix<double,0,1> x;
+        double y;
+        size_t function_idx;
+        opt.get_best_function_eval(x,y,function_idx);
+        return std::make_pair(function_idx, function_evaluation(x,std::move(y)));
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename funct
+        >
+    function_evaluation find_global_maximum (
+        funct f,
+        const matrix<double,0,1>& lower,
+        const matrix<double,0,1>& upper,
+        const max_function_calls num,
+        double solver_epsilon = 1e-11
+    ) 
+    {
+        std::vector<funct> functions(1,f);
+        std::vector<function_spec> specs(1, function_spec(lower, upper));
+        auto forever = std::chrono::hours(24*356*290);
+        return find_global_maximum(functions, specs, num, forever, solver_epsilon).second;
+    }
+
+    template <
+        typename funct
+        >
+    function_evaluation find_global_maximum (
+        funct f,
+        const double lower,
+        const double upper,
+        const max_function_calls num,
+        double solver_epsilon = 1e-11
+    ) 
+    {
+        return find_global_maximum(f, matrix<double,0,1>({lower}), matrix<double,0,1>({upper}), num, solver_epsilon);
+    }
+
+    template <
+        typename funct
+        >
+    function_evaluation find_global_maximum (
+        funct f,
+        const matrix<double,0,1>& lower,
+        const matrix<double,0,1>& upper,
+        const std::vector<bool>& is_integer_variable,
+        const max_function_calls num,
+        double solver_epsilon = 1e-11
+    ) 
+    {
+        std::vector<funct> functions(1, std::move(f));
+        std::vector<function_spec> specs(1, function_spec(lower, upper, is_integer_variable));
+        auto forever = std::chrono::hours(24*356*290);
+        return find_global_maximum(functions, specs, num, forever, solver_epsilon).second;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename funct
+        >
+    function_evaluation find_global_maximum (
+        funct f,
+        const matrix<double,0,1>& lower,
+        const matrix<double,0,1>& upper,
+        const std::chrono::nanoseconds max_runtime,
+        double solver_epsilon = 1e-11
+    ) 
+    {
+        std::vector<funct> functions(1,f);
+        std::vector<function_spec> specs(1, function_spec(lower, upper));
+        return find_global_maximum(functions, specs, max_function_calls(), max_runtime, solver_epsilon).second;
+    }
+
+    template <
+        typename funct
+        >
+    function_evaluation find_global_maximum (
+        funct f,
+        const double lower,
+        const double upper,
+        const std::chrono::nanoseconds max_runtime,
+        double solver_epsilon = 1e-11
+    ) 
+    {
+        return find_global_maximum(f, matrix<double,0,1>({lower}), matrix<double,0,1>({upper}), max_runtime, solver_epsilon);
+    }
+
+    template <
+        typename funct
+        >
+    function_evaluation find_global_maximum (
+        funct f,
+        const matrix<double,0,1>& lower,
+        const matrix<double,0,1>& upper,
+        const std::vector<bool>& is_integer_variable,
+        const std::chrono::nanoseconds max_runtime,
+        double solver_epsilon = 1e-11
+    ) 
+    {
+        std::vector<funct> functions(1, std::move(f));
+        std::vector<function_spec> specs(1, function_spec(lower, upper, is_integer_variable));
+        return find_global_maximum(functions, specs, max_function_calls(), max_runtime, solver_epsilon).second;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_FiND_GLOBAL_MAXIMUM_hH_
+

dlib/global_optimization/global_function_search.h

+// Copyright (C) 2017  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_GLOBAL_FuNCTION_SEARCH_Hh_
+#define DLIB_GLOBAL_FuNCTION_SEARCH_Hh_
+
+#include <vector>
+#include "../matrix.h"
+#include <mutex>
+#include "../rand.h"
+#include "upper_bound_function.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    struct function_spec
+    {
+        function_spec(const matrix<double,0,1>& lower_, const matrix<double,0,1>& upper_);
+
+        function_spec(const matrix<double,0,1>& lower, const matrix<double,0,1>& upper, std::vector<bool> is_integer);
+
+        matrix<double,0,1> lower;
+        matrix<double,0,1> upper;
+        std::vector<bool> is_integer_variable;
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    namespace gopt_impl 
+    {
+        struct outstanding_function_eval_request
+        {
+            size_t request_id = 0;   // unique id for this eval request
+            matrix<double,0,1> x;   // function x to evaluate 
+
+            // trust region specific stuff
+            bool was_trust_region_generated_request = false;
+            double predicted_improvement = std::numeric_limits<double>::quiet_NaN();
+            double anchor_objective_value = std::numeric_limits<double>::quiet_NaN(); // objective value at center of TR step
+
+            bool operator==(const outstanding_function_eval_request& item) const { return request_id == item.request_id; }
+        };
+
+        struct funct_info
+        {
+            funct_info() = delete;
+            funct_info(const funct_info&) = delete;
+            funct_info& operator=(const funct_info&) = delete;
+
+            funct_info(const function_spec& spec, size_t function_idx, const std::shared_ptr<std::mutex>& m) : spec(spec), function_idx(function_idx), m(m)
+            {
+                best_x = zeros_matrix(spec.lower);
+            }
+
+            std::vector<function_evaluation> all_function_evals (
+            ) const;
+
+            static double find_nn (
+                const std::vector<function_evaluation>& evals,
+                const matrix<double,0,1>& x
+            );
+
+
+            function_spec spec;
+            size_t function_idx = 0;
+            std::shared_ptr<std::mutex> m;
+            std::vector<function_evaluation> complete_evals;
+            std::vector<outstanding_function_eval_request> incomplete_evals;
+            matrix<double,0,1> best_x; 
+            double best_objective_value = -std::numeric_limits<double>::infinity(); 
+            double radius = 0;
+        };
+
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    class function_evaluation_request
+    {
+    public:
+
+        function_evaluation_request() = delete;
+
+        function_evaluation_request(const function_evaluation_request&) = delete;
+        function_evaluation_request& operator=(const function_evaluation_request&) = delete;
+
+        function_evaluation_request(function_evaluation_request&& item);
+
+        function_evaluation_request& operator=(function_evaluation_request&& item);
+
+        void swap(function_evaluation_request& item);
+
+        size_t function_idx (
+        ) const;
+
+        const matrix<double,0,1>& x (
+        ) const;
+
+        bool has_been_evaluated (
+        ) const;
+
+        ~function_evaluation_request();
+
+        void set (
+            double y
+        );
+        /*!
+            requires
+                - has_been_evaluated() == false
+            ensures
+                - #has_been_evaluated() == true
+        !*/
+
+    private:
+
+        friend class global_function_search;
+
+        explicit function_evaluation_request(
+            const gopt_impl::outstanding_function_eval_request& req,
+            const std::shared_ptr<gopt_impl::funct_info>& info
+        ) : req(req), info(info) {}
+
+        bool m_has_been_evaluated = false;
+        gopt_impl::outstanding_function_eval_request req;
+        std::shared_ptr<gopt_impl::funct_info> info;
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    class global_function_search
+    {
+    public:
+
+        global_function_search() = delete;
+
+        explicit global_function_search(
+            const function_spec& function
+        ); 
+
+        explicit global_function_search(
+            const std::vector<function_spec>& functions_
+        );
+
+        global_function_search(
+            const std::vector<function_spec>& functions_,
+            const std::vector<std::vector<function_evaluation>>& initial_function_evals
+        ); 
+
+        global_function_search(const global_function_search&) = delete;
+        global_function_search& operator=(const global_function_search& item) = delete;
+
+        size_t num_functions() const;
+
+        void set_seed (
+            time_t seed
+        );
+
+        void get_function_evaluations (
+            std::vector<function_spec>& specs,
+            std::vector<std::vector<function_evaluation>>& function_evals
+        ) const;
+
+        void get_best_function_eval (
+            matrix<double,0,1>& x,
+            double& y,
+            size_t& function_idx
+        ) const;
+
+        function_evaluation_request get_next_x (
+        ); 
+
+        double get_pure_random_search_probability (
+        ) const; 
+
+        void set_pure_random_search_probability (
+            double prob
+        );
+
+        double get_solver_epsilon (
+        ) const; 
+
+        void set_solver_epsilon (
+            double eps
+        );
+
+        double get_relative_noise_magnitude (
+        ) const; 
+
+        void set_relative_noise_magnitude (
+            double value
+        );
+
+        size_t get_monte_carlo_upper_bound_sample_num (
+        ) const; 
+
+        void set_monte_carlo_upper_bound_sample_num (
+            size_t num
+        );
+
+    private:
+
+        std::shared_ptr<gopt_impl::funct_info> best_function() const;
+
+        std::shared_ptr<gopt_impl::funct_info> best_function(size_t& idx) const;
+
+        bool has_incomplete_trust_region_request (
+        ) const;
+
+
+        dlib::rand rnd;
+        double pure_random_search_probability = 0.02;
+        double qp_eps = 1e-11;
+        double relative_noise_magnitude = 0.001;
+        size_t num_random_samples = 5000;
+        bool do_trust_region_step = true;
+
+        size_t next_request_id = 1;
+
+        std::vector<std::shared_ptr<gopt_impl::funct_info>> functions;
+        std::shared_ptr<std::mutex> m;
+
+    };
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_GLOBAL_FuNCTION_SEARCH_Hh_
+