Broke the svm header file into 3 separate files.

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%402226

dlib/svm/function.h

+// Copyright (C) 2007  Davis E. King (davisking@users.sourceforge.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_SVm_FUNCTION
+#define DLIB_SVm_FUNCTION
+
+#include "function_abstract.h"
+#include <cmath>
+#include <limits>
+#include <sstream>
+#include "../matrix.h"
+#include "../algs.h"
+#include "../serialize.h"
+#include "../rand.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename K
+        >
+    struct decision_function
+    {
+        typedef typename K::scalar_type scalar_type;
+        typedef typename K::sample_type sample_type;
+        typedef typename K::mem_manager_type mem_manager_type;
+
+        typedef matrix<scalar_type,0,1,mem_manager_type> scalar_vector_type;
+        typedef matrix<sample_type,0,1,mem_manager_type> sample_vector_type;
+
+        const scalar_vector_type alpha;
+        const scalar_type b;
+        const K kernel_function;
+        const sample_vector_type support_vectors;
+
+        decision_function (
+        ) : b(0), kernel_function(K()) {}
+
+        decision_function (
+            const decision_function& d
+        ) : 
+            alpha(d.alpha), 
+            b(d.b),
+            kernel_function(d.kernel_function),
+            support_vectors(d.support_vectors) 
+        {}
+
+        decision_function (
+            const scalar_vector_type& alpha_,
+            const scalar_type& b_,
+            const K& kernel_function_,
+            const sample_vector_type& support_vectors_
+        ) :
+            alpha(alpha_),
+            b(b_),
+            kernel_function(kernel_function_),
+            support_vectors(support_vectors_)
+        {}
+
+        decision_function& operator= (
+            const decision_function& d
+        )
+        {
+            if (this != &d)
+            {
+                const_cast<scalar_vector_type&>(alpha) = d.alpha;
+                const_cast<scalar_type&>(b) = d.b;
+                const_cast<K&>(kernel_function) = d.kernel_function;
+                const_cast<sample_vector_type&>(support_vectors) = d.support_vectors;
+            }
+            return *this;
+        }
+
+        scalar_type operator() (
+            const sample_type& x
+        ) const
+        {
+            scalar_type temp = 0;
+            for (long i = 0; i < alpha.nr(); ++i)
+                temp += alpha(i) * kernel_function(x,support_vectors(i));
+
+            return temp - b;
+        }
+    };
+
+    template <
+        typename K
+        >
+    void serialize (
+        const decision_function<K>& item,
+        std::ostream& out
+    )
+    {
+        try
+        {
+            serialize(item.alpha, out);
+            serialize(item.b,     out);
+            serialize(item.kernel_function, out);
+            serialize(item.support_vectors, out);
+        }
+        catch (serialization_error e)
+        { 
+            throw serialization_error(e.info + "\n   while serializing object of type decision_function"); 
+        }
+    }
+
+    template <
+        typename K
+        >
+    void deserialize (
+        decision_function<K>& item,
+        std::istream& in 
+    )
+    {
+        typedef typename K::scalar_type scalar_type;
+        typedef typename K::sample_type sample_type;
+        typedef typename K::mem_manager_type mem_manager_type;
+
+        typedef matrix<scalar_type,0,1,mem_manager_type> scalar_vector_type;
+        typedef matrix<sample_type,0,1,mem_manager_type> sample_vector_type;
+        try
+        {
+            deserialize(const_cast<scalar_vector_type&>(item.alpha), in);
+            deserialize(const_cast<scalar_type&>(item.b), in);
+            deserialize(const_cast<K&>(item.kernel_function), in);
+            deserialize(const_cast<sample_vector_type&>(item.support_vectors), in);
+        }
+        catch (serialization_error e)
+        { 
+            throw serialization_error(e.info + "\n   while deserializing object of type decision_function"); 
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename K 
+        >
+    struct probabilistic_decision_function
+    {
+        typedef typename K::scalar_type scalar_type;
+        typedef typename K::sample_type sample_type;
+        typedef typename K::mem_manager_type mem_manager_type;
+
+        const scalar_type a;
+        const scalar_type b;
+        const decision_function<K> decision_funct;
+
+        probabilistic_decision_function (
+        ) : a(0), b(0), decision_funct(decision_function<K>()) {}
+
+        probabilistic_decision_function (
+            const probabilistic_decision_function& d
+        ) : 
+            a(d.a),
+            b(d.b),
+            decision_funct(d.decision_funct)
+        {}
+
+        probabilistic_decision_function (
+            const scalar_type a_,
+            const scalar_type b_,
+            const decision_function<K>& decision_funct_ 
+        ) :
+            a(a_),
+            b(b_),
+            decision_funct(decision_funct_)
+        {}
+
+        probabilistic_decision_function& operator= (
+            const probabilistic_decision_function& d
+        )
+        {
+            if (this != &d)
+            {
+                const_cast<scalar_type&>(a) = d.a;
+                const_cast<scalar_type&>(b) = d.b;
+                const_cast<decision_function<K>&>(decision_funct) = d.decision_funct;
+            }
+            return *this;
+        }
+
+        scalar_type operator() (
+            const sample_type& x
+        ) const
+        {
+            scalar_type f = decision_funct(x);
+            return 1/(1 + std::exp(a*f + b));
+        }
+    };
+
+    template <
+        typename K 
+        >
+    void serialize (
+        const probabilistic_decision_function<K>& item,
+        std::ostream& out
+    )
+    {
+        try
+        {
+            serialize(item.a, out);
+            serialize(item.b, out);
+            serialize(item.decision_funct, out);
+        }
+        catch (serialization_error& e)
+        { 
+            throw serialization_error(e.info + "\n   while serializing object of type probabilistic_decision_function"); 
+        }
+    }
+
+    template <
+        typename K 
+        >
+    void deserialize (
+        probabilistic_decision_function<K>& item,
+        std::istream& in 
+    )
+    {
+        typedef typename K::scalar_type scalar_type;
+        try
+        {
+            deserialize(const_cast<scalar_type&>(item.a), in);
+            deserialize(const_cast<scalar_type&>(item.b), in);
+            deserialize(const_cast<decision_function<K>&>(item.decision_funct), in);
+        }
+        catch (serialization_error& e)
+        { 
+            throw serialization_error(e.info + "\n   while deserializing object of type probabilistic_decision_function"); 
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_SVm_FUNCTION
+
+

dlib/svm/function_abstract.h

+// Copyright (C) 2007  Davis E. King (davisking@users.sourceforge.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_SVm_FUNCTION_ABSTRACT_
+#ifdef DLIB_SVm_FUNCTION_ABSTRACT_
+
+#include <cmath>
+#include <limits>
+#include <sstream>
+#include "../matrix/matrix_abstract.h"
+#include "../algs.h"
+#include "../serialize.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename K
+        >
+    struct decision_function 
+    {
+        /*!
+            REQUIREMENTS ON K
+                K must be a kernel function object type as defined at the top 
+                of this document.
+
+            WHAT THIS OBJECT REPRESENTS 
+                This object represents a decision or regression function that was 
+                learned by a kernel based learning algorithm.  
+        !*/
+
+        typedef typename K::scalar_type scalar_type;
+        typedef typename K::sample_type sample_type;
+        typedef typename K::mem_manager_type mem_manager_type;
+
+        typedef matrix<scalar_type,0,1,mem_manager_type> scalar_vector_type;
+        typedef matrix<sample_type,0,1,mem_manager_type> sample_vector_type;
+
+        const scalar_vector_type alpha;
+        const scalar_type        b;
+        const K                  kernel_function;
+        const sample_vector_type support_vectors;
+
+        decision_function (
+        );
+        /*!
+            ensures
+                - #b == 0
+                - #alpha.nr() == 0
+                - #support_vectors.nr() == 0
+        !*/
+
+        decision_function (
+            const decision_function& f
+        );
+        /*!
+            ensures
+                - #*this is a copy of f
+        !*/
+
+        decision_function (
+            const scalar_vector_type& alpha_,
+            const scalar_type& b_,
+            const K& kernel_function_,
+            const sample_vector_type& support_vectors_
+        ) : alpha(alpha_), b(b_), kernel_function(kernel_function_), support_vectors(support_vectors_) {}
+        /*!
+            ensures
+                - populates the decision function with the given support vectors, weights(i.e. alphas),
+                  b term, and kernel function.
+        !*/
+
+        decision_function& operator= (
+            const decision_function& d
+        );
+        /*!
+            ensures
+                - #*this is identical to d
+                - returns *this
+        !*/
+
+        scalar_type operator() (
+            const sample_type& x
+        ) const
+        /*!
+            ensures
+                - evalutes this sample according to the decision
+                  function contained in this object.
+        !*/
+        {
+            scalar_type temp = 0;
+            for (long i = 0; i < alpha.nr(); ++i)
+                temp += alpha(i) * kernel_function(x,support_vectors(i));
+
+            returns temp - b;
+        }
+    };
+
+    template <
+        typename K
+        >
+    void serialize (
+        const decision_function<K>& item,
+        std::ostream& out
+    );
+    /*!
+        provides serialization support for decision_function
+    !*/
+
+    template <
+        typename K
+        >
+    void deserialize (
+        decision_function<K>& item,
+        std::istream& in 
+    );
+    /*!
+        provides serialization support for decision_function
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename K
+        >
+    struct probabilistic_decision_function 
+    {
+        /*!
+            REQUIREMENTS ON K
+                K must be a kernel function object type as defined at the top 
+                of this document.
+
+            WHAT THIS OBJECT REPRESENTS 
+                This object represents a binary decision function that returns an 
+                estimate of the probability that a given sample is in the +1 class.
+        !*/
+
+        typedef typename K::scalar_type scalar_type;
+        typedef typename K::sample_type sample_type;
+        typedef typename K::mem_manager_type mem_manager_type;
+
+        const scalar_type a;
+        const scalar_type b;
+        const decision_function<K> decision_funct;
+
+        probabilistic_decision_function (
+        );
+        /*!
+            ensures
+                - #a == 0
+                - #b == 0
+                - #decision_function has its initial value
+        !*/
+
+        probabilistic_decision_function (
+            const probabilistic_decision_function& f
+        );
+        /*!
+            ensures
+                - #*this is a copy of f
+        !*/
+
+        probabilistic_decision_function (
+            const scalar_type a_,
+            const scalar_type b_,
+            const decision_function<K>& decision_funct_ 
+        ) : a(a_), b(b_), decision_funct(decision_funct_) {}
+        /*!
+            ensures
+                - populates the probabilistic decision function with the given a, b, 
+                  and decision_function.
+        !*/
+
+        probabilistic_decision_function& operator= (
+            const probabilistic_decision_function& d
+        );
+        /*!
+            ensures
+                - #*this is identical to d
+                - returns *this
+        !*/
+
+        scalar_type operator() (
+            const sample_type& x
+        ) const
+        /*!
+            ensures
+                - returns a number P such that:
+                    - 0 <= P <= 1
+                    - P represents the probability that sample x is from 
+                      the class +1
+        !*/
+        {
+            // Evaluate the normal SVM decision function
+            scalar_type f = decision_funct(x);
+            // Now basically normalize the output so that it is a properly
+            // conditioned probability of x being in the +1 class given
+            // the output of the SVM.
+            return 1/(1 + std::exp(a*f + b));
+        }
+    };
+
+    template <
+        typename K
+        >
+    void serialize (
+        const probabilistic_decision_function<K>& item,
+        std::ostream& out
+    );
+    /*!
+        provides serialization support for probabilistic_decision_function
+    !*/
+
+    template <
+        typename K
+        >
+    void deserialize (
+        probabilistic_decision_function<K>& item,
+        std::istream& in 
+    );
+    /*!
+        provides serialization support for probabilistic_decision_function
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_SVm_FUNCTION_ABSTRACT_
+
+
+

dlib/svm/kernel.h

+// Copyright (C) 2007  Davis E. King (davisking@users.sourceforge.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_SVm_KERNEL
+#define DLIB_SVm_KERNEL
+
+#include "kernel_abstract.h"
+#include <cmath>
+#include <limits>
+#include <sstream>
+#include "../matrix.h"
+#include "../algs.h"
+#include "../serialize.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename T
+        >
+    struct radial_basis_kernel
+    {
+        typedef typename T::type scalar_type;
+        typedef T sample_type;
+        typedef typename T::mem_manager_type mem_manager_type;
+
+        radial_basis_kernel(const scalar_type g) : gamma(g) {}
+        radial_basis_kernel() : gamma(0.1) {}
+        radial_basis_kernel(
+            const radial_basis_kernel& k
+        ) : gamma(k.gamma) {}
+
+
+        const scalar_type gamma;
+
+        scalar_type operator() (
+            const sample_type& a,
+            const sample_type& b
+        ) const
+        { 
+            const scalar_type d = trans(a-b)*(a-b);
+            return std::exp(-gamma*d);
+        }
+
+        radial_basis_kernel& operator= (
+            const radial_basis_kernel& k
+        )
+        {
+            const_cast<scalar_type&>(gamma) = k.gamma;
+            return *this;
+        }
+    };
+
+    template <
+        typename T
+        >
+    void serialize (
+        const radial_basis_kernel<T>& item,
+        std::ostream& out
+    )
+    {
+        try
+        {
+            serialize(item.gamma, out);
+        }
+        catch (serialization_error& e)
+        { 
+            throw serialization_error(e.info + "\n   while serializing object of type radial_basis_kernel"); 
+        }
+    }
+
+    template <
+        typename T
+        >
+    void deserialize (
+        radial_basis_kernel<T>& item,
+        std::istream& in 
+    )
+    {
+        typedef typename T::type scalar_type;
+        try
+        {
+            deserialize(const_cast<scalar_type&>(item.gamma), in);
+        }
+        catch (serialization_error& e)
+        { 
+            throw serialization_error(e.info + "\n   while deserializing object of type radial_basis_kernel"); 
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename T
+        >
+    struct polynomial_kernel
+    {
+        typedef typename T::type scalar_type;
+        typedef T sample_type;
+        typedef typename T::mem_manager_type mem_manager_type;
+
+        polynomial_kernel(const scalar_type g, const scalar_type c, const scalar_type d) : gamma(g), coef(c), degree(d) {}
+        polynomial_kernel() : gamma(1), coef(0), degree(1) {}
+        polynomial_kernel(
+            const polynomial_kernel& k
+        ) : gamma(k.gamma), coef(k.coef), degree(k.degree) {}
+
+        typedef T type;
+        const scalar_type gamma;
+        const scalar_type coef;
+        const scalar_type degree;
+
+        scalar_type operator() (
+            const sample_type& a,
+            const sample_type& b
+        ) const
+        { 
+            return std::pow(gamma*(trans(a)*b) + coef, degree);
+        }
+
+        polynomial_kernel& operator= (
+            const polynomial_kernel& k
+        )
+        {
+            const_cast<scalar_type&>(gamma) = k.gamma;
+            const_cast<scalar_type&>(coef) = k.coef;
+            const_cast<scalar_type&>(degree) = k.degree;
+            return *this;
+        }
+    };
+
+    template <
+        typename T
+        >
+    void serialize (
+        const polynomial_kernel<T>& item,
+        std::ostream& out
+    )
+    {
+        try
+        {
+            serialize(item.gamma, out);
+            serialize(item.coef, out);
+            serialize(item.degree, out);
+        }
+        catch (serialization_error& e)
+        { 
+            throw serialization_error(e.info + "\n   while serializing object of type polynomial_kernel"); 
+        }
+    }
+
+    template <
+        typename T
+        >
+    void deserialize (
+        polynomial_kernel<T>& item,
+        std::istream& in 
+    )
+    {
+        typedef typename T::type scalar_type;
+        try
+        {
+            deserialize(const_cast<scalar_type&>(item.gamma), in);
+            deserialize(const_cast<scalar_type&>(item.coef), in);
+            deserialize(const_cast<scalar_type&>(item.degree), in);
+        }
+        catch (serialization_error& e)
+        { 
+            throw serialization_error(e.info + "\n   while deserializing object of type polynomial_kernel"); 
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename T>
+    struct linear_kernel
+    {
+        typedef typename T::type scalar_type;
+        typedef T sample_type;
+        typedef typename T::mem_manager_type mem_manager_type;
+
+        scalar_type operator() (
+            const sample_type& a,
+            const sample_type& b
+        ) const
+        { 
+            return trans(a)*b;
+        }
+    };
+
+    template <
+        typename T
+        >
+    void serialize (
+        const linear_kernel<T>& item,
+        std::ostream& out
+    ){}
+
+    template <
+        typename T
+        >
+    void deserialize (
+        linear_kernel<T>& item,
+        std::istream& in 
+    ){}
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_SVm_KERNEL
+
+

dlib/svm/kernel_abstract.h

+// Copyright (C) 2007  Davis E. King (davisking@users.sourceforge.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_SVm_KERNEL_ABSTRACT_
+#ifdef DLIB_SVm_KERNEL_ABSTRACT_
+
+#include <cmath>
+#include <limits>
+#include <sstream>
+#include "../matrix/matrix_abstract.h"
+#include "../algs.h"
+#include "../serialize.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+/*!A                               Kernel_Function_Objects                               */
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    /*! 
+        WHAT IS A KERNEL FUNCTION OBJECT?
+            In the context of the dlib library documentation a kernel function object
+            is an object with an interface with the following properties:
+                - a public typedef named sample_type
+                - a public typedef named scalar_type which should be a float, double, or 
+                  long double type.
+                - an overloaded operator() that operates on two items of sample_type 
+                  and returns a scalar_type.  
+                  (e.g. scalar_type val = kernel_function(sample(i),sample(j)); 
+                   would be a valid expression)
+                - a public typedef named mem_manager_type that is an implementation of 
+                  dlib/memory_manager/memory_manager_kernel_abstract.h or
+                  dlib/memory_manager_global/memory_manager_global_kernel_abstract.h or
+                  dlib/memory_manager_stateless/memory_manager_stateless_kernel_abstract.h 
+
+        For examples of kernel functions see the following objects
+        (e.g. the radial_basis_kernel).
+    !*/
+
+    template <
+        typename T
+        >
+    struct radial_basis_kernel
+    {
+        /*!
+            REQUIREMENTS ON T
+                T must be a dlib::matrix object 
+
+            WHAT THIS OBJECT REPRESENTS
+                This object represents a radial basis function kernel
+        !*/
+
+        typedef typename T::type scalar_type;
+        typedef T sample_type;
+        typedef typename T::mem_manager_type mem_manager_type;
+
+        const scalar_type gamma;
+
+        radial_basis_kernel(
+        );
+        /*!
+            ensures
+                - #gamma == 0.1 
+        !*/
+
+        radial_basis_kernel(
+            const radial_basis_kernel& k
+        );
+        /*!
+            ensures
+                - #gamma == k.gamma
+        !*/
+
+        radial_basis_kernel(
+            const scalar_type g
+        );
+        /*!
+            ensures
+                - #gamma == g
+        !*/
+
+        scalar_type operator() (
+            const sample_type& a,
+            const sample_type& b
+        ) const;
+        /*!
+            requires
+                - a.nc() == 1
+                - b.nc() == 1
+                - a.nr() == b.nr()
+            ensures
+                - returns exp(-gamma * ||a-b||^2)
+        !*/
+
+        radial_basis_kernel& operator= (
+            const radial_basis_kernel& k
+        );
+        /*!
+            ensures
+                - #gamma = k.gamma
+                - returns *this
+        !*/
+
+    };
+
+    template <
+        typename T
+        >
+    void serialize (
+        const radial_basis_kernel<T>& item,
+        std::ostream& out
+    );
+    /*!
+        provides serialization support for radial_basis_kernel
+    !*/
+
+    template <
+        typename K
+        >
+    void deserialize (
+        radial_basis_kernel<T>& item,
+        std::istream& in 
+    );
+    /*!
+        provides deserialization support for radial_basis_kernel
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename T
+        >
+    struct polynomial_kernel
+    {
+        /*!
+            REQUIREMENTS ON T
+                T must be a dlib::matrix object 
+
+            WHAT THIS OBJECT REPRESENTS
+                This object represents a polynomial kernel
+        !*/
+
+        typedef typename T::type scalar_type;
+        typedef T sample_type;
+        typedef typename T::mem_manager_type mem_manager_type;
+
+        const scalar_type gamma;
+        const scalar_type coef;
+        const scalar_type degree;
+
+        polynomial_kernel(
+        );
+        /*!
+            ensures
+                - #gamma == 1 
+                - #coef == 0 
+                - #degree == 1 
+        !*/
+
+        polynomial_kernel(
+            const radial_basis_kernel& k
+        );
+        /*!
+            ensures
+                - #gamma == k.gamma
+        !*/
+
+        polynomial_kernel(
+            const scalar_type g,
+            const scalar_type c,
+            const scalar_type d
+        );
+        /*!
+            ensures
+                - #gamma == g
+                - #coef == c
+                - #degree == d
+        !*/
+
+        scalar_type operator() (
+            const sample_type& a,
+            const sample_type& b
+        ) const;
+        /*!
+            requires
+                - a.nc() == 1
+                - b.nc() == 1
+                - a.nr() == b.nr()
+            ensures
+                - returns pow(gamma*trans(a)*b + coef, degree)
+        !*/
+
+        polynomial_kernel& operator= (
+            const polynomial_kernel& k
+        );
+        /*!
+            ensures
+                - #gamma = k.gamma
+                - #coef = k.coef
+                - #degree = k.degree
+                - returns *this
+        !*/
+
+    };
+
+    template <
+        typename T
+        >
+    void serialize (
+        const polynomial_kernel<T>& item,
+        std::ostream& out
+    );
+    /*!
+        provides serialization support for polynomial_kernel
+    !*/
+
+    template <
+        typename K
+        >
+    void deserialize (
+        polynomial_kernel<T>& item,
+        std::istream& in 
+    );
+    /*!
+        provides deserialization support for polynomial_kernel
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename T
+        >
+    struct linear_kernel
+    {
+        /*!
+            REQUIREMENTS ON T
+                T must be a dlib::matrix object 
+
+            WHAT THIS OBJECT REPRESENTS
+                This object represents a linear function kernel
+        !*/
+
+        typedef typename T::type scalar_type;
+        typedef T sample_type;
+        typedef typename T::mem_manager_type mem_manager_type;
+
+        scalar_type operator() (
+            const sample_type& a,
+            const sample_type& b
+        ) const;
+        /*!
+            requires
+                - a.nc() == 1
+                - b.nc() == 1
+                - a.nr() == b.nr()
+            ensures
+                - returns trans(a)*b
+        !*/
+    };
+
+    template <
+        typename T
+        >
+    void serialize (
+        const linear_kernel<T>& item,
+        std::ostream& out
+    );
+    /*!
+        provides serialization support for linear_kernel
+    !*/
+
+    template <
+        typename K
+        >
+    void deserialize (
+        linear_kernel<T>& item,
+        std::istream& in 
+    );
+    /*!
+        provides deserialization support for linear_kernel 
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_SVm_KERNEL_ABSTRACT_
+
+
+

dlib/svm/svm.h

@@ -12,6 +12,8 @@
 #include "../serialize.h"
 #include "../rand.h"
 #include "dlib/std_allocator.h"
+#include "function.h"
+#include "kernel.h"
 
 namespace dlib
 {
@@ -174,413 +176,6 @@ namespace dlib
 
     };
 
-// ----------------------------------------------------------------------------------------
-
-    template <
-        typename T
-        >
-    struct radial_basis_kernel
-    {
-        typedef typename T::type scalar_type;
-        typedef T sample_type;
-        typedef typename T::mem_manager_type mem_manager_type;
-
-        radial_basis_kernel(const scalar_type g) : gamma(g) {}
-        radial_basis_kernel() : gamma(0.1) {}
-        radial_basis_kernel(
-            const radial_basis_kernel& k
-        ) : gamma(k.gamma) {}
-
-
-        const scalar_type gamma;
-
-        scalar_type operator() (
-            const sample_type& a,
-            const sample_type& b
-        ) const
-        { 
-            const scalar_type d = trans(a-b)*(a-b);
-            return std::exp(-gamma*d);
-        }
-
-        radial_basis_kernel& operator= (
-            const radial_basis_kernel& k
-        )
-        {
-            const_cast<scalar_type&>(gamma) = k.gamma;
-            return *this;
-        }
-    };
-
-    template <
-        typename T
-        >
-    void serialize (
-        const radial_basis_kernel<T>& item,
-        std::ostream& out
-    )
-    {
-        try
-        {
-            serialize(item.gamma, out);
-        }
-        catch (serialization_error& e)
-        { 
-            throw serialization_error(e.info + "\n   while serializing object of type radial_basis_kernel"); 
-        }
-    }
-
-    template <
-        typename T
-        >
-    void deserialize (
-        radial_basis_kernel<T>& item,
-        std::istream& in 
-    )
-    {
-        typedef typename T::type scalar_type;
-        try
-        {
-            deserialize(const_cast<scalar_type&>(item.gamma), in);
-        }
-        catch (serialization_error& e)
-        { 
-            throw serialization_error(e.info + "\n   while deserializing object of type radial_basis_kernel"); 
-        }
-    }
-
-// ----------------------------------------------------------------------------------------
-
-    template <
-        typename T
-        >
-    struct polynomial_kernel
-    {
-        typedef typename T::type scalar_type;
-        typedef T sample_type;
-        typedef typename T::mem_manager_type mem_manager_type;
-
-        polynomial_kernel(const scalar_type g, const scalar_type c, const scalar_type d) : gamma(g), coef(c), degree(d) {}
-        polynomial_kernel() : gamma(1), coef(0), degree(1) {}
-        polynomial_kernel(
-            const polynomial_kernel& k
-        ) : gamma(k.gamma), coef(k.coef), degree(k.degree) {}
-
-        typedef T type;
-        const scalar_type gamma;
-        const scalar_type coef;
-        const scalar_type degree;
-
-        scalar_type operator() (
-            const sample_type& a,
-            const sample_type& b
-        ) const
-        { 
-            return std::pow(gamma*(trans(a)*b) + coef, degree);
-        }
-
-        polynomial_kernel& operator= (
-            const polynomial_kernel& k
-        )
-        {
-            const_cast<scalar_type&>(gamma) = k.gamma;
-            const_cast<scalar_type&>(coef) = k.coef;
-            const_cast<scalar_type&>(degree) = k.degree;
-            return *this;
-        }
-    };
-
-    template <
-        typename T
-        >
-    void serialize (
-        const polynomial_kernel<T>& item,
-        std::ostream& out
-    )
-    {
-        try
-        {
-            serialize(item.gamma, out);
-            serialize(item.coef, out);
-            serialize(item.degree, out);
-        }
-        catch (serialization_error& e)
-        { 
-            throw serialization_error(e.info + "\n   while serializing object of type polynomial_kernel"); 
-        }
-    }
-
-    template <
-        typename T
-        >
-    void deserialize (
-        polynomial_kernel<T>& item,
-        std::istream& in 
-    )
-    {
-        typedef typename T::type scalar_type;
-        try
-        {
-            deserialize(const_cast<scalar_type&>(item.gamma), in);
-            deserialize(const_cast<scalar_type&>(item.coef), in);
-            deserialize(const_cast<scalar_type&>(item.degree), in);
-        }
-        catch (serialization_error& e)
-        { 
-            throw serialization_error(e.info + "\n   while deserializing object of type polynomial_kernel"); 
-        }
-    }
-
-// ----------------------------------------------------------------------------------------
-
-    template <typename T>
-    struct linear_kernel
-    {
-        typedef typename T::type scalar_type;
-        typedef T sample_type;
-        typedef typename T::mem_manager_type mem_manager_type;
-
-        scalar_type operator() (
-            const sample_type& a,
-            const sample_type& b
-        ) const
-        { 
-            return trans(a)*b;
-        }
-    };
-
-    template <
-        typename T
-        >
-    void serialize (
-        const linear_kernel<T>& item,
-        std::ostream& out
-    ){}
-
-    template <
-        typename T
-        >
-    void deserialize (
-        linear_kernel<T>& item,
-        std::istream& in 
-    ){}
-
-// ----------------------------------------------------------------------------------------
-
-    template <
-        typename K
-        >
-    struct decision_function
-    {
-        typedef typename K::scalar_type scalar_type;
-        typedef typename K::sample_type sample_type;
-        typedef typename K::mem_manager_type mem_manager_type;
-
-        typedef matrix<scalar_type,0,1,mem_manager_type> scalar_vector_type;
-        typedef matrix<sample_type,0,1,mem_manager_type> sample_vector_type;
-
-        const scalar_vector_type alpha;
-        const scalar_type b;
-        const K kernel_function;
-        const sample_vector_type support_vectors;
-
-        decision_function (
-        ) : b(0), kernel_function(K()) {}
-
-        decision_function (
-            const decision_function& d
-        ) : 
-            alpha(d.alpha), 
-            b(d.b),
-            kernel_function(d.kernel_function),
-            support_vectors(d.support_vectors) 
-        {}
-
-        decision_function (
-            const scalar_vector_type& alpha_,
-            const scalar_type& b_,
-            const K& kernel_function_,
-            const sample_vector_type& support_vectors_
-        ) :
-            alpha(alpha_),
-            b(b_),
-            kernel_function(kernel_function_),
-            support_vectors(support_vectors_)
-        {}
-
-        decision_function& operator= (
-            const decision_function& d
-        )
-        {
-            if (this != &d)
-            {
-                const_cast<scalar_vector_type&>(alpha) = d.alpha;
-                const_cast<scalar_type&>(b) = d.b;
-                const_cast<K&>(kernel_function) = d.kernel_function;
-                const_cast<sample_vector_type&>(support_vectors) = d.support_vectors;
-            }
-            return *this;
-        }
-
-        scalar_type operator() (
-            const sample_type& x
-        ) const
-        {
-            scalar_type temp = 0;
-            for (long i = 0; i < alpha.nr(); ++i)
-                temp += alpha(i) * kernel_function(x,support_vectors(i));
-
-            return temp - b;
-        }
-    };
-
-    template <
-        typename K
-        >
-    void serialize (
-        const decision_function<K>& item,
-        std::ostream& out
-    )
-    {
-        try
-        {
-            serialize(item.alpha, out);
-            serialize(item.b,     out);
-            serialize(item.kernel_function, out);
-            serialize(item.support_vectors, out);
-        }
-        catch (serialization_error e)
-        { 
-            throw serialization_error(e.info + "\n   while serializing object of type decision_function"); 
-        }
-    }
-
-    template <
-        typename K
-        >
-    void deserialize (
-        decision_function<K>& item,
-        std::istream& in 
-    )
-    {
-        typedef typename K::scalar_type scalar_type;
-        typedef typename K::sample_type sample_type;
-        typedef typename K::mem_manager_type mem_manager_type;
-
-        typedef matrix<scalar_type,0,1,mem_manager_type> scalar_vector_type;
-        typedef matrix<sample_type,0,1,mem_manager_type> sample_vector_type;
-        try
-        {
-            deserialize(const_cast<scalar_vector_type&>(item.alpha), in);
-            deserialize(const_cast<scalar_type&>(item.b), in);
-            deserialize(const_cast<K&>(item.kernel_function), in);
-            deserialize(const_cast<sample_vector_type&>(item.support_vectors), in);
-        }
-        catch (serialization_error e)
-        { 
-            throw serialization_error(e.info + "\n   while deserializing object of type decision_function"); 
-        }
-    }
-
-// ----------------------------------------------------------------------------------------
-
-    template <
-        typename K 
-        >
-    struct probabilistic_decision_function
-    {
-        typedef typename K::scalar_type scalar_type;
-        typedef typename K::sample_type sample_type;
-        typedef typename K::mem_manager_type mem_manager_type;
-
-        const scalar_type a;
-        const scalar_type b;
-        const decision_function<K> decision_funct;
-
-        probabilistic_decision_function (
-        ) : a(0), b(0), decision_funct(decision_function<K>()) {}
-
-        probabilistic_decision_function (
-            const probabilistic_decision_function& d
-        ) : 
-            a(d.a),
-            b(d.b),
-            decision_funct(d.decision_funct)
-        {}
-
-        probabilistic_decision_function (
-            const scalar_type a_,
-            const scalar_type b_,
-            const decision_function<K>& decision_funct_ 
-        ) :
-            a(a_),
-            b(b_),
-            decision_funct(decision_funct_)
-        {}
-
-        probabilistic_decision_function& operator= (
-            const probabilistic_decision_function& d
-        )
-        {
-            if (this != &d)
-            {
-                const_cast<scalar_type&>(a) = d.a;
-                const_cast<scalar_type&>(b) = d.b;
-                const_cast<decision_function<K>&>(decision_funct) = d.decision_funct;
-            }
-            return *this;
-        }
-
-        scalar_type operator() (
-            const sample_type& x
-        ) const
-        {
-            scalar_type f = decision_funct(x);
-            return 1/(1 + std::exp(a*f + b));
-        }
-    };
-
-    template <
-        typename K 
-        >
-    void serialize (
-        const probabilistic_decision_function<K>& item,
-        std::ostream& out
-    )
-    {
-        try
-        {
-            serialize(item.a, out);
-            serialize(item.b, out);
-            serialize(item.decision_funct, out);
-        }
-        catch (serialization_error& e)
-        { 
-            throw serialization_error(e.info + "\n   while serializing object of type probabilistic_decision_function"); 
-        }
-    }
-
-    template <
-        typename K 
-        >
-    void deserialize (
-        probabilistic_decision_function<K>& item,
-        std::istream& in 
-    )
-    {
-        typedef typename K::scalar_type scalar_type;
-        try
-        {
-            deserialize(const_cast<scalar_type&>(item.a), in);
-            deserialize(const_cast<scalar_type&>(item.b), in);
-            deserialize(const_cast<decision_function<K>&>(item.decision_funct), in);
-        }
-        catch (serialization_error& e)
-        { 
-            throw serialization_error(e.info + "\n   while deserializing object of type probabilistic_decision_function"); 
-        }
-    }
-
 // ----------------------------------------------------------------------------------------
 
     template <

dlib/svm/svm_abstract.h

@@ -9,493 +9,12 @@
 #include "../matrix/matrix_abstract.h"
 #include "../algs.h"
 #include "../serialize.h"
+#include "function_abstract.h"
+#include "kernel_abstract.h"
 
 namespace dlib
 {
 
-// ----------------------------------------------------------------------------------------
-// ----------------------------------------------------------------------------------------
-/*!A                               Kernel_Function_Objects                               */
-// ----------------------------------------------------------------------------------------
-// ----------------------------------------------------------------------------------------
-
-    /*! 
-        WHAT IS A KERNEL FUNCTION OBJECT?
-            In the context of the dlib library documentation a kernel function object
-            is an object with an interface with the following properties:
-                - a public typedef named sample_type
-                - a public typedef named scalar_type which should be a float, double, or 
-                  long double type.
-                - an overloaded operator() that operates on two items of sample_type 
-                  and returns a scalar_type.  
-                  (e.g. scalar_type val = kernel_function(sample(i),sample(j)); 
-                   would be a valid expression)
-                - a public typedef named mem_manager_type that is an implementation of 
-                  dlib/memory_manager/memory_manager_kernel_abstract.h or
-                  dlib/memory_manager_global/memory_manager_global_kernel_abstract.h or
-                  dlib/memory_manager_stateless/memory_manager_stateless_kernel_abstract.h 
-
-        For examples of kernel functions see the following objects
-        (e.g. the radial_basis_kernel).
-    !*/
-
-    template <
-        typename T
-        >
-    struct radial_basis_kernel
-    {
-        /*!
-            REQUIREMENTS ON T
-                T must be a dlib::matrix object 
-
-            WHAT THIS OBJECT REPRESENTS
-                This object represents a radial basis function kernel
-        !*/
-
-        typedef typename T::type scalar_type;
-        typedef T sample_type;
-        typedef typename T::mem_manager_type mem_manager_type;
-
-        const scalar_type gamma;
-
-        radial_basis_kernel(
-        );
-        /*!
-            ensures
-                - #gamma == 0.1 
-        !*/
-
-        radial_basis_kernel(
-            const radial_basis_kernel& k
-        );
-        /*!
-            ensures
-                - #gamma == k.gamma
-        !*/
-
-        radial_basis_kernel(
-            const scalar_type g
-        );
-        /*!
-            ensures
-                - #gamma == g
-        !*/
-
-        scalar_type operator() (
-            const sample_type& a,
-            const sample_type& b
-        ) const;
-        /*!
-            requires
-                - a.nc() == 1
-                - b.nc() == 1
-                - a.nr() == b.nr()
-            ensures
-                - returns exp(-gamma * ||a-b||^2)
-        !*/
-
-        radial_basis_kernel& operator= (
-            const radial_basis_kernel& k
-        );
-        /*!
-            ensures
-                - #gamma = k.gamma
-                - returns *this
-        !*/
-
-    };
-
-    template <
-        typename T
-        >
-    void serialize (
-        const radial_basis_kernel<T>& item,
-        std::ostream& out
-    );
-    /*!
-        provides serialization support for radial_basis_kernel
-    !*/
-
-    template <
-        typename K
-        >
-    void deserialize (
-        radial_basis_kernel<T>& item,
-        std::istream& in 
-    );
-    /*!
-        provides deserialization support for radial_basis_kernel
-    !*/
-
-// ----------------------------------------------------------------------------------------
-
-    template <
-        typename T
-        >
-    struct polynomial_kernel
-    {
-        /*!
-            REQUIREMENTS ON T
-                T must be a dlib::matrix object 
-
-            WHAT THIS OBJECT REPRESENTS
-                This object represents a polynomial kernel
-        !*/
-
-        typedef typename T::type scalar_type;
-        typedef T sample_type;
-        typedef typename T::mem_manager_type mem_manager_type;
-
-        const scalar_type gamma;
-        const scalar_type coef;
-        const scalar_type degree;
-
-        polynomial_kernel(
-        );
-        /*!
-            ensures
-                - #gamma == 1 
-                - #coef == 0 
-                - #degree == 1 
-        !*/
-
-        polynomial_kernel(
-            const radial_basis_kernel& k
-        );
-        /*!
-            ensures
-                - #gamma == k.gamma
-        !*/
-
-        polynomial_kernel(
-            const scalar_type g,
-            const scalar_type c,
-            const scalar_type d
-        );
-        /*!
-            ensures
-                - #gamma == g
-                - #coef == c
-                - #degree == d
-        !*/
-
-        scalar_type operator() (
-            const sample_type& a,
-            const sample_type& b
-        ) const;
-        /*!
-            requires
-                - a.nc() == 1
-                - b.nc() == 1
-                - a.nr() == b.nr()
-            ensures
-                - returns pow(gamma*trans(a)*b + coef, degree)
-        !*/
-
-        polynomial_kernel& operator= (
-            const polynomial_kernel& k
-        );
-        /*!
-            ensures
-                - #gamma = k.gamma
-                - #coef = k.coef
-                - #degree = k.degree
-                - returns *this
-        !*/
-
-    };
-
-    template <
-        typename T
-        >
-    void serialize (
-        const polynomial_kernel<T>& item,
-        std::ostream& out
-    );
-    /*!
-        provides serialization support for polynomial_kernel
-    !*/
-
-    template <
-        typename K
-        >
-    void deserialize (
-        polynomial_kernel<T>& item,
-        std::istream& in 
-    );
-    /*!
-        provides deserialization support for polynomial_kernel
-    !*/
-
-// ----------------------------------------------------------------------------------------
-
-    template <
-        typename T
-        >
-    struct linear_kernel
-    {
-        /*!
-            REQUIREMENTS ON T
-                T must be a dlib::matrix object 
-
-            WHAT THIS OBJECT REPRESENTS
-                This object represents a linear function kernel
-        !*/
-
-        typedef typename T::type scalar_type;
-        typedef T sample_type;
-        typedef typename T::mem_manager_type mem_manager_type;
-
-        scalar_type operator() (
-            const sample_type& a,
-            const sample_type& b
-        ) const;
-        /*!
-            requires
-                - a.nc() == 1
-                - b.nc() == 1
-                - a.nr() == b.nr()
-            ensures
-                - returns trans(a)*b
-        !*/
-    };
-
-    template <
-        typename T
-        >
-    void serialize (
-        const linear_kernel<T>& item,
-        std::ostream& out
-    );
-    /*!
-        provides serialization support for linear_kernel
-    !*/
-
-    template <
-        typename K
-        >
-    void deserialize (
-        linear_kernel<T>& item,
-        std::istream& in 
-    );
-    /*!
-        provides deserialization support for linear_kernel 
-    !*/
-
-// ----------------------------------------------------------------------------------------
-// ----------------------------------------------------------------------------------------
-// ----------------------------------------------------------------------------------------
-
-    template <
-        typename K
-        >
-    struct decision_function 
-    {
-        /*!
-            REQUIREMENTS ON K
-                K must be a kernel function object type as defined at the top 
-                of this document.
-
-            WHAT THIS OBJECT REPRESENTS 
-                This object represents a binary decision function.
-        !*/
-
-        typedef typename K::scalar_type scalar_type;
-        typedef typename K::sample_type sample_type;
-        typedef typename K::mem_manager_type mem_manager_type;
-
-        typedef matrix<scalar_type,0,1,mem_manager_type> scalar_vector_type;
-        typedef matrix<sample_type,0,1,mem_manager_type> sample_vector_type;
-
-        const scalar_vector_type alpha;
-        const scalar_type        b;
-        const K                  kernel_function;
-        const sample_vector_type support_vectors;
-
-        decision_function (
-        );
-        /*!
-            ensures
-                - #b == 0
-                - #alpha.nr() == 0
-                - #support_vectors.nr() == 0
-        !*/
-
-        decision_function (
-            const decision_function& f
-        );
-        /*!
-            ensures
-                - #*this is a copy of f
-        !*/
-
-        decision_function (
-            const scalar_vector_type& alpha_,
-            const scalar_type& b_,
-            const K& kernel_function_,
-            const sample_vector_type& support_vectors_
-        ) : alpha(alpha_), b(b_), kernel_function(kernel_function_), support_vectors(support_vectors_) {}
-        /*!
-            ensures
-                - populates the decision function with the given support vectors, weights(i.e. alphas),
-                  b term, and kernel function.
-        !*/
-
-        decision_function& operator= (
-            const decision_function& d
-        );
-        /*!
-            ensures
-                - #*this is identical to d
-                - returns *this
-        !*/
-
-        scalar_type operator() (
-            const sample_type& x
-        ) const
-        /*!
-            ensures
-                - predicts the class of x.  
-                - if (the class is predicted to be +1) then
-                    - returns a number >= 0
-                - else
-                    - returns a number < 0
-        !*/
-        {
-            scalar_type temp = 0;
-            for (long i = 0; i < alpha.nr(); ++i)
-                temp += alpha(i) * kernel_function(x,support_vectors(i));
-
-            returns temp - b;
-        }
-    };
-
-    template <
-        typename K
-        >
-    void serialize (
-        const decision_function<K>& item,
-        std::ostream& out
-    );
-    /*!
-        provides serialization support for decision_function
-    !*/
-
-    template <
-        typename K
-        >
-    void deserialize (
-        decision_function<K>& item,
-        std::istream& in 
-    );
-    /*!
-        provides serialization support for decision_function
-    !*/
-
-// ----------------------------------------------------------------------------------------
-
-    template <
-        typename K
-        >
-    struct probabilistic_decision_function 
-    {
-        /*!
-            REQUIREMENTS ON K
-                K must be a kernel function object type as defined at the top 
-                of this document.
-
-            WHAT THIS OBJECT REPRESENTS 
-                This object represents a binary decision function that returns an 
-                estimate of the probability that a given sample is in the +1 class.
-        !*/
-
-        typedef typename K::scalar_type scalar_type;
-        typedef typename K::sample_type sample_type;
-        typedef typename K::mem_manager_type mem_manager_type;
-
-        const scalar_type a;
-        const scalar_type b;
-        const decision_function<K> decision_funct;
-
-        probabilistic_decision_function (
-        );
-        /*!
-            ensures
-                - #a == 0
-                - #b == 0
-                - #decision_function has its initial value
-        !*/
-
-        probabilistic_decision_function (
-            const probabilistic_decision_function& f
-        );
-        /*!
-            ensures
-                - #*this is a copy of f
-        !*/
-
-        probabilistic_decision_function (
-            const scalar_type a_,
-            const scalar_type b_,
-            const decision_function<K>& decision_funct_ 
-        ) : a(a_), b(b_), decision_funct(decision_funct_) {}
-        /*!
-            ensures
-                - populates the probabilistic decision function with the given a, b, 
-                  and decision_function.
-        !*/
-
-        probabilistic_decision_function& operator= (
-            const probabilistic_decision_function& d
-        );
-        /*!
-            ensures
-                - #*this is identical to d
-                - returns *this
-        !*/
-
-        scalar_type operator() (
-            const sample_type& x
-        ) const
-        /*!
-            ensures
-                - returns a number P such that:
-                    - 0 <= P <= 1
-                    - P represents the probability that sample x is from 
-                      the class +1
-        !*/
-        {
-            // Evaluate the normal SVM decision function
-            scalar_type f = decision_funct(x);
-            // Now basically normalize the output so that it is a properly
-            // conditioned probability of x being in the +1 class given
-            // the output of the SVM.
-            return 1/(1 + std::exp(a*f + b));
-        }
-    };
-
-    template <
-        typename K
-        >
-    void serialize (
-        const probabilistic_decision_function<K>& item,
-        std::ostream& out
-    );
-    /*!
-        provides serialization support for probabilistic_decision_function
-    !*/
-
-    template <
-        typename K
-        >
-    void deserialize (
-        probabilistic_decision_function<K>& item,
-        std::istream& in 
-    );
-    /*!
-        provides serialization support for probabilistic_decision_function
-    !*/
-
 // ----------------------------------------------------------------------------------------
 // ----------------------------------------------------------------------------------------
 //                              Functions that perform SVM training 
@@ -551,7 +70,11 @@ namespace dlib
             - caches approximately at most cache_size megabytes of the kernel matrix. 
               (bigger values of this may make training go faster but doesn't affect the 
               result.  However, too big a value will cause you to run out of memory.)
-            - returns the resulting decision function 
+            - returns a decision function F with the following properties:
+                - if (new_x is a sample predicted have +1 label) then
+                    - F(new_x) >= 0
+                - else
+                    - F(new_x) < 0
     !*/
 
     /*