Worked on the spec for the manifold regularization stuff more.

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%403594

dlib/manifold_regularization/graph_creation.h

@@ -180,13 +180,13 @@ namespace dlib
 // ----------------------------------------------------------------------------------------
 
     template <
-        typename alloc
+        typename vector_type
         >
     bool contains_duplicate_pairs (
-        const std::vector<sample_pair, alloc>& pairs
+        const vector_type& pairs
     )
     {
-        std::vector<sample_pair, alloc> temp(pairs);
+        vector_type temp(pairs);
         std::sort(temp.begin(), temp.end(), &order_by_index);
 
         for (unsigned long i = 1; i < temp.size(); ++i)
@@ -199,6 +199,34 @@ namespace dlib
         return false;
     }
 
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename vector_type 
+        >
+    unsigned long max_index_value_plus_one (
+        const vector_type& pairs
+    )
+    {
+        if (pairs.size() == 0)
+        {
+            return 0;
+        }
+        else
+        {
+            unsigned long max_idx = 0;
+            for (unsigned long i = 0; i < pairs.size(); ++i)
+            {
+                if (pairs[i].index1() > max_idx)
+                    max_idx = pairs[i].index1();
+                if (pairs[i].index2() > max_idx)
+                    max_idx = pairs[i].index2();
+            }
+
+            return max_idx + 1;
+        }
+    }
+
 // ----------------------------------------------------------------------------------------
 
 }

dlib/manifold_regularization/graph_creation_abstract.h

@@ -80,12 +80,15 @@ namespace dlib
 // ----------------------------------------------------------------------------------------
 
     template <
-        typename alloc
+        typename vector_type 
         >
     bool contains_duplicate_pairs (
-        const std::vector<sample_pair, alloc>& pairs
+        const vector_type& pairs
     );
     /*!
+        requires
+            - vector_type == a type with an interface compatible with std::vector and 
+              it must in turn contain objects with an interface compatible with dlib::sample_pair
         ensures
             - if (pairs contains any elements that are equal according to operator==) then
                 - returns true
@@ -93,6 +96,27 @@ namespace dlib
                 - returns false
     !*/
 
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename vector_type 
+        >
+    unsigned long max_index_value_plus_one (
+        const vector_type& pairs
+    );
+    /*!
+        requires
+            - vector_type == a type with an interface compatible with std::vector and 
+              it must in turn contain objects with an interface compatible with dlib::sample_pair
+        ensures
+            - if (pairs.size() == 0) then
+                - returns 0
+            - else
+                - returns a number N such that: 
+                    - for all i:  pairs[i].index1()   <  N && pairs[i].index2()   <  N
+                    - for some j: pairs[j].index1()+1 == N || pairs[j].index2()+1 == N
+    !*/
+
 // ----------------------------------------------------------------------------------------
 
 }

dlib/manifold_regularization/linear_manifold_regularizer.h

@@ -3,7 +3,7 @@
 #ifndef DLIB_LINEAR_MANIFOLD_ReGULARIZER_H__
 #define DLIB_LINEAR_MANIFOLD_ReGULARIZER_H__
 
-#include "linear_manifold_regularizer.h"
+#include "linear_manifold_regularizer_abstract.h"
 #include <limits>
 #include <vector>
 #include "../serialize.h"
@@ -41,6 +41,10 @@ namespace dlib
 
             unsigned long size (
             ) const
+            /*!
+                ensures
+                    - returns the number of vertices in this graph
+            !*/
             {
                 return _size;
             }
@@ -51,6 +55,9 @@ namespace dlib
             /*!
                 requires
                     - idx < size()
+                ensures
+                    - returns an iterator that points to the first neighbor of 
+                      the idx'th vertex.
             !*/
             {
                 return blocks[idx];
@@ -62,6 +69,9 @@ namespace dlib
             /*!
                 requires
                     - idx < size()
+                ensures
+                    - returns an iterator that points one past the last neighbor
+                      of the idx'th vertex.
             !*/
             {
                 return blocks[idx+1];
@@ -78,11 +88,14 @@ namespace dlib
                     - vector_type == a type with an interface compatible with std::vector and 
                       it must in turn contain objects with an interface compatible with dlib::sample_pair
                     - edges.size() > 0
-                    - all the elements of edges are unique.  That is:
-                        - for all valid i and j where i != j:
-                          it must be true that edges[i] != edges[j]
+                    - contains_duplicate_pairs(edges) == false
                     - weight_funct(edges[i]) must be a valid expression that evaluates to a
-                      floating point number
+                      floating point number >= 0
+                ensures
+                    - #size() == one greater than the max index in edges.
+                    - builds the adjacency list so that it contains all the given edges.
+                    - The weight in each neighbor is set to the output of the weight_funct()
+                      for the associated edge.
             !*/
             {
 
@@ -160,15 +173,6 @@ namespace dlib
         >
     class linear_manifold_regularizer
     {
-        /*!
-            REQUIREMENTS ON matrix_type
-                Must be some type of dlib::matrix.
-
-            WHAT THIS OBJECT REPRESENTS
-                This object computes the inv(T) matrix described in the following paper:
-                    Linear Manifold Regularization for Large Scale Semi-supervised Learning
-                    by Vikas Sindhwani, Partha Niyogi, and Mikhail Belkin
-        !*/
 
     public:
         typedef typename matrix_type::mem_manager_type mem_manager_type;
@@ -194,14 +198,18 @@ namespace dlib
             make_mr_matrix(samples, graph);
         }
 
+        long dimensionality (
+        ) const { return reg_mat.nr(); }
+
         general_matrix get_transformation_matrix (
             scalar_type intrinsic_regularization_strength
         ) const
-        /*!
-            requires
-                - intrinsic_regularization_strength >= 0
-        !*/
         {
+            if (dimensionality() == 0)
+                return general_matrix();
+
+            // TODO: should we divide intrinsic_regularization_strength by the number of edges?  That maybe a more 
+            // reasonable interface.
             return inv_lower_triangular(chol(identity_matrix<scalar_type>(reg_mat.nr()) + intrinsic_regularization_strength*reg_mat));
         }
 
@@ -215,15 +223,17 @@ namespace dlib
         /*!
             requires
                 - samples.size() == graph.size()
+            ensures
+                - computes trans(X)*lap(graph)*X where X is the data matrix 
+                  (i.e. the matrix that contains all the samples in its rows)
+                  and lap(graph) is the laplacian matrix of the graph.  The
+                  resulting matrix is stored in reg_mat.
         !*/
         {
             const unsigned long dims = samples[0].size();
             reg_mat.set_size(dims,dims);
             reg_mat = 0;
 
-            // Compute trans(X)*lap(graph)*X where X is the data matrix 
-            // (i.e. the matrix that contains all the samples in its rows)
-            // and lap(graph) is the laplacian matrix of the graph.
 
             typename impl::undirected_adjacency_list::const_iterator beg, end;
 

dlib/manifold_regularization/linear_manifold_regularizer_abstract.h

+// Copyright (C) 2010  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_LINEAR_MANIFOLD_ReGULARIZER_ABSTRACT_H__
+#ifdef DLIB_LINEAR_MANIFOLD_ReGULARIZER_ABSTRACT_H__
+
+#include <limits>
+#include <vector>
+#include "../serialize.h"
+#include "../matrix.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename matrix_type
+        >
+    class linear_manifold_regularizer
+    {
+        /*!
+            REQUIREMENTS ON matrix_type
+                Must be some type of dlib::matrix.
+
+            INITIAL VALUE
+                - dimensionality() == 0
+
+            WHAT THIS OBJECT REPRESENTS
+                Many learning algorithms attempt to minimize a loss function that,
+                at a high level, looks like this:   
+                    loss(w) == complexity + training_set_error
+
+                The idea is to find the set of parameters, w, that gives low error on 
+                your training data but also is not "complex" according to some particular
+                measure of complexity.  This strategy of penalizing complexity is 
+                usually called regularization.
+
+                In the above setting all the training data consists of labeled samples.  
+                However, it would be nice to be able to benefit from unlabeled data 
+                (see the example program for this object for an example where unlabeled 
+                data is useful).  The idea of manifold regularization is to extract useful 
+                information from unlabeled data by defining which data samples are "close" 
+                to each other (perhaps by using their 3 nearest neighbors) and then adding 
+                a term to the loss function that penalizes any decision rule which produces 
+                different output on data samples that we have designated as being close.
+                
+                It turns out that it is possible to turn these manifold regularized loss
+                functions into the normal form shown above by applying a certain kind
+                of processing to all of our data samples.  Once this is done we can use
+                a normal learning algorithm, such as the svm_c_linear_trainer, on just the
+                labeled data samples and obtain the same output as the manifold regularized
+                learner would have produced.  Therefore, the linear_manifold_regularizer is 
+                a tool for creating this preprocessing transformation.  In particular, the
+                transformation is linear.  That is, it is just a matrix you multiply with
+                all your samples.
+
+
+                For a more detailed discussion of this topic you should consult the following
+                paper.  In particular, see section 4.2.  This object computes the inverse T 
+                matrix described in that section.
+
+                    Linear Manifold Regularization for Large Scale Semi-supervised Learning
+                    by Vikas Sindhwani, Partha Niyogi, and Mikhail Belkin
+        !*/
+
+    public:
+        typedef typename matrix_type::mem_manager_type mem_manager_type;
+        typedef typename matrix_type::type scalar_type;
+        typedef typename matrix_type::layout_type layout_type;
+        typedef matrix<scalar_type,0,0,mem_manager_type,layout_type> general_matrix;
+
+        template <
+            typename vector_type1, 
+            typename vector_type2, 
+            typename weight_function_type
+            >
+        void build (
+            const vector_type1& samples,
+            const vector_type2& edges,
+            const weight_function_type& weight_funct
+        );
+        /*!
+            requires
+                - vector_type1 == a type with an interface compatible with std::vector and it must
+                  in turn contain dlib::matrix objects.
+                - vector_type2 == a type with an interface compatible with std::vector and 
+                  it must in turn contain objects with an interface compatible with dlib::sample_pair
+                - edges.size() > 0
+                - contains_duplicate_pairs(edges) == false
+                - max_index_value_plus_one(edges) <= samples.size()
+                - weight_funct(edges[i]) must be a valid expression that evaluates to a
+                  floating point number >= 0
+            ensures
+                - #dimensionality() == samples[0].size()
+                - This function sets up the transformation matrix describe above.  The manifold
+                  regularization is done assuming that the following samples are meant to
+                  be "close" according to the graph defined by the given edges.  I.e:
+                    - for all valid i:  samples[edges[i].index1()] is close to samples[edges[i].index2()].
+                      How much we care about these two samples having similar outputs according
+                      to the learned rule is given by weight_funct(edges[i]).  Bigger weights mean
+                      we care more.
+        !*/
+
+        long dimensionality (
+        ) const;
+        /*!
+            ensures
+                - returns the number of rows and columns in the transformation matrix
+                  produced by this object
+        !*/
+
+        general_matrix get_transformation_matrix (
+            scalar_type intrinsic_regularization_strength
+        ) const;
+        /*!
+            requires
+                - intrinsic_regularization_strength >= 0
+            ensures
+                - returns a matrix that represents the preprocessing transformation described above.
+                - You must choose how important the manifold regularizer is relative to the basic
+                  "don't be complex" regularizer describe above.  The intrinsic_regularization_strength
+                  is the parameter that controls this trade-off.  A large value of 
+                  intrinsic_regularization_strength means that more emphasis should be placed on
+                  finding decision rules which produce the same output on similar samples.  On 
+                  the other hand, a small value would mean that we don't care much about the 
+                  manifold regularizer.  For example, using 0 will cause this function to return the 
+                  identity matrix.
+                - The returned matrix will have dimensionality() rows and columns
+        !*/
+
+    };
+
+}
+
+// ----------------------------------------------------------------------------------------
+
+#endif // DLIB_LINEAR_MANIFOLD_ReGULARIZER_ABSTRACT_H__
+
+