Added the ability to compute projection error to the empirical_kernel_map.

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%403400

dlib/svm/empirical_kernel_map.h

@@ -253,6 +253,26 @@ namespace dlib
             return temp2;
         }
 
+        const matrix<scalar_type,0,1,mem_manager_type>& project (
+            const sample_type& samp,
+            scalar_type& projection_error 
+        ) const
+        {
+            // make sure requires clause is not broken
+            DLIB_ASSERT(out_vector_size() != 0,
+                "\tconst matrix empirical_kernel_map::project()"
+                << "\n\t You have to load this object with data before you can call this function"
+                << "\n\t this: " << this
+                );
+
+            temp1 = kernel_matrix(kernel, basis, samp);
+            temp2 = weights*temp1;
+            // The std::abs is here because rounding error could make the expresison negative
+            // which would be bad since sqrt() would get upset.
+            projection_error = std::sqrt(std::abs( kernel(samp,samp) - dot(temp2,temp2)));
+            return temp2;
+        }
+
         void swap (
             empirical_kernel_map& item
         )

dlib/svm/empirical_kernel_map_abstract.h

@@ -194,6 +194,21 @@ namespace dlib
                       function.
         !*/
 
+        const matrix<scalar_type,0,1,mem_manager_type>& project (
+            const sample_type& sample,
+            scalar_type& projection_error
+        ) const;
+        /*!
+            requires
+                - out_vector_size() != 0
+            ensures
+                - This function returns project(sample)
+                  (i.e. it returns the same thing as the above project() function)
+                - #projection_error == the distance between the point sample gets projected
+                  onto and sample's true image in kernel feature space.  That is, this value
+                  is equal to: convert_to_distance_function(project(sample))(sample)
+        !*/
+
         template <typename EXP>
         const decision_function<kernel_type> convert_to_decision_function (
             const matrix_exp<EXP>& vect
@@ -238,7 +253,10 @@ namespace dlib
                       if S is within the span of the set of basis samples given to the load() function.  
                       If it is not then there will be some approximation error.  Note that all the basis 
                       samples are always within their own span.  So the equality is always exact for the 
-                      samples given to the load() function.
+                      samples given to the load() function.  Note further that the distance computed
+                      by DF(S) is always the correct distance in kernel feature space between vect and
+                      the true projection of S.  That is, the above equality is approximate only because 
+                      of potential error in the project() function.
                     - DF.kernel_function == get_kernel()
                     - DF.b == dot(vect,vect) 
                     - DF.basis_vectors == these will be the basis samples given to the previous call to load().  Note

dlib/test/empirical_kernel_map.cpp

@@ -39,6 +39,95 @@ namespace
         time_t thetime;
         dlib::rand::float_1a rnd;
 
+        void test_projection_error()
+        {
+            for (int runs = 0; runs < 10; ++runs)
+            {
+                print_spinner();
+                typedef matrix<double,0,1> sample_type;
+                typedef radial_basis_kernel<sample_type> kernel_type;
+                const kernel_type kern(0.2);
+
+                empirical_kernel_map<kernel_type> ekm;
+
+                // generate samples
+                const int num = rnd.get_random_8bit_number()%50 + 1;
+                std::vector<sample_type> samples;
+                for (int i = 0; i < num; ++i)
+                {
+                    samples.push_back(randm(5,1,rnd));
+                }
+
+
+                ekm.load(kern, samples);
+
+                double err;
+
+                // the samples in the basis should have zero projection error
+                for (int i = 0; i < samples.size(); ++i)
+                {
+                    ekm.project(samples[i], err);
+                    DLIB_TEST_MSG(abs(err) < 1e-7, abs(err));
+
+                }
+                
+                // Do some sanity tests on the conversion to distance functions while we are at it.
+                for (int i = 0; i < 30; ++i)
+                {
+                    // pick two random samples
+                    const sample_type samp1 = samples[rnd.get_random_32bit_number()%samples.size()];
+                    const sample_type samp2 = samples[rnd.get_random_32bit_number()%samples.size()];
+
+                    matrix<double,0,1> proj1 = ekm.project(samp1);
+                    matrix<double,0,1> proj2 = ekm.project(samp2);
+
+                    distance_function<kernel_type> df1 = ekm.convert_to_distance_function(proj1);
+                    distance_function<kernel_type> df2 = ekm.convert_to_distance_function(proj2);
+
+                    const double true_dist = std::sqrt(kern(samp1,samp1) + kern(samp2,samp2) - 2*kern(samp1,samp2));
+                    DLIB_TEST_MSG(abs(df1(df2) - true_dist) < 1e-7, abs(df1(df2) - true_dist));
+                    DLIB_TEST_MSG(abs(length(proj1-proj2) - true_dist) < 1e-7, abs(length(proj1-proj2) - true_dist));
+                }
+                // Do some sanity tests on the conversion to distance functions while we are at it.  This
+                // time multiply one of the projections by 30 and see that it still all works out right.
+                for (int i = 0; i < 30; ++i)
+                {
+                    // pick two random samples
+                    const sample_type samp1 = samples[rnd.get_random_32bit_number()%samples.size()];
+                    const sample_type samp2 = samples[rnd.get_random_32bit_number()%samples.size()];
+
+                    matrix<double,0,1> proj1 = ekm.project(samp1);
+                    matrix<double,0,1> proj2 = 30*ekm.project(samp2);
+
+                    distance_function<kernel_type> df1 = ekm.convert_to_distance_function(proj1);
+                    distance_function<kernel_type> df2 = ekm.convert_to_distance_function(proj2);
+
+                    DLIB_TEST_MSG(abs(length(proj1-proj2) - df1(df2)) < 1e-7, abs(length(proj1-proj2) - df1(df2)));
+                }
+
+
+                // now generate points with projection error
+                for (double i = 1; i < 10; ++i)
+                {
+                    sample_type test_point = i*randm(5,1,rnd);
+                    ekm.project(test_point, err);
+                    dlog << LTRACE << "projection error: " << err;
+
+                    distance_function<kernel_type> df = ekm.convert_to_distance_function(ekm.project(test_point));
+
+                    // the projection error should be the distance between the test_point and the point it gets
+                    // projected onto
+                    DLIB_TEST_MSG(abs(df(test_point) - err) < 1e-10, abs(df(test_point) - err));
+                    // while we are at it make sure the squared norm in the distance function is right
+                    double df_error = abs(df.b - trans(df.alpha)*kernel_matrix(kern, samples)*df.alpha);
+                    DLIB_TEST_MSG( df_error < 1e-10, df_error);
+                }
+
+
+
+            }
+        }
+
         template <typename kernel_type>
         void test_with_kernel(const kernel_type& kern)
         {
@@ -241,6 +330,8 @@ namespace
                     df2 = ekm2.convert_to_distance_function(transform*samp);
                     DLIB_TEST_MSG(df1(df2) < eps, df1(df2));
                 }
+
+
             }
         }
 
@@ -252,6 +343,8 @@ namespace
             dlog << LINFO << "time seed: " << thetime;
             rnd.set_seed(cast_to_string(thetime));
 
+            print_spinner();
+            test_projection_error();
             print_spinner();
             dlog << LINFO << "test with linear kernel";
             test_with_kernel(linear_kernel<sample_type>());