Added unit tests for the new svm_c_linear_dcd_trainer object.

dlib/test/CMakeLists.txt

@@ -116,6 +116,7 @@ set (tests
    std_vector_c.cpp
    string.cpp
    svm_c_linear.cpp
+   svm_c_linear_dcd.cpp
    svm.cpp
    svm_multiclass_linear.cpp
    svm_struct.cpp

dlib/test/makefile

@@ -131,6 +131,7 @@ SRC += statistics.cpp
 SRC += std_vector_c.cpp
 SRC += string.cpp
 SRC += svm_c_linear.cpp
+SRC += svm_c_linear_dcd.cpp
 SRC += svm.cpp
 SRC += svm_multiclass_linear.cpp
 SRC += svm_struct.cpp

dlib/test/svm_c_linear_dcd.cpp

+// Copyright (C) 2012  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+
+
+#include <dlib/svm.h>
+#include <dlib/rand.h>
+#include <dlib/statistics.h>
+
+#include "tester.h"
+
+
+namespace  
+{
+
+    using namespace test;
+    using namespace dlib;
+    using namespace std;
+
+    logger dlog("test.svm_c_linear_dcd");
+
+// ----------------------------------------------------------------------------------------
+
+    void test_sparse()
+    {
+        typedef std::map<unsigned long,double> sample_type;
+
+
+        typedef sparse_linear_kernel<sample_type> kernel_type;
+
+
+
+        svm_c_linear_trainer<kernel_type> linear_trainer_cpa;
+        svm_c_linear_dcd_trainer<kernel_type> linear_trainer;
+
+        svm_c_linear_dcd_trainer<kernel_type>::optimizer_state state;
+
+        const double C = 10;
+        linear_trainer.set_epsilon(1e-10);
+        linear_trainer_cpa.set_epsilon(1e-10);
+
+
+        std::vector<sample_type> samples;
+        std::vector<double> labels;
+
+        // make an instance of a sample vector so we can use it below
+        sample_type sample;
+
+        decision_function<kernel_type> df, df2, df3;
+
+        dlib::rand rnd;
+        // Now lets go into a loop and randomly generate 10000 samples.
+        double label = +1;
+        for (int i = 0; i < 100; ++i)
+        {
+            // flip this flag
+            label *= -1;
+
+            sample.clear();
+
+            // now make a random sparse sample with at most 10 non-zero elements
+            for (int j = 0; j < 5; ++j)
+            {
+                int idx = rnd.get_random_32bit_number()%10;
+                double value = static_cast<double>(std::rand())/RAND_MAX;
+
+                sample[idx] = label*value;
+            }
+
+            // Also save the samples we are generating so we can let the svm_c_linear_trainer
+            // learn from them below.  
+            samples.push_back(sample);
+            labels.push_back(label);
+
+            if (samples.size() > 1)
+            {
+                linear_trainer_cpa.set_c_class1(C);
+                linear_trainer_cpa.set_c_class2(1.5*C);
+                linear_trainer.set_c_class1(C/samples.size());
+                linear_trainer.set_c_class2(1.5*C/samples.size());
+
+                df = linear_trainer.train(samples, labels, state);
+                df2 = linear_trainer_cpa.train(samples, labels);
+                df3 = linear_trainer.train(samples, labels);
+
+                DLIB_TEST( dlib::distance(df.basis_vectors(0), df2.basis_vectors(0)) < 1e-8);
+                DLIB_TEST( std::abs(df.b - df2.b) < 1e-8);
+                DLIB_TEST( dlib::distance(df.basis_vectors(0), df3.basis_vectors(0)) < 1e-8);
+                DLIB_TEST( std::abs(df.b - df3.b) < 1e-8);
+            }
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    void test_normal_no_bias()
+    {
+        typedef matrix<double,10,1> sample_type;
+
+
+        typedef linear_kernel<sample_type> kernel_type;
+
+
+
+        svm_c_linear_trainer<kernel_type> linear_trainer_cpa;
+        svm_c_linear_dcd_trainer<kernel_type> linear_trainer;
+
+        svm_c_linear_dcd_trainer<kernel_type>::optimizer_state state;
+
+        const double C = 10;
+        linear_trainer.set_epsilon(1e-10);
+        linear_trainer_cpa.set_epsilon(1e-10);
+
+        linear_trainer.include_bias(false);
+
+
+        std::vector<sample_type> samples, samples_explict_bias;
+        std::vector<double> labels;
+
+        // make an instance of a sample vector so we can use it below
+        sample_type sample;
+
+        decision_function<kernel_type> df, df2, df3;
+
+        dlib::rand rnd;
+        // Now lets go into a loop and randomly generate 10000 samples.
+        double label = +1;
+        for (int i = 0; i < 100; ++i)
+        {
+            // flip this flag
+            label *= -1;
+
+            sample = 0;
+
+            // now make a random sparse sample with at most 10 non-zero elements
+            for (int j = 0; j < 5; ++j)
+            {
+                int idx = rnd.get_random_32bit_number()%9;
+                double value = static_cast<double>(std::rand())/RAND_MAX;
+
+                sample(idx) = label*value;
+            }
+
+            // Also save the samples we are generating so we can let the svm_c_linear_trainer
+            // learn from them below.  
+            samples.push_back(sample);
+            labels.push_back(label);
+
+            sample(9) = -1;
+            samples_explict_bias.push_back(sample);
+
+            if (samples.size() > 1)
+            {
+                linear_trainer_cpa.set_c_class1(C);
+                linear_trainer_cpa.set_c_class2(1.5*C);
+                linear_trainer.set_c_class1(C/samples.size());
+                linear_trainer.set_c_class2(1.5*C/samples.size());
+
+                df = linear_trainer.train(samples_explict_bias, labels, state);
+                df2 = linear_trainer_cpa.train(samples, labels);
+                df3 = linear_trainer.train(samples_explict_bias, labels);
+
+                DLIB_TEST( std::abs(df2.basis_vectors(0)(9)) < 1e-8);
+                DLIB_TEST( max(abs(colm(df.basis_vectors(0),0,9) - colm(df2.basis_vectors(0),0,9))) < 1e-8);
+                DLIB_TEST( std::abs(df.basis_vectors(0)(9) - df2.b) < 1e-8);
+                DLIB_TEST( max(abs(df.basis_vectors(0) - df3.basis_vectors(0))) < 1e-8);
+                DLIB_TEST( std::abs(df.b - df3.b) < 1e-8);
+            }
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    void test_normal()
+    {
+        typedef matrix<double,10,1> sample_type;
+
+
+        typedef linear_kernel<sample_type> kernel_type;
+
+
+
+        svm_c_linear_trainer<kernel_type> linear_trainer_cpa;
+        svm_c_linear_dcd_trainer<kernel_type> linear_trainer;
+
+        svm_c_linear_dcd_trainer<kernel_type>::optimizer_state state;
+
+        const double C = 10;
+        linear_trainer.set_epsilon(1e-10);
+        linear_trainer_cpa.set_epsilon(1e-10);
+
+        std::vector<sample_type> samples;
+        std::vector<double> labels;
+
+        // make an instance of a sample vector so we can use it below
+        sample_type sample;
+
+        decision_function<kernel_type> df, df2, df3;
+
+        dlib::rand rnd;
+        // Now lets go into a loop and randomly generate 10000 samples.
+        double label = +1;
+        for (int i = 0; i < 100; ++i)
+        {
+            // flip this flag
+            label *= -1;
+
+            sample = 0;
+
+            // now make a random sparse sample with at most 10 non-zero elements
+            for (int j = 0; j < 5; ++j)
+            {
+                int idx = rnd.get_random_32bit_number()%10;
+                double value = static_cast<double>(std::rand())/RAND_MAX;
+
+                sample(idx) = label*value;
+            }
+
+            // Also save the samples we are generating so we can let the svm_c_linear_trainer
+            // learn from them below.  
+            samples.push_back(sample);
+            labels.push_back(label);
+
+            if (samples.size() > 1)
+            {
+                linear_trainer_cpa.set_c_class1(C);
+                linear_trainer_cpa.set_c_class2(1.5*C);
+                linear_trainer.set_c_class1(C/samples.size());
+                linear_trainer.set_c_class2(1.5*C/samples.size());
+
+                df = linear_trainer.train(samples, labels, state);
+                df2 = linear_trainer_cpa.train(samples, labels);
+                df3 = linear_trainer.train(samples, labels);
+
+                DLIB_TEST( max(abs(df.basis_vectors(0) - df2.basis_vectors(0))) < 1e-8);
+                DLIB_TEST( std::abs(df.b - df2.b) < 1e-8);
+                DLIB_TEST( max(abs(df.basis_vectors(0) - df3.basis_vectors(0))) < 1e-8);
+                DLIB_TEST( std::abs(df.b - df3.b) < 1e-8);
+            }
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    void test_normal_force_last_weight(bool have_bias, bool force_weight)
+    {
+        typedef matrix<double,10,1> sample_type;
+
+
+        typedef linear_kernel<sample_type> kernel_type;
+
+
+
+        svm_c_linear_dcd_trainer<kernel_type> linear_trainer;
+
+        svm_c_linear_dcd_trainer<kernel_type>::optimizer_state state;
+
+        const double C = 10;
+        linear_trainer.set_epsilon(1e-10);
+
+
+        linear_trainer.force_last_weight_to_1(force_weight);
+        linear_trainer.include_bias(have_bias);
+
+        std::vector<sample_type> samples;
+        std::vector<double> labels;
+
+        // make an instance of a sample vector so we can use it below
+        sample_type sample;
+
+        decision_function<kernel_type> df;
+
+        running_stats<double> rs;
+
+        dlib::rand rnd;
+        // Now lets go into a loop and randomly generate 10000 samples.
+        double label = +1;
+        for (int i = 0; i < 40; ++i)
+        {
+            // flip this flag
+            label *= -1;
+
+            sample = 0;
+
+            // now make a random sparse sample with at most 10 non-zero elements
+            for (int j = 0; j < 5; ++j)
+            {
+                int idx = rnd.get_random_32bit_number()%9;
+                double value = static_cast<double>(std::rand())/RAND_MAX;
+
+                sample(idx) = label*value + label;
+            }
+
+            sample(9) = 4;
+
+            // Also save the samples we are generating so we can let the svm_c_linear_trainer
+            // learn from them below.  
+            samples.push_back(sample);
+            labels.push_back(label);
+
+            linear_trainer.set_c(C);
+
+            df = linear_trainer.train(samples, labels, state);
+
+            if (force_weight)
+                DLIB_TEST(std::abs(df.basis_vectors(0)(9) - 1) < 1e-8);
+
+            if (!have_bias)
+                DLIB_TEST(std::abs(df.b) < 1e-8);
+
+
+            for (unsigned long k = 0; k < samples.size(); ++k)
+            {
+                //cout << "pred: "<< labels[k]*df(samples[k]) << endl;
+                rs.add(labels[k]*df(samples[k]));
+            }
+        }
+        DLIB_TEST(std::abs(rs.min()-1) < 1e-8);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    void test_normal_1_sample(double label)
+    {
+        typedef matrix<double,10,1> sample_type;
+
+
+        typedef linear_kernel<sample_type> kernel_type;
+
+
+
+        svm_c_linear_dcd_trainer<kernel_type> linear_trainer;
+
+        svm_c_linear_dcd_trainer<kernel_type>::optimizer_state state;
+
+        const double C = 10;
+        linear_trainer.set_epsilon(1e-10);
+        linear_trainer.set_c(C);
+
+
+        linear_trainer.force_last_weight_to_1(true);
+        linear_trainer.include_bias(false);
+
+        std::vector<sample_type> samples;
+        std::vector<double> labels;
+
+        // make an instance of a sample vector so we can use it below
+        sample_type sample;
+
+        sample = 0;
+        sample(0) = -1;
+        sample(1) = -1;
+        sample(9) = 4;
+
+        samples.push_back(sample);
+        labels.push_back(label);
+
+        for (int i = 0; i < 4; ++i)
+        {
+            decision_function<kernel_type> df;
+            df = linear_trainer.train(samples, labels);
+
+            if (label > 0)
+            {
+                DLIB_TEST(std::abs(df(samples[0])-4) < 1e-8);
+            }
+            else
+            {
+                DLIB_TEST(std::abs(df(samples[0])+1) < 1e-8);
+            }
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    void test_sparse_1_sample(double label)
+    {
+        typedef std::vector<std::pair<unsigned long,double> > sample_type;
+
+
+        typedef sparse_linear_kernel<sample_type> kernel_type;
+
+
+
+        svm_c_linear_dcd_trainer<kernel_type> linear_trainer;
+
+        svm_c_linear_dcd_trainer<kernel_type>::optimizer_state state;
+
+        const double C = 10;
+        linear_trainer.set_epsilon(1e-10);
+        linear_trainer.set_c(C);
+
+
+        linear_trainer.force_last_weight_to_1(true);
+        linear_trainer.include_bias(false);
+
+        std::vector<sample_type> samples;
+        std::vector<double> labels;
+
+        // make an instance of a sample vector so we can use it below
+        sample_type sample;
+
+        sample.push_back(make_pair(0,-1));
+        sample.push_back(make_pair(1,1));
+        sample.push_back(make_pair(9,4));
+
+        for (int i = 0; i < 4; ++i)
+        {
+            samples.push_back(sample);
+            labels.push_back(label);
+
+            decision_function<kernel_type> df;
+            df = linear_trainer.train(samples, labels);
+
+
+            if (label > 0)
+            {
+                DLIB_TEST(std::abs(df(samples[0])-4) < 1e-8);
+            }
+            else
+            {
+                DLIB_TEST(std::abs(df(samples[0])+1) < 1e-8);
+            }
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    class tester_svm_c_linear_dcd : public tester
+    {
+    public:
+        tester_svm_c_linear_dcd (
+        ) :
+            tester ("test_svm_c_linear_dcd",
+                "Runs tests on the svm_c_linear_dcd_trainer.")
+        {}
+
+        void perform_test (
+        )
+        {
+            test_normal();
+            print_spinner();
+            test_normal_no_bias();
+            print_spinner();
+            test_sparse();
+            print_spinner();
+            test_normal_force_last_weight(false,false);
+            print_spinner();
+            test_normal_force_last_weight(false,true);
+            print_spinner();
+            test_normal_force_last_weight(true,false);
+            print_spinner();
+            test_normal_force_last_weight(true,true);
+            print_spinner();
+            test_normal_1_sample(+1);
+            print_spinner();
+            test_normal_1_sample(-1);
+            print_spinner();
+            test_sparse_1_sample(+1);
+            print_spinner();
+            test_sparse_1_sample(-1);
+            print_spinner();
+        }
+    } a;
+
+}
+
+
+
+