Added the option to make the poly features rotationally invariant.

This change breaks backwards compatibility with the previous version.

dlib/image_keypoint/build_separable_poly_filters.h

@@ -60,8 +60,8 @@ namespace dlib
 
                 if (X.nc() > 5)
                 {
-                    X(cnt, 3) = x*y;
-                    X(cnt, 4) = x*x;
+                    X(cnt, 3) = x*x;
+                    X(cnt, 4) = x*y;
                     X(cnt, 5) = y*y;
                 }
                 if (X.nc() > 9)

dlib/image_keypoint/poly_image.h

@@ -29,11 +29,13 @@ namespace dlib
         poly_image(
             long order_,
             long window_size_,
-            bool normalization = true
+            bool normalization = true,
+            bool rotation_invariance_ = false
         )
         {
             setup(order_, window_size_);
             set_uses_normalization(normalization);
+            set_is_rotationally_invariant(rotation_invariance_);
         }
 
         poly_image (
@@ -46,6 +48,7 @@ namespace dlib
         )
         {
             normalize = true;
+            rotation_invariance = false;
             poly_coef.clear();
             order = 3;
             window_size = 13;
@@ -102,11 +105,22 @@ namespace dlib
             normalize = normalization;
         }
 
+        bool is_rotationally_invariant (
+        ) const { return rotation_invariance; }
+
+        void set_is_rotationally_invariant (
+            bool rotation_invariance_
+        )
+        {
+            rotation_invariance = rotation_invariance_;
+        }
+
         void copy_configuration (
             const poly_image& item
         )
         {
             normalize = item.normalize;
+            rotation_invariance = item.rotation_invariance;
             if (order != item.order || 
                 window_size != item.window_size)
             {
@@ -153,6 +167,9 @@ namespace dlib
                         }
                     }
                 }
+
+                if (rotation_invariance)
+                    rotate_polys(rect);
             }
             else
             {
@@ -163,6 +180,9 @@ namespace dlib
                 }
                 num_rows = rect.height();
                 num_cols = rect.width();
+
+                if (rotation_invariance)
+                    rotate_polys(rect);
             }
         }
 
@@ -273,6 +293,7 @@ namespace dlib
             serialize(item.num_rows, out);
             serialize(item.num_cols, out);
             serialize(item.normalize, out);
+            serialize(item.rotation_invariance, out);
             serialize(item.filters, out);
         }
 
@@ -290,11 +311,303 @@ namespace dlib
             deserialize(item.num_rows, in);
             deserialize(item.num_cols, in);
             deserialize(item.normalize, in);
+            deserialize(item.rotation_invariance, in);
             deserialize(item.filters, in);
         }
 
     private:
 
+        matrix<float,2,1> rotate_order_1 (
+            const matrix<float,2,1>& w,
+            double cos_theta,
+            double sin_theta
+        ) const
+        {
+            const double w1 = w(0);
+            const double w2 = w(1);
+            matrix<double,2,2> M;
+            M = w1,  w2,
+                w2, -w1;
+
+            matrix<double,2,1> x;
+            x = cos_theta, 
+                sin_theta;
+
+            return matrix_cast<float>(M*x);
+        }
+
+        matrix<float,3,1> rotate_order_2 (
+            const matrix<float,3,1>& w,
+            double cos_theta,
+            double sin_theta
+        ) const
+        {
+            const double w1 = w(0);
+            const double w2 = w(1);
+            const double w3 = w(2);
+            matrix<double,3,3> M;
+            M = w1, w2,           w3,
+                w2, (2*w3-2*w1), -w2,
+                w3, -w2,          w1;
+
+            matrix<double,3,1> x;
+            x = std::pow(cos_theta,2.0), 
+                cos_theta*sin_theta,
+                std::pow(sin_theta,2.0);
+
+            return matrix_cast<float>(M*x);
+        }
+
+        matrix<float,4,1> rotate_order_3 (
+            const matrix<float,4,1>& w,
+            double cos_theta,
+            double sin_theta
+        ) const
+        {
+            const double w1 = w(0);
+            const double w2 = w(1);
+            const double w3 = w(2);
+            const double w4 = w(3);
+            matrix<double,4,4> M;
+            M = w1, w2,            w3,           w4,
+                w2, (2*w3-3*w1), (3*w4-2*w2),   -w3,
+                w3, (3*w4-2*w2), (3*w1-2*w3),    w2,
+                w4, -w3,           w2,          -w1;
+
+            matrix<double,4,1> x;
+            x = std::pow(cos_theta,3.0), 
+                std::pow(cos_theta,2.0)*sin_theta,
+                cos_theta*std::pow(sin_theta,2.0),
+                std::pow(sin_theta,3.0);
+
+            return matrix_cast<float>(M*x);
+        }
+
+        matrix<float,5,1> rotate_order_4 (
+            const matrix<float,5,1>& w,
+            double cos_theta,
+            double sin_theta
+        ) const
+        {
+            const double w1 = w(0);
+            const double w2 = w(1);
+            const double w3 = w(2);
+            const double w4 = w(3);
+            const double w5 = w(4);
+            matrix<double,5,5> M;
+            M = w1, w2,              w3,            w4,          w5,
+                w2, (2*w3-4*w1), (3*w4-3*w2),      (4*w5-2*w3), -w4,
+                w3, (3*w4-3*w2), (6*w1-4*w3+6*w5), (3*w2-3*w4),  w3,
+                w4, (4*w5-2*w3), (3*w2-3*w4),      (2*w3-4*w1), -w2,
+                w5, -w4,            w3,            -w2,          w1;
+
+            matrix<double,5,1> x;
+            x = std::pow(cos_theta,4.0), 
+                std::pow(cos_theta,3.0)*sin_theta,
+                std::pow(cos_theta,2.0)*std::pow(sin_theta,2.0),
+                cos_theta*std::pow(sin_theta,3.0),
+                std::pow(sin_theta,4.0);
+
+            return matrix_cast<float>(M*x);
+        }
+
+        matrix<float,6,1> rotate_order_5 (
+            const matrix<float,6,1>& w,
+            double cos_theta,
+            double sin_theta
+        ) const
+        {
+            const double w1 = w(0);
+            const double w2 = w(1);
+            const double w3 = w(2);
+            const double w4 = w(3);
+            const double w5 = w(4);
+            const double w6 = w(5);
+            matrix<double,6,6> M;
+            M = w1,     w2,          w3,             w4,                 w5,          w6,
+                w2, (2*w3-5*w1), (3*w4-4*w2),       (4*w5-3*w3),        (5*w6-2*w4), -w5,
+                w3, (3*w4-4*w2), (10*w1-6*w3+6*w5), (6*w2-6*w4+10*w6),  (3*w3-4*w5),  w4,
+                w4, (4*w5-3*w3), (6*w2-6*w4+10*w6), (-10*w1+6*w3-6*w5), (3*w4-4*w2), -w3,
+                w5, (5*w6-2*w4), (3*w3-4*w5),       (3*w4-4*w2),        (5*w1-2*w3),  w2,
+                w6,    -w5,          w4,            -w3,                 w2,         -w1;
+
+            matrix<double,6,1> x;
+            x = std::pow(cos_theta,5.0), 
+                std::pow(cos_theta,4.0)*sin_theta,
+                std::pow(cos_theta,3.0)*std::pow(sin_theta,2.0),
+                std::pow(cos_theta,2.0)*std::pow(sin_theta,3.0),
+                cos_theta*std::pow(sin_theta,4.0),
+                std::pow(sin_theta,5.0);
+
+            return matrix_cast<float>(M*x);
+        }
+
+        matrix<float,7,1> rotate_order_6 (
+            const matrix<float,7,1>& w,
+            double cos_theta,
+            double sin_theta
+        ) const
+        {
+            const double w1 = w(0);
+            const double w2 = w(1);
+            const double w3 = w(2);
+            const double w4 = w(3);
+            const double w5 = w(4);
+            const double w6 = w(5);
+            const double w7 = w(6);
+            matrix<double,7,7> M;
+            M = w1,     w2,          w3,              w4,                           w5,                 w6,         w7,
+                w2, (2*w3-6*w1), (3*w4-5*w2),        (4*w5-4*w3),                (5*w6-3*w4),         (6*w7-2*w5), -w6,
+                w3, (3*w4-5*w2), (15*w1-8*w3+ 6*w5), ( 10*w2 -9*w4+10*w6),       (  6*w3-8*w5+15*w7), (3*w4-5*w6),  w5,
+                w4, (4*w5-4*w3), (10*w2-9*w4+10*w6), (-20*w1+12*w3-12*w5+20*w7), (-10*w2+9*w4-10*w6), (4*w5-4*w3), -w4,
+                w5, (5*w6-3*w4), ( 6*w3-8*w5+15*w7), (-10*w2 +9*w4-10*w6),       ( 15*w1-8*w3 +6*w5), (5*w2-3*w4),  w3,
+                w6, (6*w7-2*w5), (3*w4-5*w6),        (4*w5-4*w3),                (5*w2-3*w4),         (2*w3-6*w1), -w2,
+                w7,     -w6,          w5,            -w4,                          w3,                 -w2,         w1;
+
+            matrix<double,7,1> x;
+            x = std::pow(cos_theta,6.0), 
+                std::pow(cos_theta,5.0)*sin_theta,
+                std::pow(cos_theta,4.0)*std::pow(sin_theta,2.0),
+                std::pow(cos_theta,3.0)*std::pow(sin_theta,3.0),
+                std::pow(cos_theta,2.0)*std::pow(sin_theta,4.0),
+                cos_theta*std::pow(sin_theta,5.0),
+                std::pow(sin_theta,6.0);
+
+            return matrix_cast<float>(M*x);
+        }
+
+        void rotate_polys (
+            const rectangle& rect
+        )
+        /*!
+            ensures
+                - rotates all the polynomials in poly_coef so that they are
+                  rotationally invariant
+        !*/
+        {
+            // The idea here is to use a rotation matrix to rotate the 
+            // coordinate system for the polynomial so that the x axis
+            // always lines up with the gradient vector (or direction of
+            // max curvature).  This way we can make the representation 
+            // rotation invariant.
+
+            // Note that the rotation matrix is given by:
+            // [ cos_theta -sin_theta ]
+            // [ sin_theta cos_theta  ]
+
+            // need to offset poly_coef to get past the constant term if there isn't any normalization.
+            const int off = (normalize) ? 0 : 1;
+
+            for (long r = rect.top(); r <= rect.bottom(); ++r)
+            {
+                for (long c = rect.left(); c <= rect.right(); ++c)
+                {
+                    dlib::vector<double,2> g(poly_coef[off+0][r][c],
+                                             poly_coef[off+1][r][c]);
+
+                    const double len = g.length();
+                    if (len != 0)
+                    {
+                        g /= len;
+                    }
+                    else
+                    {
+                        g.x() = 1;
+                        g.y() = 0;
+                    }
+                    // since we normalized g we can find the sin/cos of its angle easily. 
+                    const double cos_theta = g.x();
+                    const double sin_theta = g.y();
+
+                    if (order >= 1)
+                    {
+                        matrix<float,2,1> w;
+                        w = poly_coef[off+0][r][c],
+                            poly_coef[off+1][r][c];
+                        w = rotate_order_1(w, cos_theta, sin_theta);
+                        poly_coef[off+0][r][c] = w(0);
+                        poly_coef[off+1][r][c] = w(1);
+                    }
+                    if (order >= 2)
+                    {
+                        matrix<float,3,1> w;
+                        w = poly_coef[off+2][r][c],
+                            poly_coef[off+3][r][c],
+                            poly_coef[off+4][r][c];
+                        w = rotate_order_2(w, cos_theta, sin_theta);
+                        poly_coef[off+2][r][c] = w(0);
+                        poly_coef[off+3][r][c] = w(1);
+                        poly_coef[off+4][r][c] = w(2);
+                    }
+                    if (order >= 3)
+                    {
+                        matrix<float,4,1> w;
+                        w = poly_coef[off+5][r][c],
+                            poly_coef[off+6][r][c],
+                            poly_coef[off+7][r][c],
+                            poly_coef[off+8][r][c];
+                        w = rotate_order_3(w, cos_theta, sin_theta);
+                        poly_coef[off+5][r][c] = w(0);
+                        poly_coef[off+6][r][c] = w(1);
+                        poly_coef[off+7][r][c] = w(2);
+                        poly_coef[off+8][r][c] = w(3);
+                    }
+                    if (order >= 4)
+                    {
+                        matrix<float,5,1> w;
+                        w = poly_coef[off+9][r][c],
+                            poly_coef[off+10][r][c],
+                            poly_coef[off+11][r][c],
+                            poly_coef[off+12][r][c],
+                            poly_coef[off+13][r][c];
+                        w = rotate_order_4(w, cos_theta, sin_theta);
+                        poly_coef[off+9][r][c]  = w(0);
+                        poly_coef[off+10][r][c] = w(1);
+                        poly_coef[off+11][r][c] = w(2);
+                        poly_coef[off+12][r][c] = w(3);
+                        poly_coef[off+13][r][c] = w(4);
+                    }
+                    if (order >= 5)
+                    {
+                        matrix<float,6,1> w;
+                        w = poly_coef[off+14][r][c],
+                            poly_coef[off+15][r][c],
+                            poly_coef[off+16][r][c],
+                            poly_coef[off+17][r][c],
+                            poly_coef[off+18][r][c],
+                            poly_coef[off+19][r][c];
+                        w = rotate_order_5(w, cos_theta, sin_theta);
+                        poly_coef[off+14][r][c] = w(0);
+                        poly_coef[off+15][r][c] = w(1);
+                        poly_coef[off+16][r][c] = w(2);
+                        poly_coef[off+17][r][c] = w(3);
+                        poly_coef[off+18][r][c] = w(4);
+                        poly_coef[off+19][r][c] = w(5);
+                    }
+                    if (order >= 6)
+                    {
+                        matrix<float,7,1> w;
+                        w = poly_coef[off+20][r][c],
+                            poly_coef[off+21][r][c],
+                            poly_coef[off+22][r][c],
+                            poly_coef[off+23][r][c],
+                            poly_coef[off+24][r][c],
+                            poly_coef[off+25][r][c],
+                            poly_coef[off+26][r][c];
+                        w = rotate_order_6(w, cos_theta, sin_theta);
+                        poly_coef[off+20][r][c] = w(0);
+                        poly_coef[off+21][r][c] = w(1);
+                        poly_coef[off+22][r][c] = w(2);
+                        poly_coef[off+23][r][c] = w(3);
+                        poly_coef[off+24][r][c] = w(4);
+                        poly_coef[off+25][r][c] = w(5);
+                        poly_coef[off+26][r][c] = w(6);
+                    }
+                }
+            }
+
+        }
+
         template <typename image_type>
         rectangle filter_image (
             const image_type& img,
@@ -322,6 +635,7 @@ namespace dlib
         long num_cols;
 
         bool normalize;
+        bool rotation_invariance;
 
         mutable descriptor_type des;
     };

dlib/image_keypoint/poly_image_abstract.h

@@ -54,12 +54,14 @@ namespace dlib
                 - #get_window_size() == 13
                 - #size() == 0
                 - #uses_normalization() == true
+                - #is_rotationally_invariant() == false 
         !*/
 
         poly_image(
             long order,
             long window_size,
-            bool normalization = true
+            bool normalization = true,
+            bool rotation_invariance = false
         );
         /*!
             requires
@@ -70,6 +72,7 @@ namespace dlib
                 - #get_window_size() == window_size
                 - #size() == 0
                 - #uses_normalization() == normalization
+                - #is_rotationally_invariant() == rotation_invariance
         !*/
 
         void clear (
@@ -124,6 +127,23 @@ namespace dlib
                 - #uses_normalization() == normalization
         !*/
 
+        bool is_rotationally_invariant (
+        );
+        /*!
+            ensures
+                - returns true if the feature extractor will adjust the output so that it
+                  is rotationally invariant.  This is done by rotating each patch such that
+                  the gradient vector always points in the same direction.
+        !*/
+
+        void set_is_rotationally_invariant (
+            bool rotation_invariance
+        );
+        /*!
+            ensures
+                - #is_rotationally_invariant() == rotation_invariance
+        !*/
+
         void copy_configuration (
             const poly_image& item
         );