object_detection.cpp 19.6 KB
Newer Older
1
// Copyright (C) 2015 Davis E. King (davis@dlib.net)
2
3
4
5
6
7
// License: Boost Software License   See LICENSE.txt for the full license.

#include <dlib/python.h>
#include <dlib/matrix.h>
#include <dlib/geometry.h>
#include <dlib/image_processing/frontal_face_detector.h>
Patrick Snape's avatar
Patrick Snape committed
8
#include "simple_object_detector.h"
9
#include "simple_object_detector_py.h"
Patrick Snape's avatar
Patrick Snape committed
10
#include "conversion.h"
11
12
13

using namespace dlib;
using namespace std;
14
15

namespace py = pybind11;
16
17
18

// ----------------------------------------------------------------------------------------

Patrick Snape's avatar
Patrick Snape committed
19
20
21
22
23
24
25
26
27
string print_simple_test_results(const simple_test_results& r)
{
    std::ostringstream sout;
    sout << "precision: "<<r.precision << ", recall: "<< r.recall << ", average precision: " << r.average_precision;
    return sout.str();
}

// ----------------------------------------------------------------------------------------

28
inline simple_object_detector_py train_simple_object_detector_on_images_py (
29
30
    const py::list& pyimages,
    const py::list& pyboxes,
31
    const simple_object_detector_training_options& options
32
33
)
{
34
35
    const unsigned long num_images = py::len(pyimages);
    if (num_images != py::len(pyboxes))
36
37
38
39
40
        throw dlib::error("The length of the boxes list must match the length of the images list.");

    // We never have any ignore boxes for this version of the API.
    std::vector<std::vector<rectangle> > ignore(num_images), boxes(num_images);
    dlib::array<array2d<rgb_pixel> > images(num_images);
Patrick Snape's avatar
Patrick Snape committed
41
    images_and_nested_params_to_dlib(pyimages, pyboxes, images, boxes);
42

43
    return train_simple_object_detector_on_images("", images, boxes, ignore, options);
44
45
}

46
inline simple_test_results test_simple_object_detector_with_images_py (
47
48
        const py::list& pyimages,
        const py::list& pyboxes,
49
        simple_object_detector& detector,
50
        const unsigned int upsampling_amount
51
52
)
{
53
54
    const unsigned long num_images = py::len(pyimages);
    if (num_images != py::len(pyboxes))
55
56
57
58
59
        throw dlib::error("The length of the boxes list must match the length of the images list.");

    // We never have any ignore boxes for this version of the API.
    std::vector<std::vector<rectangle> > ignore(num_images), boxes(num_images);
    dlib::array<array2d<rgb_pixel> > images(num_images);
Patrick Snape's avatar
Patrick Snape committed
60
    images_and_nested_params_to_dlib(pyimages, pyboxes, images, boxes);
61

62
63
64
65
66
67
    return test_simple_object_detector_with_images(images, upsampling_amount, boxes, ignore, detector);
}

// ----------------------------------------------------------------------------------------

inline simple_test_results test_simple_object_detector_py_with_images_py (
68
69
        const py::list& pyimages,
        const py::list& pyboxes,
70
71
72
73
74
75
76
77
78
79
80
81
82
        simple_object_detector_py& detector,
        const int upsampling_amount
)
{
    // Allow users to pass an upsampling amount ELSE use the one cached on the object
    // Anything less than 0 is ignored and the cached value is used.
    unsigned int final_upsampling_amount = 0;
    if (upsampling_amount >= 0)
        final_upsampling_amount = upsampling_amount;
    else
        final_upsampling_amount = detector.upsampling_amount;

    return test_simple_object_detector_with_images_py(pyimages, pyboxes, detector.detector, final_upsampling_amount);
83
84
}

85
86
// ----------------------------------------------------------------------------------------

87
inline void find_candidate_object_locations_py (
88
89
90
    py::object pyimage,
    py::list& pyboxes,
    py::tuple pykvals,
91
92
93
94
95
96
97
98
99
100
101
102
103
    unsigned long min_size,
    unsigned long max_merging_iterations
)
{
    // Copy the data into dlib based objects
    array2d<rgb_pixel> image;
    if (is_gray_python_image(pyimage))
        assign_image(image, numpy_gray_image(pyimage));
    else if (is_rgb_python_image(pyimage))
        assign_image(image, numpy_rgb_image(pyimage));
    else
        throw dlib::error("Unsupported image type, must be 8bit gray or RGB image.");

104
    if (py::len(pykvals) != 3)
105
106
        throw dlib::error("kvals must be a tuple with three elements for start, end, num.");

107
108
109
    double start = pykvals[0].cast<double>();
    double end   = pykvals[1].cast<double>();
    long num     = pykvals[2].cast<long>();
110
111
112
    matrix_range_exp<double> kvals = linspace(start, end, num);

    std::vector<rectangle> rects;
113
    const long count = py::len(pyboxes);
114
115
116
    // Copy any rectangles in the input pyboxes into rects so that any rectangles will be
    // properly deduped in the resulting output.
    for (long i = 0; i < count; ++i)
117
        rects.push_back(pyboxes[i].cast<rectangle>());
118
    // Find candidate objects
119
120
121
122
123
124
125
126
127
128
    find_candidate_object_locations(image, rects, kvals, min_size, max_merging_iterations);

    // Collect boxes containing candidate objects
    std::vector<rectangle>::iterator iter;
    for (iter = rects.begin(); iter != rects.end(); ++iter)
        pyboxes.append(*iter);
}

// ----------------------------------------------------------------------------------------

129
void bind_object_detection(py::module& m)
130
{
131
132
    {
    typedef simple_object_detector_training_options type;
133
    py::class_<type>(m, "simple_object_detector_training_options",
Davis King's avatar
Davis King committed
134
        "This object is a container for the options to the train_simple_object_detector() routine.")
135
136
        .def(py::init())
        .def_readwrite("be_verbose", &type::be_verbose,
137
"If true, train_simple_object_detector() will print out a lot of information to the screen while training.")
138
        .def_readwrite("add_left_right_image_flips", &type::add_left_right_image_flips,
Davis King's avatar
Davis King committed
139
140
"if true, train_simple_object_detector() will assume the objects are \n\
left/right symmetric and add in left right flips of the training \n\
141
images.  This doubles the size of the training dataset.")
142
        .def_readwrite("detection_window_size", &type::detection_window_size,
Davis King's avatar
Davis King committed
143
                                               "The sliding window used will have about this many pixels inside it.")
144
        .def_readwrite("C", &type::C,
Davis King's avatar
Davis King committed
145
146
147
148
"C is the usual SVM C regularization parameter.  So it is passed to \n\
structural_object_detection_trainer::set_c().  Larger values of C \n\
will encourage the trainer to fit the data better but might lead to \n\
overfitting.  Therefore, you must determine the proper setting of \n\
149
this parameter experimentally.")
150
        .def_readwrite("epsilon", &type::epsilon,
151
"epsilon is the stopping epsilon.  Smaller values make the trainer's \n\
152
solver more accurate but might take longer to train.")
153
        .def_readwrite("num_threads", &type::num_threads,
Davis King's avatar
Davis King committed
154
155
"train_simple_object_detector() will use this many threads of \n\
execution.  Set this to the number of CPU cores on your machine to \n\
156
obtain the fastest training speed.")
157
        .def_readwrite("upsample_limit", &type::upsample_limit,
158
159
160
161
162
163
"train_simple_object_detector() will upsample images if needed \n\
no more than upsample_limit times. Value 0 will forbid trainer to \n\
upsample any images. If trainer is unable to fit all boxes with \n\
required upsample_limit, exception will be thrown. Higher values \n\
of upsample_limit exponentially increases memory requiremens. \n\
Values higher than 2 (default) are not recommended.");
164
    }
165
    {
166
    typedef simple_test_results type;
167
168
169
170
    py::class_<type>(m, "simple_test_results")
        .def_readwrite("precision", &type::precision)
        .def_readwrite("recall", &type::recall)
        .def_readwrite("average_precision", &type::average_precision)
171
        .def("__str__", &::print_simple_test_results);
172
173
    }

174
    // Here, kvals is actually the result of linspace(start, end, num) and it is different from kvals used
175
    // in find_candidate_object_locations(). See dlib/image_transforms/segment_image_abstract.h for more details.
176
    m.def("find_candidate_object_locations", find_candidate_object_locations_py, py::arg("image"), py::arg("rects"), py::arg("kvals")=py::make_tuple(50, 200, 3), py::arg("min_size")=20, py::arg("max_merging_iterations")=50,
177
178
179
"Returns found candidate objects\n\
requires\n\
    - image == an image object which is a numpy ndarray\n\
180
181
182
    - len(kvals) == 3\n\
    - kvals should be a tuple that specifies the range of k values to use.  In\n\
      particular, it should take the form (start, end, num) where num > 0. \n\
183
184
185
186
187
188
189
190
191
192
ensures\n\
    - This function takes an input image and generates a set of candidate\n\
      rectangles which are expected to bound any objects in the image.  It does\n\
      this by running a version of the segment_image() routine on the image and\n\
      then reports rectangles containing each of the segments as well as rectangles\n\
      containing unions of adjacent segments.  The basic idea is described in the\n\
      paper: \n\
          Segmentation as Selective Search for Object Recognition by Koen E. A. van de Sande, et al.\n\
      Note that this function deviates from what is described in the paper slightly. \n\
      See the code for details.\n\
193
194
195
    - The basic segmentation is performed kvals[2] times, each time with the k parameter\n\
      (see segment_image() and the Felzenszwalb paper for details on k) set to a different\n\
      value from the range of numbers linearly spaced between kvals[0] to kvals[1].\n\
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
    - When doing the basic segmentations prior to any box merging, we discard all\n\
      rectangles that have an area < min_size.  Therefore, all outputs and\n\
      subsequent merged rectangles are built out of rectangles that contain at\n\
      least min_size pixels.  Note that setting min_size to a smaller value than\n\
      you might otherwise be interested in using can be useful since it allows a\n\
      larger number of possible merged boxes to be created.\n\
    - There are max_merging_iterations rounds of neighboring blob merging.\n\
      Therefore, this parameter has some effect on the number of output rectangles\n\
      you get, with larger values of the parameter giving more output rectangles.\n\
    - This function appends the output rectangles into #rects.  This means that any\n\
      rectangles in rects before this function was called will still be in there\n\
      after it terminates.  Note further that #rects will not contain any duplicate\n\
      rectangles.  That is, for all valid i and j where i != j it will be true\n\
      that:\n\
        - #rects[i] != rects[j]");

212
    m.def("get_frontal_face_detector", get_frontal_face_detector,
213
214
        "Returns the default face detector");

215
216
    m.def("train_simple_object_detector", train_simple_object_detector,
        py::arg("dataset_filename"), py::arg("detector_output_filename"), py::arg("options"),
217
218
219
220
221
222
223
224
225
226
227
"requires \n\
    - options.C > 0 \n\
ensures \n\
    - Uses the structural_object_detection_trainer to train a \n\
      simple_object_detector based on the labeled images in the XML file \n\
      dataset_filename.  This function assumes the file dataset_filename is in the \n\
      XML format produced by dlib's save_image_dataset_metadata() routine. \n\
    - This function will apply a reasonable set of default parameters and \n\
      preprocessing techniques to the training procedure for simple_object_detector \n\
      objects.  So the point of this function is to provide you with a very easy \n\
      way to train a basic object detector.   \n\
228
    - The trained object detector is serialized to the file detector_output_filename.");
229

230
231
    m.def("train_simple_object_detector", train_simple_object_detector_on_images_py,
        py::arg("images"), py::arg("boxes"), py::arg("options"),
232
233
234
235
236
237
238
239
240
241
242
243
"requires \n\
    - options.C > 0 \n\
    - len(images) == len(boxes) \n\
    - images should be a list of numpy matrices that represent images, either RGB or grayscale. \n\
    - boxes should be a list of lists of dlib.rectangle object. \n\
ensures \n\
    - Uses the structural_object_detection_trainer to train a \n\
      simple_object_detector based on the labeled images and bounding boxes.  \n\
    - This function will apply a reasonable set of default parameters and \n\
      preprocessing techniques to the training procedure for simple_object_detector \n\
      objects.  So the point of this function is to provide you with a very easy \n\
      way to train a basic object detector.   \n\
244
    - The trained object detector is returned.");
245

246
    m.def("test_simple_object_detector", test_simple_object_detector,
247
            // Please see test_simple_object_detector for the reason upsampling_amount is -1
248
        py::arg("dataset_filename"), py::arg("detector_filename"), py::arg("upsampling_amount")=-1,
249
            "requires \n\
250
                - Optionally, take the number of times to upsample the testing images (upsampling_amount >= 0). \n\
251
             ensures \n\
252
253
254
255
256
257
258
259
260
261
262
                - Loads an image dataset from dataset_filename.  We assume dataset_filename is \n\
                  a file using the XML format written by save_image_dataset_metadata(). \n\
                - Loads a simple_object_detector from the file detector_filename.  This means \n\
                  detector_filename should be a file produced by the train_simple_object_detector()  \n\
                  routine. \n\
                - This function tests the detector against the dataset and returns the \n\
                  precision, recall, and average precision of the detector.  In fact, The \n\
                  return value of this function is identical to that of dlib's \n\
                  test_object_detection_function() routine.  Therefore, see the documentation \n\
                  for test_object_detection_function() for a detailed definition of these \n\
                  metrics. "
263
        );
264

265
266
    m.def("test_simple_object_detector", test_simple_object_detector_with_images_py,
            py::arg("images"), py::arg("boxes"), py::arg("detector"), py::arg("upsampling_amount")=0,
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
            "requires \n\
               - len(images) == len(boxes) \n\
               - images should be a list of numpy matrices that represent images, either RGB or grayscale. \n\
               - boxes should be a list of lists of dlib.rectangle object. \n\
               - Optionally, take the number of times to upsample the testing images (upsampling_amount >= 0). \n\
             ensures \n\
               - Loads a simple_object_detector from the file detector_filename.  This means \n\
                 detector_filename should be a file produced by the train_simple_object_detector() \n\
                 routine. \n\
               - This function tests the detector against the dataset and returns the \n\
                 precision, recall, and average precision of the detector.  In fact, The \n\
                 return value of this function is identical to that of dlib's \n\
                 test_object_detection_function() routine.  Therefore, see the documentation \n\
                 for test_object_detection_function() for a detailed definition of these \n\
                 metrics. "
    );

284
    m.def("test_simple_object_detector", test_simple_object_detector_py_with_images_py,
285
            // Please see test_simple_object_detector_py_with_images_py for the reason upsampling_amount is -1
286
            py::arg("images"), py::arg("boxes"), py::arg("detector"), py::arg("upsampling_amount")=-1,
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
            "requires \n\
               - len(images) == len(boxes) \n\
               - images should be a list of numpy matrices that represent images, either RGB or grayscale. \n\
               - boxes should be a list of lists of dlib.rectangle object. \n\
             ensures \n\
               - Loads a simple_object_detector from the file detector_filename.  This means \n\
                 detector_filename should be a file produced by the train_simple_object_detector() \n\
                 routine. \n\
               - This function tests the detector against the dataset and returns the \n\
                 precision, recall, and average precision of the detector.  In fact, The \n\
                 return value of this function is identical to that of dlib's \n\
                 test_object_detection_function() routine.  Therefore, see the documentation \n\
                 for test_object_detection_function() for a detailed definition of these \n\
                 metrics. "
    );
302
    {
Patrick Snape's avatar
Patrick Snape committed
303
    typedef simple_object_detector type;
304
    py::class_<type, std::shared_ptr<type>>(m, "fhog_object_detector",
305
        "This object represents a sliding window histogram-of-oriented-gradients based object detector.")
306
        .def(py::init(&load_object_from_file<type>),
307
308
309
"Loads an object detector from a file that contains the output of the \n\
train_simple_object_detector() routine or a serialized C++ object of type\n\
object_detector<scan_fhog_pyramid<pyramid_down<6>>>.")
310
        .def("__call__", run_detector_with_upscale2, py::arg("image"), py::arg("upsample_num_times")=0,
311
312
313
314
315
316
317
"requires \n\
    - image is a numpy ndarray containing either an 8bit grayscale or RGB \n\
      image. \n\
    - upsample_num_times >= 0 \n\
ensures \n\
    - This function runs the object detector on the input image and returns \n\
      a list of detections.   \n\
318
    - Upsamples the image upsample_num_times before running the basic \n\
Davis King's avatar
Davis King committed
319
      detector.")
320
       .def("run", run_rect_detector, py::arg("image"), py::arg("upsample_num_times")=0, py::arg("adjust_threshold")=0.0,
321
322
323
324
325
326
327
"requires \n\
    - image is a numpy ndarray containing either an 8bit grayscale or RGB \n\
      image. \n\
    - upsample_num_times >= 0 \n\
ensures \n\
    - This function runs the object detector on the input image and returns \n\
      a tuple of (list of detections, list of scores, list of weight_indices).   \n\
328
    - Upsamples the image upsample_num_times before running the basic \n\
Davis King's avatar
Davis King committed
329
      detector.")
330
       .def_static("run_multiple", run_multiple_rect_detectors, py::arg("detectors"),  py::arg("image"), py::arg("upsample_num_times")=0, py::arg("adjust_threshold")=0.0,
331
332
333
334
335
336
337
338
339
"requires \n\
    - detectors is a list of detectors. \n\
    - image is a numpy ndarray containing either an 8bit grayscale or RGB \n\
      image. \n\
    - upsample_num_times >= 0 \n\
ensures \n\
    - This function runs the list of object detectors at once on the input image and returns \n\
      a tuple of (list of detections, list of scores, list of weight_indices).   \n\
    - Upsamples the image upsample_num_times before running the basic \n\
Davis King's avatar
Davis King committed
340
      detector.")
341
342
           .def("save", save_simple_object_detector, py::arg("detector_output_filename"), "Save a simple_object_detector to the provided path.")
           .def(py::pickle(&getstate<type>, &setstate<type>));
343
    }
344
    {
345
    typedef simple_object_detector_py type;
346
    py::class_<type, std::shared_ptr<type>>(m, "simple_object_detector",
347
        "This object represents a sliding window histogram-of-oriented-gradients based object detector.")
348
        .def(py::init(&load_object_from_file<type>),
349
350
"Loads a simple_object_detector from a file that contains the output of the \n\
train_simple_object_detector() routine.")
351
        .def("__call__", &type::run_detector1, py::arg("image"), py::arg("upsample_num_times"),
352
353
354
355
356
357
358
359
360
361
362
"requires \n\
    - image is a numpy ndarray containing either an 8bit grayscale or RGB \n\
      image. \n\
    - upsample_num_times >= 0 \n\
ensures \n\
    - This function runs the object detector on the input image and returns \n\
      a list of detections.   \n\
    - Upsamples the image upsample_num_times before running the basic \n\
      detector.  If you don't know how many times you want to upsample then \n\
      don't provide a value for upsample_num_times and an appropriate \n\
      default will be used.")
363
        .def("__call__", &type::run_detector2, py::arg("image"),
364
365
366
367
368
369
"requires \n\
    - image is a numpy ndarray containing either an 8bit grayscale or RGB \n\
      image. \n\
ensures \n\
    - This function runs the object detector on the input image and returns \n\
      a list of detections.")
370
371
        .def("save", save_simple_object_detector_py, py::arg("detector_output_filename"), "Save a simple_object_detector to the provided path.")
        .def(py::pickle(&getstate<type>, &setstate<type>));
372
    }
373
374
375
}

// ----------------------------------------------------------------------------------------