metadata_editor.cpp 22.3 KB
Newer Older
1
2
3
4
5
6
// Copyright (C) 2011  Davis E. King (davis@dlib.net)
// License: Boost Software License   See LICENSE.txt for the full license.

#include "metadata_editor.h"
#include <dlib/array.h>
#include <dlib/queue.h>
7
#include <dlib/static_set.h>
8
9
10
11
#include <dlib/misc_api.h>
#include <dlib/image_io.h>
#include <dlib/array2d.h>
#include <dlib/pixel.h>
12
#include <dlib/image_transforms.h>
13
#include <dlib/image_processing.h>
Davis King's avatar
Davis King committed
14
#include <sstream>
15
#include <ctime>
16
17
18
19

using namespace std;
using namespace dlib;

Davis King's avatar
Davis King committed
20
21
extern const char* VERSION;

22
23
24
25
26
27
28
29
// ----------------------------------------------------------------------------------------

metadata_editor::
metadata_editor(
    const std::string& filename_
) : 
    mbar(*this),
    lb_images(*this),
30
31
32
    image_pos(0),
    display(*this),
    overlay_label_name(*this),
33
34
35
    overlay_label(*this),
    keyboard_jump_pos(0),
    last_keyboard_jump_pos_update(0)
36
{
37
38
    file metadata_file(filename_);
    filename = metadata_file.full_name();
39
40
41
    // Make our current directory be the one that contains the metadata file.  We 
    // do this because that file might contain relative paths to the image files
    // we are supposed to be loading.
42
43
    set_current_dir(get_parent_directory(metadata_file).full_name());

44
45
46
47
48
49
50
51
52
53
54
55
56
    load_image_dataset_metadata(metadata, filename);

    dlib::array<std::string>::expand_1a files;
    files.resize(metadata.images.size());
    for (unsigned long i = 0; i < metadata.images.size(); ++i)
    {
        files[i] = metadata.images[i].filename;
    }
    lb_images.load(files);
    lb_images.enable_multiple_select();

    lb_images.set_click_handler(*this, &metadata_editor::on_lb_images_clicked);

57
58
    overlay_label_name.set_text("Next Label: ");
    overlay_label.set_width(200);
59

60
    display.set_image_clicked_handler(*this, &metadata_editor::on_image_clicked);
61
    display.set_overlay_rects_changed_handler(*this, &metadata_editor::on_overlay_rects_changed);
62
    display.set_overlay_rect_selected_handler(*this, &metadata_editor::on_overlay_rect_selected);
63
64
    overlay_label.set_text_modified_handler(*this, &metadata_editor::on_overlay_label_changed);

Davis King's avatar
Davis King committed
65
    mbar.set_number_of_menus(2);
66
    mbar.set_menu_name(0,"File",'F');
Davis King's avatar
Davis King committed
67
    mbar.set_menu_name(1,"Help",'H');
68
69
70
71
72
73
74


    mbar.menu(0).add_menu_item(menu_item_text("Save",*this,&metadata_editor::file_save,'S'));
    mbar.menu(0).add_menu_item(menu_item_text("Save As",*this,&metadata_editor::file_save_as,'A'));
    mbar.menu(0).add_menu_item(menu_item_separator());
    mbar.menu(0).add_menu_item(menu_item_text("Remove Selected Images",*this,&metadata_editor::remove_selected_images,'R'));
    mbar.menu(0).add_menu_item(menu_item_separator());
75
    mbar.menu(0).add_menu_item(menu_item_text("Exit",static_cast<base_window&>(*this),&drawable_window::close_window,'x'));
76

Davis King's avatar
Davis King committed
77
    mbar.menu(1).add_menu_item(menu_item_text("About",*this,&metadata_editor::display_about,'A'));
78

79
    // set the size of this window.
80
    on_window_resized();
Davis King's avatar
Davis King committed
81
    load_image_and_set_size(0);
82
83
84
    on_window_resized();
    if (image_pos < lb_images.size() )
        lb_images.select(image_pos);
85

86
87
88
89
90
91
92
    // make sure the window is centered on the screen.
    unsigned long width, height;
    get_size(width, height);
    unsigned long screen_width, screen_height;
    get_display_size(screen_width, screen_height);
    set_pos((screen_width-width)/2, (screen_height-height)/2);

93
94
95
96
97
98
99
100
101
102
103
104
105
106
    show();
} 

// ----------------------------------------------------------------------------------------

metadata_editor::
~metadata_editor(
)
{
    close_window();
}

// ----------------------------------------------------------------------------------------

107
108
109
110
111
112
113
114
115
116
void metadata_editor::
add_labelable_part_name (
    const std::string& name
)
{
    display.add_labelable_part_name(name);
}

// ----------------------------------------------------------------------------------------

117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
void metadata_editor::
file_save()
{
    save_metadata_to_file(filename);
}

// ----------------------------------------------------------------------------------------

void metadata_editor::
save_metadata_to_file (
    const std::string& file
)
{
    try
    {
        save_image_dataset_metadata(metadata, file);
    }
    catch (dlib::error& e)
    {
        message_box("Error saving file", e.what());
    }
}

// ----------------------------------------------------------------------------------------

void metadata_editor::
file_save_as()
{
    save_file_box(*this, &metadata_editor::save_metadata_to_file);
}

// ----------------------------------------------------------------------------------------

void metadata_editor::
remove_selected_images()
{
153
154
155
    dlib::queue<unsigned long>::kernel_1a list;
    lb_images.get_selected(list);
    list.reset();
156
    unsigned long min_idx = lb_images.size();
157
158
159
    while (list.move_next())
    {
        lb_images.unselect(list.element());
160
        min_idx = std::min(min_idx, list.element());
161
162
    }

163

164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
    // remove all the selected items from metadata.images
    dlib::static_set<unsigned long>::kernel_1a to_remove;
    to_remove.load(list);
    std::vector<dlib::image_dataset_metadata::image> images;
    for (unsigned long i = 0; i < metadata.images.size(); ++i)
    {
        if (to_remove.is_member(i) == false)
        {
            images.push_back(metadata.images[i]);
        }
    }
    images.swap(metadata.images);


    // reload metadata into lb_images
    dlib::array<std::string>::expand_1a files;
    files.resize(metadata.images.size());
    for (unsigned long i = 0; i < metadata.images.size(); ++i)
    {
        files[i] = metadata.images[i].filename;
    }
    lb_images.load(files);


188
189
190
    if (min_idx != 0)
        min_idx--;
    select_image(min_idx);
191
192
193
194
195
196
197
198
199
200
201
202
203
204
}

// ----------------------------------------------------------------------------------------

void metadata_editor::
on_window_resized(
)
{
    drawable_window::on_window_resized();

    unsigned long width, height;
    get_size(width, height);

    lb_images.set_pos(0,mbar.bottom()+1);
205
    lb_images.set_size(180, height - mbar.height());
206
207
208
209
210
211

    overlay_label_name.set_pos(lb_images.right()+10, mbar.bottom() + (overlay_label.height()-overlay_label_name.height())/2+1);
    overlay_label.set_pos(overlay_label_name.right(), mbar.bottom()+1);
    display.set_pos(lb_images.right(), overlay_label.bottom()+3);

    display.set_size(width - display.left(), height - display.top());
212
213
214
215
}

// ----------------------------------------------------------------------------------------

216
217
218
219
220
221
222
223
224
225
226
void propagate_boxes(
    dlib::image_dataset_metadata::dataset& data,
    unsigned long prev,
    unsigned long next
)
{
    if (prev == next || next >= data.images.size())
        return;

    array2d<rgb_pixel> img1, img2;
    dlib::load_image(img1, data.images[prev].filename);
Adrià Arrufat's avatar
Adrià Arrufat committed
227
228
229
    data.images[prev].width = img1.nc();
    data.images[prev].height = img1.nr();

230
    dlib::load_image(img2, data.images[next].filename);
Adrià Arrufat's avatar
Adrià Arrufat committed
231
232
    data.images[next].width = img2.nc();
    data.images[next].height = img2.nr();
233
234
235
236
237
238
239
240
241
242
243
244
245
    for (unsigned long i = 0; i < data.images[prev].boxes.size(); ++i)
    {
        correlation_tracker tracker;
        tracker.start_track(img1, data.images[prev].boxes[i].rect);
        tracker.update(img2);
        dlib::image_dataset_metadata::box box = data.images[prev].boxes[i];
        box.rect = tracker.get_position();
        data.images[next].boxes.push_back(box);
    }
}

// ----------------------------------------------------------------------------------------

246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
void propagate_labels(
    const std::string& label,
    dlib::image_dataset_metadata::dataset& data,
    unsigned long prev,
    unsigned long next
)
{
    if (prev == next || next >= data.images.size())
        return;


    for (unsigned long i = 0; i < data.images[prev].boxes.size(); ++i)
    {
        if (data.images[prev].boxes[i].label != label)
            continue;

        // figure out which box in the next image matches the current one the best
        const rectangle cur = data.images[prev].boxes[i].rect;
        double best_overlap = 0;
        unsigned long best_idx = 0;
        for (unsigned long j = 0; j < data.images[next].boxes.size(); ++j)
        {
            const rectangle next_box = data.images[next].boxes[j].rect;
            const double overlap = cur.intersect(next_box).area()/(double)(cur+next_box).area();
            if (overlap > best_overlap)
            {
                best_overlap = overlap;
                best_idx = j;
            }
        }

        // If we found a matching rectangle in the next image and the best match doesn't
        // already have a label.
        if (best_overlap > 0.5 && data.images[next].boxes[best_idx].label == "")
        {
            data.images[next].boxes[best_idx].label = label;
        }
    }

}

287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
// ----------------------------------------------------------------------------------------

bool has_label_or_all_boxes_labeled (
    const std::string& label,
    const dlib::image_dataset_metadata::image& img 
)
{
    if (label.size() == 0)
        return true;

    bool all_boxes_labeled = true;
    for (unsigned long i = 0; i < img.boxes.size(); ++i)
    {
        if (img.boxes[i].label == label)
            return true;
        if (img.boxes[i].label.size() == 0)
            all_boxes_labeled = false;
    }

    return all_boxes_labeled;
}

// ----------------------------------------------------------------------------------------

311
312
313
314
315
316
317
318
319
320
void metadata_editor::
on_keydown (
    unsigned long key,
    bool is_printable,
    unsigned long state
)
{
    drawable_window::on_keydown(key, is_printable, state);

    if (is_printable)
321
322
323
324
325
326
327
    {
        if (key == '\t')
        {
            overlay_label.give_input_focus();
            overlay_label.select_all_text();
        }

328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
        // If the user types a number then jump to that image.
        if ('0' <= key && key <= '9' && metadata.images.size() != 0 && !overlay_label.has_input_focus())
        {
            time_t curtime = time(0);
            // If it's been a while since the user typed numbers then forget the last jump
            // position and start accumulating numbers over again.
            if (curtime-last_keyboard_jump_pos_update >= 2)
                keyboard_jump_pos = 0;
            last_keyboard_jump_pos_update = curtime;

            keyboard_jump_pos *= 10;
            keyboard_jump_pos += key-'0';
            if (keyboard_jump_pos >= metadata.images.size())
                keyboard_jump_pos = metadata.images.size()-1;

            image_pos = keyboard_jump_pos;
            select_image(image_pos);
        }
        else
        {
            last_keyboard_jump_pos_update = 0;
        }

351
352
353
354
355
356
357
358
359
360
        if (key == '=')
        {
            display.zoom_in();
        }

        if (key == '-')
        {
            display.zoom_out();
        }

361
        if (key == 'd' && (state&base_window::KBD_MOD_ALT))
362
363
364
365
        {
            remove_selected_images();
        }

366
367
368
369
370
371
        if (key == 'e' && !overlay_label.has_input_focus())
        {
            display_equialized_image = !display_equialized_image;
            select_image(image_pos);
        }

372
373
374
375
376
377
378
379
380
381
382
383
384
        // Make 'w' and 's' act like KEY_UP and KEY_DOWN
        if ((key == 'w' || key == 'W') && !overlay_label.has_input_focus())
        {
            key = base_window::KEY_UP;
        }
        else if ((key == 's' || key == 'S') && !overlay_label.has_input_focus())
        {
            key = base_window::KEY_DOWN;
        }
        else
        {
            return;
        }
385
    }
386
387
388

    if (key == base_window::KEY_UP)
    {
389
390
391
392
393
394
395
396
397
398
399
400
        if ((state&KBD_MOD_CONTROL) && (state&KBD_MOD_SHIFT))
        {
            // Don't do anything if there are no boxes in the current image.
            if (metadata.images[image_pos].boxes.size() == 0)
                return;
            // Also don't do anything if there *are* boxes in the next image.
            if (image_pos > 1 && metadata.images[image_pos-1].boxes.size() != 0)
                return;

            propagate_boxes(metadata, image_pos, image_pos-1);
        }
        else if (state&base_window::KBD_MOD_CONTROL)
401
402
403
404
405
406
407
408
409
410
        {
            // If the label we are supposed to propagate doesn't exist in the current image
            // then don't advance.
            if (!has_label_or_all_boxes_labeled(display.get_default_overlay_rect_label(),metadata.images[image_pos]))
                return;

            // if the next image is going to be empty then fast forward to the next one
            while (image_pos > 1 && metadata.images[image_pos-1].boxes.size() == 0)
                --image_pos;

411
            propagate_labels(display.get_default_overlay_rect_label(), metadata, image_pos, image_pos-1);
412
        }
413
414
415
416
        select_image(image_pos-1);
    }
    else if (key == base_window::KEY_DOWN)
    {
417
418
419
420
421
422
423
424
425
426
427
428
        if ((state&KBD_MOD_CONTROL) && (state&KBD_MOD_SHIFT))
        {
            // Don't do anything if there are no boxes in the current image.
            if (metadata.images[image_pos].boxes.size() == 0)
                return;
            // Also don't do anything if there *are* boxes in the next image.
            if (image_pos+1 < metadata.images.size() && metadata.images[image_pos+1].boxes.size() != 0)
                return;

            propagate_boxes(metadata, image_pos, image_pos+1);
        }
        else if (state&base_window::KBD_MOD_CONTROL)
429
430
431
432
433
434
435
436
437
438
        {
            // If the label we are supposed to propagate doesn't exist in the current image
            // then don't advance.
            if (!has_label_or_all_boxes_labeled(display.get_default_overlay_rect_label(),metadata.images[image_pos]))
                return;

            // if the next image is going to be empty then fast forward to the next one
            while (image_pos+1 < metadata.images.size() && metadata.images[image_pos+1].boxes.size() == 0)
                ++image_pos;

439
            propagate_labels(display.get_default_overlay_rect_label(), metadata, image_pos, image_pos+1);
440
        }
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
        select_image(image_pos+1);
    }
}

// ----------------------------------------------------------------------------------------

void metadata_editor::
select_image(
    unsigned long idx
)
{
    if (idx < lb_images.size())
    {
        // unselect all currently selected images
        dlib::queue<unsigned long>::kernel_1a list;
        lb_images.get_selected(list);
        list.reset();
        while (list.move_next())
        {
            lb_images.unselect(list.element());
        }


        lb_images.select(idx);
465
        load_image(idx);
466
    }
Davis King's avatar
Davis King committed
467
    else if (lb_images.size() == 0)
Davis King's avatar
Davis King committed
468
469
470
471
472
    {
        display.clear_overlay();
        array2d<unsigned char> empty_img;
        display.set_image(empty_img);
    }
473
474
475
476
477
478
479
480
481
}

// ----------------------------------------------------------------------------------------

void metadata_editor::
on_lb_images_clicked(
    unsigned long idx
) 
{ 
482
    load_image(idx);
483
484
485
486
}

// ----------------------------------------------------------------------------------------

Davis King's avatar
Davis King committed
487
std::vector<dlib::image_display::overlay_rect> get_overlays (
488
489
    const dlib::image_dataset_metadata::image& data,
    color_mapper& string_to_color 
Davis King's avatar
Davis King committed
490
491
492
493
494
495
496
)
{
    std::vector<dlib::image_display::overlay_rect> temp(data.boxes.size());
    for (unsigned long i = 0; i < temp.size(); ++i)
    {
        temp[i].rect = data.boxes[i].rect;
        temp[i].label = data.boxes[i].label;
497
        temp[i].parts = data.boxes[i].parts;
498
        temp[i].crossed_out = data.boxes[i].ignore;
499
        temp[i].color = string_to_color(data.boxes[i].label);
Davis King's avatar
Davis King committed
500
501
502
503
504
505
    }
    return temp;
}

// ----------------------------------------------------------------------------------------

506
507
508
509
510
511
512
513
void metadata_editor::
load_image(
    unsigned long idx
)
{
    if (idx >= metadata.images.size())
        return;

Davis King's avatar
Davis King committed
514
515
    image_pos = idx; 

516
517
518
519
520
    array2d<rgb_pixel> img;
    display.clear_overlay();
    try
    {
        dlib::load_image(img, metadata.images[idx].filename);
Adrià Arrufat's avatar
Adrià Arrufat committed
521
522
        metadata.images[idx].width = img.nc();
        metadata.images[idx].height = img.nr();
523
        set_title(metadata.name + " #"+cast_to_string(idx)+": " +metadata.images[idx].filename);
524
525
526
527
528
529
    }
    catch (exception& e)
    {
        message_box("Error loading image", e.what());
    }

530
531
    if (display_equialized_image)
        equalize_histogram(img);
532
    display.set_image(img);
533
    display.add_overlay(get_overlays(metadata.images[idx], string_to_color));
534
535
536
537
538
539
540
541
542
543
544
545
}

// ----------------------------------------------------------------------------------------

void metadata_editor::
load_image_and_set_size(
    unsigned long idx
)
{
    if (idx >= metadata.images.size())
        return;

Davis King's avatar
Davis King committed
546
547
    image_pos = idx; 

548
549
550
551
552
    array2d<rgb_pixel> img;
    display.clear_overlay();
    try
    {
        dlib::load_image(img, metadata.images[idx].filename);
Adrià Arrufat's avatar
Adrià Arrufat committed
553
554
        metadata.images[idx].width = img.nc();
        metadata.images[idx].height = img.nr();
555
        set_title(metadata.name + " #"+cast_to_string(idx)+": " +metadata.images[idx].filename);
556
557
558
559
560
561
    }
    catch (exception& e)
    {
        message_box("Error loading image", e.what());
    }

562

563
564
    unsigned long screen_width, screen_height;
    get_display_size(screen_width, screen_height);
565

566

567
568
569
570
571
    unsigned long needed_width = display.left() + img.nc() + 4;
    unsigned long needed_height = display.top() + img.nr() + 4;
	if (needed_width < 300) needed_width = 300;
	if (needed_height < 300) needed_height = 300;

572
573
574
575
576
577
    if (needed_width > 100 + screen_width)
        needed_width = screen_width - 100;
    if (needed_height > 100 + screen_height)
        needed_height = screen_height - 100;

    set_size(needed_width, needed_height);
578

579

580
581
    if (display_equialized_image)
        equalize_histogram(img);
582
    display.set_image(img);
583
    display.add_overlay(get_overlays(metadata.images[idx], string_to_color));
584
585
586
587
}

// ----------------------------------------------------------------------------------------

588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
void metadata_editor::
on_overlay_rects_changed(
)
{
    using namespace dlib::image_dataset_metadata;
    if (image_pos < metadata.images.size())
    {
        const std::vector<image_display::overlay_rect>& rects = display.get_overlay_rects();

        std::vector<box>& boxes = metadata.images[image_pos].boxes;

        boxes.clear();
        for (unsigned long i = 0; i < rects.size(); ++i)
        {
            box temp;
            temp.label = rects[i].label;
            temp.rect = rects[i].rect;
605
            temp.parts = rects[i].parts;
606
            temp.ignore = rects[i].crossed_out;
607
608
609
610
611
612
613
            boxes.push_back(temp);
        }
    }
}

// ----------------------------------------------------------------------------------------

614
615
616
617
618
619
620
621
622
623
void metadata_editor::
on_image_clicked(
    const point& /*p*/, bool /*is_double_click*/, unsigned long /*btn*/
)
{
    display.set_default_overlay_rect_color(string_to_color(trim(overlay_label.text())));
}

// ----------------------------------------------------------------------------------------

624
625
626
627
628
629
630
631
void metadata_editor::
on_overlay_label_changed(
)
{
    display.set_default_overlay_rect_label(trim(overlay_label.text()));
}

// ----------------------------------------------------------------------------------------
632

633
634
635
636
637
638
639
void metadata_editor::
on_overlay_rect_selected(
    const image_display::overlay_rect& orect
)
{
    overlay_label.set_text(orect.label);
    display.set_default_overlay_rect_label(orect.label);
640
    display.set_default_overlay_rect_color(string_to_color(orect.label));
641
642
643
644
}

// ----------------------------------------------------------------------------------------

Davis King's avatar
Davis King committed
645
646
647
648
649
650
651
652
void metadata_editor::
display_about(
)
{
    std::ostringstream sout;
    sout << wrap_string("Image Labeler v" + string(VERSION) + "." ,0,0) << endl << endl;
    sout << wrap_string("This program is a tool for labeling images with rectangles. " ,0,0) << endl << endl;

653
654
655
656
657
    sout << wrap_string("You can add a new rectangle by holding the shift key, left clicking "
                        "the mouse, and dragging it.  New rectangles are given the label from the \"Next Label\" "
                        "field at the top of the application.  You can quickly edit the contents of the Next Label field "
                        "by hitting the tab key. Double clicking "
                        "a rectangle selects it and the delete key removes it.  You can also mark "
658
                        "a rectangle as ignored by hitting the i or END keys when it is selected.  Ignored "
659
                        "rectangles are visually displayed with an X through them.  You can remove an image "
660
                        "entirely by selecting it in the list on the left and pressing alt+d."
Davis King's avatar
Davis King committed
661
662
                        ,0,0) << endl << endl;

663
664
665
    sout << wrap_string("It is also possible to label object parts by selecting a rectangle and "
                        "then right clicking.  A popup menu will appear and you can select a part label. "
                        "Note that you must define the allowable part labels by giving --parts on the "
666
667
668
669
670
                        "command line.  An example would be '--parts \"leye reye nose mouth\"'. "
                        "Alternatively, if you don't give --parts you can simply select a rectangle and shift+left "
                        "click to add parts. Parts added this way will be labeled with integer labels starting from 0. "
                        "You can only use this simpler part adding mode if all the parts in a rectangle are already "
                        "labeled with integer labels or the rectangle has no parts at all."
671
672
                        ,0,0) << endl << endl;

673
674
675
    sout << wrap_string("Press the down or s key to select the next image in the list and the up or w "
                        "key to select the previous one.",0,0) << endl << endl;

676
    sout << wrap_string("Additionally, you can hold ctrl and then scroll the mouse wheel to zoom.  A normal left click "
677
678
                        "and drag allows you to navigate around the image.  Holding ctrl and "
                        "left clicking a rectangle will give it the label from the Next Label field. "
679
                        "Holding shift + right click and then dragging allows you to move things around. "
680
681
                        "Holding ctrl and pressing the up or down keyboard keys will propagate "
                        "rectangle labels from one image to the next and also skip empty images. " 
682
683
                        "Similarly, holding ctrl+shift will propagate entire boxes via a visual tracking " 
                        "algorithm from one image to the next. "
684
685
686
687
688
                        "Finally, typing a number on the keyboard will jump you to a specific image.",0,0) << endl << endl;

    sout << wrap_string("You can also toggle image histogram equalization by pressing the e key."
                        ,0,0) << endl;

Davis King's avatar
Davis King committed
689
690
691
692
693
694

    message_box("About Image Labeler",sout.str());
}

// ----------------------------------------------------------------------------------------