main.cpp 51.6 KB
Newer Older
1

2
#include "dlib/data_io.h"
3
#include "dlib/string.h"
4
#include "metadata_editor.h"
Davis King's avatar
Davis King committed
5
#include "convert_pascal_xml.h"
6
#include "convert_pascal_v1.h"
7
#include "convert_idl.h"
8
#include "cluster.h"
9
#include "flip_dataset.h"
Davis King's avatar
Davis King committed
10
#include <dlib/cmd_line_parser.h>
Davis King's avatar
Davis King committed
11
#include <dlib/image_transforms.h>
12
#include <dlib/svm.h>
13
#include <dlib/console_progress_indicator.h>
Davis King's avatar
Davis King committed
14
#include <dlib/md5.h>
15

16
17
#include <iostream>
#include <fstream>
18
#include <string>
19
#include <set>
20

21
#include <dlib/dir_nav.h>
22
23


24
const char* VERSION = "1.19";
Davis King's avatar
Davis King committed
25
26


27
28
29
30

using namespace std;
using namespace dlib;

31
32
// ----------------------------------------------------------------------------------------

Davis King's avatar
Davis King committed
33
void create_new_dataset (
Davis King's avatar
Davis King committed
34
    const command_line_parser& parser
Davis King's avatar
Davis King committed
35
36
37
38
39
40
41
42
)
{
    using namespace dlib::image_dataset_metadata;

    const std::string filename = parser.option("c").argument();
    // make sure the file exists so we can use the get_parent_directory() command to
    // figure out it's parent directory.
    make_empty_file(filename);
Davis King's avatar
Davis King committed
43
    const std::string parent_dir = get_parent_directory(file(filename));
Davis King's avatar
Davis King committed
44
45
46
47
48
49
50
51
52
53
54
55

    unsigned long depth = 0;
    if (parser.option("r"))
        depth = 30;

    dataset meta;
    meta.name = "imglab dataset";
    meta.comment = "Created by imglab tool.";
    for (unsigned long i = 0; i < parser.number_of_arguments(); ++i)
    {
        try
        {
Davis King's avatar
Davis King committed
56
            const string temp = strip_path(file(parser[i]), parent_dir);
Davis King's avatar
Davis King committed
57
58
59
60
61
62
            meta.images.push_back(image(temp));
        }
        catch (dlib::file::file_not_found&)
        {
            // then parser[i] should be a directory

63
64
            std::vector<file> files = get_files_in_directory_tree(parser[i],
                                                                  match_endings(".png .PNG .jpeg .JPEG .jpg .JPG .bmp .BMP .dng .DNG .gif .GIF .webp .WEBP"),
Davis King's avatar
Davis King committed
65
66
67
68
69
                                                                  depth);
            sort(files.begin(), files.end());

            for (unsigned long j = 0; j < files.size(); ++j)
            {
Davis King's avatar
Davis King committed
70
                meta.images.push_back(image(strip_path(files[j], parent_dir)));
Davis King's avatar
Davis King committed
71
72
73
74
75
76
77
78
            }
        }
    }

    save_image_dataset_metadata(meta, filename);
}

// ----------------------------------------------------------------------------------------
79

80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
int split_dataset (
    const command_line_parser& parser
)
{
    if (parser.number_of_arguments() != 1)
    {
        cerr << "The --split option requires you to give one XML file on the command line." << endl;
        return EXIT_FAILURE;
    }

    const std::string label = parser.option("split").argument();

    dlib::image_dataset_metadata::dataset data, data_with, data_without;
    load_image_dataset_metadata(data, parser[0]);

    data_with.name = data.name;
    data_with.comment = data.comment;
    data_without.name = data.name;
    data_without.comment = data.comment;

    for (unsigned long i = 0; i < data.images.size(); ++i)
    {
102
        auto&& temp = data.images[i];
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

        bool has_the_label = false;
        // check for the label we are looking for
        for (unsigned long j = 0; j < temp.boxes.size(); ++j)
        {
            if (temp.boxes[j].label == label)
            {
                has_the_label = true;
                break;
            }
        }

        if (has_the_label)
            data_with.images.push_back(temp);
        else
            data_without.images.push_back(temp);
    }


    save_image_dataset_metadata(data_with, left_substr(parser[0],".") + "_with_"+label + ".xml");
    save_image_dataset_metadata(data_without, left_substr(parser[0],".") + "_without_"+label + ".xml");

    return EXIT_SUCCESS;
}

// ----------------------------------------------------------------------------------------

130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
int make_train_test_splits (
    const command_line_parser& parser
)
{
    if (parser.number_of_arguments() != 1)
    {
        cerr << "The --split-train-test option requires you to give one XML file on the command line." << endl;
        return EXIT_FAILURE;
    }

    const double train_frac = get_option(parser, "split-train-test", 0.5);

    dlib::image_dataset_metadata::dataset data, data_train, data_test;
    load_image_dataset_metadata(data, parser[0]);

    data_train.name = data.name;
    data_train.comment = data.comment;
    data_test.name = data.name;
    data_test.comment = data.comment;

    const unsigned long num_train_images = static_cast<unsigned long>(std::round(train_frac*data.images.size()));

    for (unsigned long i = 0; i < data.images.size(); ++i)
    {
        if (i < num_train_images)
            data_train.images.push_back(data.images[i]);
        else
            data_test.images.push_back(data.images[i]);
    }

    save_image_dataset_metadata(data_train, left_substr(parser[0],".") + "_train.xml");
    save_image_dataset_metadata(data_test, left_substr(parser[0],".") + "_test.xml");

    return EXIT_SUCCESS;
}

// ----------------------------------------------------------------------------------------

168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
void print_all_labels (
    const dlib::image_dataset_metadata::dataset& data
)
{
    std::set<std::string> labels;
    for (unsigned long i = 0; i < data.images.size(); ++i)
    {
        for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
        {
            labels.insert(data.images[i].boxes[j].label);
        }
    }

    for (std::set<std::string>::iterator i = labels.begin(); i != labels.end(); ++i)
    {
        if (i->size() != 0)
        {
            cout << *i << endl;
        }
    }
}

// ----------------------------------------------------------------------------------------

192
193
194
195
196
void print_all_label_stats (
    const dlib::image_dataset_metadata::dataset& data
)
{
    std::map<std::string, running_stats<double> > area_stats, aspect_ratio;
197
    std::map<std::string, int> image_hits;
198
    std::set<std::string> labels;
199
    unsigned long num_unignored_boxes = 0;
200
201
    for (unsigned long i = 0; i < data.images.size(); ++i)
    {
202
        std::set<std::string> temp;
203
204
205
        for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
        {
            labels.insert(data.images[i].boxes[j].label);
206
            temp.insert(data.images[i].boxes[j].label);
207
208
209
210

            area_stats[data.images[i].boxes[j].label].add(data.images[i].boxes[j].rect.area());
            aspect_ratio[data.images[i].boxes[j].label].add(data.images[i].boxes[j].rect.width()/
                                                    (double)data.images[i].boxes[j].rect.height());
211
212
213

            if (!data.images[i].boxes[j].ignore)
                ++num_unignored_boxes;
214
        }
215
216
217
218

        // count the number of images for each label
        for (std::set<std::string>::iterator i = temp.begin(); i != temp.end(); ++i)
            image_hits[*i] += 1;
219
220
    }

221
    cout << "Number of images: "<< data.images.size() << endl;
222
223
    cout << "Number of different labels: "<< labels.size() << endl;
    cout << "Number of non-ignored boxes: " << num_unignored_boxes << endl << endl;
224
225
226

    for (std::set<std::string>::iterator i = labels.begin(); i != labels.end(); ++i)
    {
227
228
229
        if (i->size() == 0)
            cout << "Unlabeled Boxes:" << endl;
        else
230
            cout << "Label: "<< *i << endl;
231
232
233
234
235
236
237
238
239
        cout << "   number of images:      " << image_hits[*i] << endl;
        cout << "   number of occurrences: " << area_stats[*i].current_n() << endl;
        cout << "   min box area:    " << area_stats[*i].min() << endl;
        cout << "   max box area:    " << area_stats[*i].max() << endl;
        cout << "   mean box area:   " << area_stats[*i].mean() << endl;
        cout << "   stddev box area: " << area_stats[*i].stddev() << endl;
        cout << "   mean width/height ratio:   " << aspect_ratio[*i].mean() << endl;
        cout << "   stddev width/height ratio: " << aspect_ratio[*i].stddev() << endl;
        cout << endl;
240
241
242
243
244
    }
}

// ----------------------------------------------------------------------------------------

Davis King's avatar
Davis King committed
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
void rename_labels (
    dlib::image_dataset_metadata::dataset& data,
    const std::string& from,
    const std::string& to
)
{
    for (unsigned long i = 0; i < data.images.size(); ++i)
    {
        for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
        {
            if (data.images[i].boxes[j].label == from)
                data.images[i].boxes[j].label = to;
        }
    }

}

// ----------------------------------------------------------------------------------------

264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
void ignore_labels (
    dlib::image_dataset_metadata::dataset& data,
    const std::string& label
)
{
    for (unsigned long i = 0; i < data.images.size(); ++i)
    {
        for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
        {
            if (data.images[i].boxes[j].label == label)
                data.images[i].boxes[j].ignore = true;
        }
    }
}

// ----------------------------------------------------------------------------------------

281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
void merge_metadata_files (
    const command_line_parser& parser
)
{
    image_dataset_metadata::dataset src, dest;
    load_image_dataset_metadata(src, parser.option("add").argument(0));
    load_image_dataset_metadata(dest, parser.option("add").argument(1));

    std::map<string,image_dataset_metadata::image> merged_data;
    for (unsigned long i = 0; i < dest.images.size(); ++i)
        merged_data[dest.images[i].filename] = dest.images[i];
    // now add in the src data and overwrite anything if there are duplicate entries.
    for (unsigned long i = 0; i < src.images.size(); ++i)
        merged_data[src.images[i].filename] = src.images[i];

    // copy merged data into dest
    dest.images.clear();
    for (std::map<string,image_dataset_metadata::image>::const_iterator i = merged_data.begin(); 
        i != merged_data.end(); ++i)
    {
        dest.images.push_back(i->second);
    }

    save_image_dataset_metadata(dest, "merged.xml");
}

// ----------------------------------------------------------------------------------------

309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
void rotate_dataset(const command_line_parser& parser)
{
    image_dataset_metadata::dataset metadata;
    const string datasource = parser[0];
    load_image_dataset_metadata(metadata,datasource);

    double angle = get_option(parser, "rotate", 0);

    // Set the current directory to be the one that contains the
    // metadata file. We do this because the file might contain
    // file paths which are relative to this folder.
    set_current_dir(get_parent_directory(file(datasource)));

    const string file_prefix = "rotated_"+ cast_to_string(angle) + "_";
    const string metadata_filename = get_parent_directory(file(datasource)).full_name() +
        directory::get_separator() + file_prefix + file(datasource).name();


    array2d<rgb_pixel> img, temp;
    for (unsigned long i = 0; i < metadata.images.size(); ++i)
    {
        file f(metadata.images[i].filename);
331
        string filename = get_parent_directory(f).full_name() + directory::get_separator() + file_prefix + to_png_name(f.name());
332
333
334

        load_image(img, metadata.images[i].filename);
        const point_transform_affine tran = rotate_image(img, temp, angle*pi/180);
Adrià Arrufat's avatar
Adrià Arrufat committed
335
336
        metadata.images[i].width = temp.nc();
        metadata.images[i].height = temp.nr();
337
338
339
340
341
        if (parser.option("jpg"))
        {
            filename = to_jpg_name(filename);
            save_jpeg(temp, filename,JPEG_QUALITY);
        }
342
#ifdef DLIB_WEBP_SUPPORT
343
344
345
346
347
348
        else if (parser.option("webp"))
        {
            filename = to_webp_name(filename);
            const float webp_quality = std::stof(parser.option("webp").argument());
            save_webp(temp, filename, webp_quality);
        }
349
#endif
350
351
352
353
        else
        {
            save_png(temp, filename);
        }
354

Davis King's avatar
Davis King committed
355
        rectangle_transform rtran = tran;
356
357
        for (unsigned long j = 0; j < metadata.images[i].boxes.size(); ++j)
        {
Davis King's avatar
Davis King committed
358
359
360
361
            metadata.images[i].boxes[j].rect = rtran(metadata.images[i].boxes[j].rect);

            for (auto& p : metadata.images[i].boxes[j].parts)
                p.second = tran(p.second);
362
363
364
365
366
367
368
369
370
371
        }

        metadata.images[i].filename = filename;
    }

    save_image_dataset_metadata(metadata, metadata_filename);
}

// ----------------------------------------------------------------------------------------

Adrià Arrufat's avatar
Adrià Arrufat committed
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
void add_width_and_height_metadata(const command_line_parser& parser)
{
    for (unsigned long i = 0; i < parser.number_of_arguments(); ++i) {
        image_dataset_metadata::dataset metadata;
        const string datasource = parser[i];
        load_image_dataset_metadata(metadata,datasource);

        // Set the current directory to be the one that contains the
        // metadata file. We do this because the file might contain
        // file paths which are relative to this folder.
        set_current_dir(get_parent_directory(file(datasource)));

        parallel_for(0, metadata.images.size(), [&](long i) 
        {
            array2d<rgb_pixel> img;
            load_image(img, metadata.images[i].filename);
            metadata.images[i].width = img.nc();
            metadata.images[i].height = img.nr();
        });

        save_image_dataset_metadata(metadata, datasource);
    }
}

// ----------------------------------------------------------------------------------------

Davis King's avatar
Davis King committed
398
399
400
401
402
403
404
405
int resample_dataset(const command_line_parser& parser)
{
    if (parser.number_of_arguments() != 1)
    {
        cerr << "The --resample option requires you to give one XML file on the command line." << endl;
        return EXIT_FAILURE;
    }

Davis King's avatar
Davis King committed
406
407
    const size_t obj_size = get_option(parser,"cropped-object-size",100*100); 
    const double margin_scale =  get_option(parser,"crop-size",2.5); // cropped image will be this times wider than the object.
408
    const unsigned long min_object_size = get_option(parser,"min-object-size",1);
409
    const bool one_object_per_image = parser.option("one-object-per-image");
Davis King's avatar
Davis King committed
410
411

    dlib::image_dataset_metadata::dataset data, resampled_data;
Davis King's avatar
Davis King committed
412
413
414
415
416
    std::ostringstream sout;
    sout << "\nThe --resample parameters which generated this dataset were:" << endl;
    sout << "   cropped-object-size: "<< obj_size << endl;
    sout << "   crop-size: "<< margin_scale << endl;
    sout << "   min-object-size: "<< min_object_size << endl;
417
418
    if (one_object_per_image)
        sout << "   one_object_per_image: true" << endl;
Davis King's avatar
Davis King committed
419
    resampled_data.comment = data.comment + sout.str();
Davis King's avatar
Davis King committed
420
421
422
423
    resampled_data.name = data.name + " RESAMPLED";

    load_image_dataset_metadata(data, parser[0]);
    locally_change_current_dir chdir(get_parent_directory(file(parser[0])));
424
    dlib::rand rnd;
Davis King's avatar
Davis King committed
425

426
427
428
    const size_t image_size = std::round(std::sqrt(obj_size*margin_scale*margin_scale));
    const chip_dims cdims(image_size, image_size);

Davis King's avatar
Davis King committed
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
    console_progress_indicator pbar(data.images.size());
    for (unsigned long i = 0; i < data.images.size(); ++i)
    {
        // don't even bother loading images that don't have objects.
        if (data.images[i].boxes.size() == 0)
            continue;

        pbar.print_status(i);
        array2d<rgb_pixel> img, chip;
        load_image(img, data.images[i].filename);


        // figure out what chips we want to take from this image
        for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
        {
            const rectangle rect = data.images[i].boxes[j].rect;
445
            if (data.images[i].boxes[j].ignore || rect.area() < min_object_size)
Davis King's avatar
Davis King committed
446
447
                continue;

448
            const auto max_dim = std::max(rect.width(), rect.height());
449

450
            const double rand_scale_perturb = 1 - 0.3*(rnd.get_random_double()-0.5);
451
            const rectangle crop_rect = centered_rect(rect, max_dim*margin_scale*rand_scale_perturb, max_dim*margin_scale*rand_scale_perturb);
Davis King's avatar
Davis King committed
452

453
454
            const rectangle_transform tform = get_mapping_to_chip(chip_details(crop_rect, cdims));
            extract_image_chip(img, chip_details(crop_rect, cdims), chip);
Davis King's avatar
Davis King committed
455
456
457
458
459
460
461
462
463
464
465
466
467

            image_dataset_metadata::image dimg;
            // Now transform the boxes to the crop and also mark them as ignored if they
            // have already been cropped out or are outside the crop.
            for (size_t k = 0; k < data.images[i].boxes.size(); ++k)
            {
                image_dataset_metadata::box box = data.images[i].boxes[k];
                // ignore boxes outside the cropped image
                if (crop_rect.intersect(box.rect).area() == 0)
                    continue;

                // mark boxes we include in the crop as ignored.  Also mark boxes that
                // aren't totally within the crop as ignored.
468
                if (crop_rect.contains(grow_rect(box.rect,10)) && (!one_object_per_image || k==j))
Davis King's avatar
Davis King committed
469
470
471
472
                    data.images[i].boxes[k].ignore = true;
                else
                    box.ignore = true;

473
474
475
                if (box.rect.area() < min_object_size)
                    box.ignore = true;

Davis King's avatar
Davis King committed
476
                box.rect = tform(box.rect);
477
478
                for (auto&& p : box.parts)
                    p.second = tform.get_tform()(p.second);
Davis King's avatar
Davis King committed
479
480
                dimg.boxes.push_back(box);
            }
481
482
483
484
485
            // Put a 64bit hash of the image data into the name to make sure there are no
            // file name conflicts.
            std::ostringstream sout;
            sout << hex << murmur_hash3_128bit(&chip[0][0], chip.size()*sizeof(chip[0][0])).second;
            dimg.filename = data.images[i].filename + "_RESAMPLED_"+sout.str()+".png";
Adrià Arrufat's avatar
Adrià Arrufat committed
486
487
            dimg.width = chip.nc();
            dimg.height = chip.nr();
Davis King's avatar
Davis King committed
488

489
490
491
492
493
            if (parser.option("jpg"))
            {
                dimg.filename = to_jpg_name(dimg.filename);
                save_jpeg(chip,dimg.filename, JPEG_QUALITY);
            }
494
#ifdef DLIB_WEBP_SUPPORT
495
496
497
498
499
500
            else if (parser.option("webp"))
            {
                dimg.filename = to_webp_name(dimg.filename);
                const float webp_quality = std::stof(parser.option("webp").argument());
                save_webp(chip, dimg.filename, webp_quality);
            }
501
#endif
502
503
504
505
            else
            {
                save_png(chip,dimg.filename);
            }
Davis King's avatar
Davis King committed
506
507
508
509
510
511
512
513
514
515
            resampled_data.images.push_back(dimg);
        }
    }

    save_image_dataset_metadata(resampled_data, parser[0] + ".RESAMPLED.xml");

    return EXIT_SUCCESS;
}

// ----------------------------------------------------------------------------------------
516
517
518
519
520
521
522
523
524
525
526

int tile_dataset(const command_line_parser& parser)
{
    if (parser.number_of_arguments() != 1)
    {
        cerr << "The --tile option requires you to give one XML file on the command line." << endl;
        return EXIT_FAILURE;
    }

    string out_image = parser.option("tile").argument();
    string ext = right_substr(out_image,".");
527
    if (ext != "png" && ext != "jpg" && ext != "webp")
528
    {
529
        cerr << "The output image file must have either .png, .jpg or .webp extension." << endl;
530
531
532
        return EXIT_FAILURE;
    }

533
    const unsigned long chip_size = get_option(parser, "size", 8000);
534
535
536
537
538
539
540
541

    dlib::image_dataset_metadata::dataset data;
    load_image_dataset_metadata(data, parser[0]);
    locally_change_current_dir chdir(get_parent_directory(file(parser[0])));
    dlib::array<array2d<rgb_pixel> > images;
    console_progress_indicator pbar(data.images.size());
    for (unsigned long i = 0; i < data.images.size(); ++i)
    {
Davis King's avatar
Davis King committed
542
543
544
545
        // don't even bother loading images that don't have objects.
        if (data.images[i].boxes.size() == 0)
            continue;

546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
        pbar.print_status(i);
        array2d<rgb_pixel> img;
        load_image(img, data.images[i].filename);

        // figure out what chips we want to take from this image
        std::vector<chip_details> dets;
        for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
        {
            if (data.images[i].boxes[j].ignore)
                continue;

            rectangle rect = data.images[i].boxes[j].rect;
            dets.push_back(chip_details(rect, chip_size));
        }
        // Now grab all those chips at once.
        dlib::array<array2d<rgb_pixel> > chips;
        extract_image_chips(img, dets, chips);
        // and put the chips into the output.
        for (unsigned long j = 0; j < chips.size(); ++j)
            images.push_back(chips[j]);
    }

    chdir.revert();

    if (ext == "png")
571
    {
572
        save_png(tile_images(images), out_image);
573
    }
574
#ifdef DLIB_WEBP_SUPPORT
575
576
577
578
579
580
    else if (ext == "webp")
    {
        // Lossless by default
        const float webp_quality = get_option(parser, "webp", 101.f);
        save_webp(tile_images(images), out_image, webp_quality);
    }
581
#endif
582
    else
583
584
585
    {
        save_jpeg(tile_images(images), out_image, JPEG_QUALITY);
    }
586
587
588
589
590

    return EXIT_SUCCESS;
}


Davis King's avatar
Davis King committed
591
592
// ----------------------------------------------------------------------------------------

593
594
595
596
597
int main(int argc, char** argv)
{
    try
    {

Davis King's avatar
Davis King committed
598
        command_line_parser parser;
599
600

        parser.add_option("h","Displays this information.");
601
602
603
        parser.add_option("v","Display version.");

        parser.set_group_name("Creating XML files");
604
        parser.add_option("c","Create an XML file named <arg> listing a set of images.",1);
605
        parser.add_option("r","Search directories recursively for images.");
606
607
608
        parser.add_option("convert","Convert foreign image Annotations from <arg> format to the imglab format. "
                          "Supported formats: pascal-xml, pascal-v1, idl.",1);

609
        parser.set_group_name("Viewing XML files");
610
        parser.add_option("tile","Chip out all the objects and save them as one big image called <arg>.",1);
611
612
        parser.add_option("size","When using --tile or --cluster, make each extracted object contain "
                                 "about <arg> pixels (default 8000).",1);
613
        parser.add_option("l","List all the labels in the given XML file.");
614
        parser.add_option("stats","List detailed statistics on the object labels in the given XML file.");
Davis King's avatar
Davis King committed
615
        parser.add_option("files","List all the files in the given XML file.");
616

Davis King's avatar
Davis King committed
617
        parser.set_group_name("Editing/Transforming XML datasets");
Davis King's avatar
Davis King committed
618
        parser.add_option("rename", "Rename all labels of <arg1> to <arg2>.",2);
619
        parser.add_option("parts","The display will allow image parts to be labeled.  The set of allowable parts "
Davis King's avatar
Davis King committed
620
                          "is defined by <arg> which should be a space separated list of parts.",1);
Davis King's avatar
Davis King committed
621
        parser.add_option("rmempty","Remove all images that don't contain non-ignored annotations and save the results to a new XML file.");
Davis King's avatar
Davis King committed
622
623
        parser.add_option("rmdupes","Remove duplicate images from the dataset.  This is done by comparing "
                                    "the md5 hash of each image file and removing duplicate images. " );
624
625
        parser.add_option("rmdiff","Set the ignored flag to true for boxes marked as difficult.");
        parser.add_option("rmtrunc","Set the ignored flag to true for boxes that are partially outside the image.");
Davis King's avatar
Davis King committed
626
        parser.add_option("box-images","Add a box to each image that contains the entire image.");
627
        parser.add_option("sort-num-objects","Sort the images listed an XML file so images with many objects are listed first.");
628
        parser.add_option("sort","Alphabetically sort the images in an XML file.");
629
        parser.add_option("shuffle","Randomly shuffle the order of the images listed in an XML file.");
630
631
        parser.add_option("seed", "When using --shuffle, set the random seed to the string <arg>.",1);
        parser.add_option("split", "Split the contents of an XML file into two separate files.  One containing the "
632
            "images with objects labeled <arg> and another file with all the other images. ",1);
633
634
635
636
        parser.add_option("split-train-test", "Split the contents of an XML file into two separate files.  A training "
            "file containing <arg> fraction of the images and a testing file containing the remaining (1-<arg>) images. "
            "The partitioning is done deterministically by putting the first images in the input xml file into the training split "
            "and the later images into the test split.",1);
637
638
639
640
        parser.add_option("add", "Add the image metadata from <arg1> into <arg2>.  If any of the image "
                                 "tags are in both files then the ones in <arg2> are deleted and replaced with the "
                                 "image tags from <arg1>.  The results are saved into merged.xml and neither <arg1> or "
                                 "<arg2> files are modified.",2);
Davis King's avatar
Davis King committed
641
        parser.add_option("flip", "Read an XML image dataset from the <arg> XML file and output a left-right flipped "
642
643
644
645
646
                                  "version of the dataset and an accompanying flipped XML file named flipped_<arg>. " 
                                  "We also adjust object part labels after flipping so that the new flipped dataset "
                                  "has the same average part layout as the source dataset." ,1);
        parser.add_option("flip-basic", "This option is just like --flip, except we don't adjust any object part labels after flipping. "
                                        "The parts are instead simply mirrored to the flipped dataset.", 1);
647
648
        parser.add_option("rotate", "Read an XML image dataset and output a copy that is rotated counter clockwise by <arg> degrees. "
                                  "The output is saved to an XML file prefixed with rotated_<arg>.",1);
Adrià Arrufat's avatar
Adrià Arrufat committed
649
650
        parser.add_option("add-width-height-metadata", "Open the given xml files and set the width and height image metadata fields "
                            "for every image. This involves loading each image to find these values.");
651
        parser.add_option("cluster", "Cluster all the objects in an XML file into <arg> different clusters (pass 0 to find automatically) and save "
652
                                     "the results as cluster_###.xml and cluster_###.jpg files.",1);
Davis King's avatar
Davis King committed
653
        parser.add_option("ignore", "Mark boxes labeled as <arg> as ignored.  The resulting XML file is output as a separate file and the original is not modified.",1);
654
        parser.add_option("rmlabel","Remove all boxes labeled <arg> and save the results to a new XML file.",1);
655
        parser.add_option("rm-other-labels","Remove all boxes not labeled <arg> and save the results to a new XML file.",1);
Davis King's avatar
Davis King committed
656
        parser.add_option("rmignore","Remove all boxes marked ignore and save the results to a new XML file.");
657
        parser.add_option("rm-if-overlaps","Remove all boxes labeled <arg> if they overlap any box not labeled <arg> and save the results to a new XML file.",1);
Davis King's avatar
Davis King committed
658
        parser.add_option("jpg", "When saving images to disk, write them as jpg files instead of png.");
659
#ifdef DLIB_WEBP_SUPPORT
660
        parser.add_option("webp", "When saving images to disk, write them as webp files instead of png or jpg, using <arg> as the quality factor.", 1);
661
#endif
Davis King's avatar
Davis King committed
662
663
664
665
666

        parser.set_group_name("Cropping sub images");
        parser.add_option("resample", "Crop out images that are centered on each object in the dataset. "
                                      "The output is a new XML dataset."); 
        parser.add_option("cropped-object-size", "When doing --resample, make the cropped objects contain about <arg> pixels (default 10000).",1);
667
        parser.add_option("min-object-size", "When doing --resample, skip objects that have fewer than <arg> pixels in them (default 1).",1);
Davis King's avatar
Davis King committed
668
        parser.add_option("crop-size", "When doing --resample, the entire cropped image will be <arg> times wider than the object (default 2.5).",1); 
669
        parser.add_option("one-object-per-image", "When doing --resample, only include one non-ignored object per image (i.e. the central object).");
Davis King's avatar
Davis King committed
670
671


672
673
674

        parser.parse(argc, argv);

Davis King's avatar
Davis King committed
675
        const char* singles[] = {"h","c","r","l","files","convert","parts","rmdiff", "rmtrunc", "rmdupes", "seed", "shuffle", "split", "add", 
676
                                 "flip-basic", "flip", "rotate", "tile", "size", "cluster", "resample", "min-object-size", "rmempty",
677
                                 "crop-size", "cropped-object-size", "rmlabel", "rm-other-labels", "rm-if-overlaps", "sort-num-objects", 
Adrià Arrufat's avatar
Adrià Arrufat committed
678
                                 "one-object-per-image", "jpg", "rmignore", "sort", "split-train-test", "box-images", "add-width-height-metadata"};
679
        parser.check_one_time_options(singles);
680
681
        const char* c_sub_ops[] = {"r", "convert"};
        parser.check_sub_options("c", c_sub_ops);
682
        parser.check_sub_option("shuffle", "seed");
683
        const char* resample_sub_ops[] = {"min-object-size", "crop-size", "cropped-object-size", "one-object-per-image"};
Davis King's avatar
Davis King committed
684
        parser.check_sub_options("resample", resample_sub_ops);
685
686
        const char* size_parent_ops[] = {"tile", "cluster"};
        parser.check_sub_options(size_parent_ops, "size");
Davis King's avatar
Davis King committed
687
        parser.check_incompatible_options("c", "l");
Davis King's avatar
Davis King committed
688
        parser.check_incompatible_options("c", "files");
689
        parser.check_incompatible_options("c", "rmdiff");
Davis King's avatar
Davis King committed
690
        parser.check_incompatible_options("c", "rmempty");
691
        parser.check_incompatible_options("c", "rmlabel");
692
        parser.check_incompatible_options("c", "rm-other-labels");
Davis King's avatar
Davis King committed
693
        parser.check_incompatible_options("c", "rmignore");
694
        parser.check_incompatible_options("c", "rm-if-overlaps");
Davis King's avatar
Davis King committed
695
        parser.check_incompatible_options("c", "rmdupes");
696
        parser.check_incompatible_options("c", "rmtrunc");
Davis King's avatar
Davis King committed
697
        parser.check_incompatible_options("c", "box-images");
698
        parser.check_incompatible_options("c", "add");
Davis King's avatar
Davis King committed
699
        parser.check_incompatible_options("c", "flip");
700
701
        parser.check_incompatible_options("c", "flip-basic");
        parser.check_incompatible_options("flip", "flip-basic");
702
        parser.check_incompatible_options("c", "rotate");
Adrià Arrufat's avatar
Adrià Arrufat committed
703
        parser.check_incompatible_options("c", "add-width-height-metadata");
Davis King's avatar
Davis King committed
704
        parser.check_incompatible_options("c", "rename");
705
        parser.check_incompatible_options("c", "ignore");
706
        parser.check_incompatible_options("c", "parts");
707
        parser.check_incompatible_options("c", "tile");
708
        parser.check_incompatible_options("c", "cluster");
Davis King's avatar
Davis King committed
709
        parser.check_incompatible_options("c", "resample");
Davis King's avatar
Davis King committed
710
        parser.check_incompatible_options("l", "rename");
711
        parser.check_incompatible_options("l", "ignore");
712
        parser.check_incompatible_options("l", "add");
713
        parser.check_incompatible_options("l", "parts");
Davis King's avatar
Davis King committed
714
        parser.check_incompatible_options("l", "flip");
715
        parser.check_incompatible_options("l", "flip-basic");
716
        parser.check_incompatible_options("l", "rotate");
Adrià Arrufat's avatar
Adrià Arrufat committed
717
        parser.check_incompatible_options("l", "add-width-height-metadata");
Davis King's avatar
Davis King committed
718
719
720
721
722
        parser.check_incompatible_options("files", "rename");
        parser.check_incompatible_options("files", "ignore");
        parser.check_incompatible_options("files", "add");
        parser.check_incompatible_options("files", "parts");
        parser.check_incompatible_options("files", "flip");
723
        parser.check_incompatible_options("files", "flip-basic");
Davis King's avatar
Davis King committed
724
        parser.check_incompatible_options("files", "rotate");
Adrià Arrufat's avatar
Adrià Arrufat committed
725
        parser.check_incompatible_options("files", "add-width-height-metadata");
Davis King's avatar
Davis King committed
726
        parser.check_incompatible_options("add", "flip");
727
        parser.check_incompatible_options("add", "flip-basic");
728
        parser.check_incompatible_options("add", "rotate");
Adrià Arrufat's avatar
Adrià Arrufat committed
729
        parser.check_incompatible_options("add", "add-width-height-metadata");
730
731
        parser.check_incompatible_options("add", "tile");
        parser.check_incompatible_options("flip", "tile");
732
        parser.check_incompatible_options("flip-basic", "tile");
733
        parser.check_incompatible_options("rotate", "tile");
Adrià Arrufat's avatar
Adrià Arrufat committed
734
        parser.check_incompatible_options("add-width-height-metadata", "tile");
735
        parser.check_incompatible_options("cluster", "tile");
Davis King's avatar
Davis King committed
736
        parser.check_incompatible_options("resample", "tile");
737
        parser.check_incompatible_options("flip", "cluster");
738
        parser.check_incompatible_options("flip-basic", "cluster");
739
        parser.check_incompatible_options("rotate", "cluster");
Adrià Arrufat's avatar
Adrià Arrufat committed
740
        parser.check_incompatible_options("add-width-height-metadata", "cluster");
741
        parser.check_incompatible_options("add", "cluster");
Davis King's avatar
Davis King committed
742
        parser.check_incompatible_options("flip", "resample");
743
        parser.check_incompatible_options("flip-basic", "resample");
Davis King's avatar
Davis King committed
744
        parser.check_incompatible_options("rotate", "resample");
Adrià Arrufat's avatar
Adrià Arrufat committed
745
        parser.check_incompatible_options("add-width-height-metadata", "resample");
Davis King's avatar
Davis King committed
746
        parser.check_incompatible_options("add", "resample");
747
        parser.check_incompatible_options("shuffle", "tile");
748
        parser.check_incompatible_options("sort-num-objects", "tile");
749
        parser.check_incompatible_options("sort", "tile");
750
        parser.check_incompatible_options("convert", "l");
Davis King's avatar
Davis King committed
751
        parser.check_incompatible_options("convert", "files");
752
        parser.check_incompatible_options("convert", "rename");
753
        parser.check_incompatible_options("convert", "ignore");
754
        parser.check_incompatible_options("convert", "parts");
755
        parser.check_incompatible_options("convert", "cluster");
Davis King's avatar
Davis King committed
756
        parser.check_incompatible_options("convert", "resample");
757
        parser.check_incompatible_options("rmdiff", "rename");
758
        parser.check_incompatible_options("rmdiff", "ignore");
Davis King's avatar
Davis King committed
759
760
        parser.check_incompatible_options("rmempty", "ignore");
        parser.check_incompatible_options("rmempty", "rename");
761
762
        parser.check_incompatible_options("rmlabel", "ignore");
        parser.check_incompatible_options("rmlabel", "rename");
763
764
        parser.check_incompatible_options("rm-other-labels", "ignore");
        parser.check_incompatible_options("rm-other-labels", "rename");
Davis King's avatar
Davis King committed
765
766
        parser.check_incompatible_options("rmignore", "ignore");
        parser.check_incompatible_options("rmignore", "rename");
767
768
        parser.check_incompatible_options("rm-if-overlaps", "ignore");
        parser.check_incompatible_options("rm-if-overlaps", "rename");
Davis King's avatar
Davis King committed
769
        parser.check_incompatible_options("rmdupes", "rename");
770
        parser.check_incompatible_options("rmdupes", "ignore");
771
        parser.check_incompatible_options("rmtrunc", "rename");
772
        parser.check_incompatible_options("rmtrunc", "ignore");
Davis King's avatar
Davis King committed
773
774
        parser.check_incompatible_options("box-images", "rename");
        parser.check_incompatible_options("box-images", "ignore");
775
#ifdef DLIB_WEBP_SUPPORT
776
        parser.check_incompatible_options("jpg", "webp");
777
#endif
778
        const char* convert_args[] = {"pascal-xml","pascal-v1","idl"};
779
        parser.check_option_arg_range("convert", convert_args);
780
        parser.check_option_arg_range("cluster", 0, 999);
Davis King's avatar
Davis King committed
781
        parser.check_option_arg_range("rotate", -360, 360);
782
        parser.check_option_arg_range("size", 10*10, 1000*1000);
783
        parser.check_option_arg_range("min-object-size", 1, 10000*10000);
Davis King's avatar
Davis King committed
784
785
        parser.check_option_arg_range("cropped-object-size", 4, 10000*10000);
        parser.check_option_arg_range("crop-size", 1.0, 100.0);
786
        parser.check_option_arg_range("split-train-test", 0.0, 1.0);
787
#ifdef DLIB_WEBP_SUPPORT
788
        parser.check_option_arg_range("webp", 0.f, std::numeric_limits<float>::max());
789
#endif
790
791
792

        if (parser.option("h"))
        {
793
            cout << "Usage: imglab [options] <image files/directories or XML file>\n";
794
            parser.print_options(cout);
Davis King's avatar
Davis King committed
795
            cout << endl << endl;
796
797
798
            return EXIT_SUCCESS;
        }

799
800
801
802
803
804
        if (parser.option("add"))
        {
            merge_metadata_files(parser);
            return EXIT_SUCCESS;
        }

805
        if (parser.option("flip") || parser.option("flip-basic"))
Davis King's avatar
Davis King committed
806
807
808
809
810
        {
            flip_dataset(parser);
            return EXIT_SUCCESS;
        }

811
812
813
814
815
816
        if (parser.option("rotate"))
        {
            rotate_dataset(parser);
            return EXIT_SUCCESS;
        }

Adrià Arrufat's avatar
Adrià Arrufat committed
817
818
819
820
821
822
        if (parser.option("add-width-height-metadata"))
        {
            add_width_and_height_metadata(parser);
            return EXIT_SUCCESS;
        }

Davis King's avatar
Davis King committed
823
824
825
826
827
828
829
830
831
        if (parser.option("v"))
        {
            cout << "imglab v" << VERSION 
                 << "\nCompiled: " << __TIME__ << " " << __DATE__ 
                 << "\nWritten by Davis King\n";
            cout << "Check for updates at http://dlib.net\n\n";
            return EXIT_SUCCESS;
        }

832
833
834
835
836
        if (parser.option("tile"))
        {
            return tile_dataset(parser);
        }

837
838
839
840
841
        if (parser.option("cluster"))
        {
            return cluster_dataset(parser);
        }

Davis King's avatar
Davis King committed
842
843
844
845
846
        if (parser.option("resample"))
        {
            return resample_dataset(parser);
        }

847
848
        if (parser.option("c"))
        {
849
850
            if (parser.option("convert"))
            {
Davis King's avatar
Davis King committed
851
852
                if (parser.option("convert").argument() == "pascal-xml")
                    convert_pascal_xml(parser);
853
854
                else if (parser.option("convert").argument() == "pascal-v1")
                    convert_pascal_v1(parser);
855
856
                else if (parser.option("convert").argument() == "idl")
                    convert_idl(parser);
857
858
859
860
861
            }
            else
            {
                create_new_dataset(parser);
            }
862
863
            return EXIT_SUCCESS;
        }
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
        
        if (parser.option("rmdiff"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --rmdiff option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);
            for (unsigned long i = 0; i < data.images.size(); ++i)
            {
                for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
                {
879
880
881
882
883
884
885
886
                    if (data.images[i].boxes[j].difficult)
                        data.images[i].boxes[j].ignore = true;
                }
            }
            save_image_dataset_metadata(data, parser[0]);
            return EXIT_SUCCESS;
        }

Davis King's avatar
Davis King committed
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
        if (parser.option("rmempty"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --rmempty option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data, data2;
            load_image_dataset_metadata(data, parser[0]);

            data2 = data;
            data2.images.clear();
            for (unsigned long i = 0; i < data.images.size(); ++i)
            {
                bool has_label = false;
                for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
                {
                    if (!data.images[i].boxes[j].ignore)
                        has_label = true;
                }
                if (has_label)
                    data2.images.push_back(data.images[i]);
            }
            save_image_dataset_metadata(data2, parser[0] + ".rmempty.xml");
            return EXIT_SUCCESS;
        }

915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
        if (parser.option("rmlabel"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --rmlabel option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);

            const auto label = parser.option("rmlabel").argument();

            for (auto&& img : data.images)
            {
                std::vector<dlib::image_dataset_metadata::box> boxes;
                for (auto&& b : img.boxes)
                {
                    if (b.label != label)
                        boxes.push_back(b);
                }
                img.boxes = boxes;
            }

            save_image_dataset_metadata(data, parser[0] + ".rmlabel-"+label+".xml");
            return EXIT_SUCCESS;
        }

943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
        if (parser.option("rm-other-labels"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --rm-other-labels option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);

            const auto labels = parser.option("rm-other-labels").argument();
            // replace comma by dash to form the file name
            std::string strlabels = labels;
            std::replace(strlabels.begin(), strlabels.end(), ',', '-');
            std::vector<string> all_labels = split(labels, ",");
            for (auto&& img : data.images)
            {
                std::vector<dlib::image_dataset_metadata::box> boxes;
                for (auto&& b : img.boxes)
                {
                    if (std::find(all_labels.begin(), all_labels.end(), b.label) != all_labels.end())
                        boxes.push_back(b);
                }
                img.boxes = boxes;
            }

            save_image_dataset_metadata(data, parser[0] + ".rm-other-labels-"+ strlabels +".xml");
            return EXIT_SUCCESS;
        }

Davis King's avatar
Davis King committed
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
        if (parser.option("rmignore"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --rmignore option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);

            for (auto&& img : data.images)
            {
                std::vector<dlib::image_dataset_metadata::box> boxes;
                for (auto&& b : img.boxes)
                {
                    if (!b.ignore)
                        boxes.push_back(b);
                }
                img.boxes = boxes;
            }

            save_image_dataset_metadata(data, parser[0] + ".rmignore.xml");
            return EXIT_SUCCESS;
        }

1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
        if (parser.option("rm-if-overlaps"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --rm-if-overlaps option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);

            const auto label = parser.option("rm-if-overlaps").argument();

1013
            test_box_overlap overlaps(0.5);
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045

            for (auto&& img : data.images)
            {
                std::vector<dlib::image_dataset_metadata::box> boxes;
                for (auto&& b : img.boxes)
                {
                    if (b.label != label)
                    {
                        boxes.push_back(b);
                    }
                    else
                    {
                        bool has_overlap = false;
                        for (auto&& b2 : img.boxes)
                        {
                            if (b2.label != label && overlaps(b2.rect, b.rect))
                            {
                                has_overlap = true;
                                break;
                            }
                        }
                        if (!has_overlap)
                            boxes.push_back(b);
                    }
                }
                img.boxes = boxes;
            }

            save_image_dataset_metadata(data, parser[0] + ".rm-if-overlaps-"+label+".xml");
            return EXIT_SUCCESS;
        }

Davis King's avatar
Davis King committed
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
        if (parser.option("rmdupes"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --rmdupes option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data, data_out;
            std::set<std::string> hashes;
            load_image_dataset_metadata(data, parser[0]);
            data_out = data;
            data_out.images.clear();

            for (unsigned long i = 0; i < data.images.size(); ++i)
            {
                ifstream fin(data.images[i].filename.c_str(), ios::binary);
                string hash = md5(fin);
                if (hashes.count(hash) == 0)
                {
                    hashes.insert(hash);
                    data_out.images.push_back(data.images[i]);
                }
            }
            save_image_dataset_metadata(data_out, parser[0]);
            return EXIT_SUCCESS;
        }

Davis King's avatar
Davis King committed
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
        if (parser.option("box-images"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --box-images option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset   data;
            load_image_dataset_metadata(data, parser[0]);
            {
                locally_change_current_dir chdir(get_parent_directory(file(parser[0])));
                parallel_for(0, data.images.size(), [&](long i) 
                {
                    array2d<unsigned char> img;
                    load_image(img, data.images[i].filename);
                    data.images[i].boxes.emplace_back(get_rect(img));
                });
            }
            save_image_dataset_metadata(data, parser[0]+".boxed.xml");
            return EXIT_SUCCESS;
        }

1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
        if (parser.option("rmtrunc"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --rmtrunc option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);
            {
                locally_change_current_dir chdir(get_parent_directory(file(parser[0])));
                for (unsigned long i = 0; i < data.images.size(); ++i)
                {
                    array2d<unsigned char> img;
                    load_image(img, data.images[i].filename);
                    const rectangle area = get_rect(img);
                    for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
                    {
                        if (!area.contains(data.images[i].boxes[j].rect))
                            data.images[i].boxes[j].ignore = true;
                    }
1119
1120
1121
1122
1123
                }
            }
            save_image_dataset_metadata(data, parser[0]);
            return EXIT_SUCCESS;
        }
1124

1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
        if (parser.option("l"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The -l option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);
            print_all_labels(data);
            return EXIT_SUCCESS;
        }

Davis King's avatar
Davis King committed
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
        if (parser.option("files"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --files option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);
            for (size_t i = 0; i < data.images.size(); ++i)
                cout << data.images[i].filename << "\n";
            return EXIT_SUCCESS;
        }

1154
1155
1156
1157
1158
        if (parser.option("split"))
        {
            return split_dataset(parser);
        }

1159
1160
1161
1162
1163
        if (parser.option("split-train-test"))
        {
            return make_train_test_splits(parser);
        }

1164
1165
1166
1167
        if (parser.option("shuffle"))
        {
            if (parser.number_of_arguments() != 1)
            {
1168
                cerr << "The --shuffle option requires you to give one XML file on the command line." << endl;
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);
            const string default_seed = cast_to_string(time(0));
            const string seed = get_option(parser, "seed", default_seed);
            dlib::rand rnd(seed);
            randomize_samples(data.images, rnd);
            save_image_dataset_metadata(data, parser[0]);
            return EXIT_SUCCESS;
        }

1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
        if (parser.option("sort-num-objects"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --sort-num-objects option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);
            std::sort(data.images.rbegin(),  data.images.rend(), 
                [](const image_dataset_metadata::image& a, const image_dataset_metadata::image& b) { return a.boxes.size() < b.boxes.size(); });
            save_image_dataset_metadata(data, parser[0]);
            return EXIT_SUCCESS;
        }

1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
        if (parser.option("sort"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --sort option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);
            std::sort(data.images.begin(),  data.images.end(), 
                [](const image_dataset_metadata::image& a, const image_dataset_metadata::image& b) { return a.filename < b.filename; });
            save_image_dataset_metadata(data, parser[0]);
            return EXIT_SUCCESS;
        }

1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
        if (parser.option("stats"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --stats option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);
            print_all_label_stats(data);
            return EXIT_SUCCESS;
        }

Davis King's avatar
Davis King committed
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
        if (parser.option("rename"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --rename option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);
            for (unsigned long i = 0; i < parser.option("rename").count(); ++i)
            {
                rename_labels(data, parser.option("rename").argument(0,i), parser.option("rename").argument(1,i));
            }
            save_image_dataset_metadata(data, parser[0]);
            return EXIT_SUCCESS;
        }

1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
        if (parser.option("ignore"))
        {
            if (parser.number_of_arguments() != 1)
            {
                cerr << "The --ignore option requires you to give one XML file on the command line." << endl;
                return EXIT_FAILURE;
            }

            dlib::image_dataset_metadata::dataset data;
            load_image_dataset_metadata(data, parser[0]);
            for (unsigned long i = 0; i < parser.option("ignore").count(); ++i)
            {
                ignore_labels(data, parser.option("ignore").argument());
            }
            save_image_dataset_metadata(data, parser[0]+".ignored.xml");
            return EXIT_SUCCESS;
        }

1264
1265
        if (parser.number_of_arguments() == 1)
        {
1266
            metadata_editor editor(parser[0]);
1267
1268
1269
1270
1271
1272
1273
1274
            if (parser.option("parts"))
            {
                std::vector<string> parts = split(parser.option("parts").argument());
                for (unsigned long i = 0; i < parts.size(); ++i)
                {
                    editor.add_labelable_part_name(parts[i]);
                }
            }
1275
            editor.wait_until_closed();
1276
            return EXIT_SUCCESS;
1277
        }
Davis King's avatar
Davis King committed
1278
1279
1280

        cout << "Invalid command, give -h to see options." << endl;
        return EXIT_FAILURE;
1281
1282
1283
    }
    catch (exception& e)
    {
1284
        cerr << e.what() << endl;
1285
1286
1287
1288
        return EXIT_FAILURE;
    }
}

Davis King's avatar
Davis King committed
1289
1290
// ----------------------------------------------------------------------------------------