dlib for dcu, version=19.24

dlib/data_io/cifar.h

+// Copyright (C) 2020  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_CIFAR_Hh_
+#define DLIB_CIFAR_Hh_
+
+#include "cifar_abstract.h"
+#include <string>
+#include <vector>
+#include "../matrix.h"
+#include "../pixel.h"
+
+// ----------------------------------------------------------------------------------------
+
+namespace dlib
+{
+    void load_cifar_10_dataset (
+        const std::string& folder_name,
+        std::vector<matrix<rgb_pixel>>& training_images,
+        std::vector<unsigned long>& training_labels,
+        std::vector<matrix<rgb_pixel>>& testing_images,
+        std::vector<unsigned long>& testing_labels
+    );
+}
+
+// ----------------------------------------------------------------------------------------
+
+#ifdef NO_MAKEFILE
+#include "cifar.cpp"
+#endif
+
+#endif // DLIB_CIFAR_Hh_

dlib/data_io/cifar_abstract.h

+// Copyright (C) 2020  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_CIFAR_ABSTRACT_Hh_
+#ifdef DLIB_CIFAR_ABSTRACT_Hh_
+
+#include <string>
+#include <vector>
+#include "../matrix.h"
+#include "../pixel.h"
+
+// ----------------------------------------------------------------------------------------
+
+namespace dlib
+{
+    void load_cifar_10_dataset (
+        const std::string& folder_name,
+        std::vector<matrix<rgb_pixel>>& training_images,
+        std::vector<unsigned long>& training_labels,
+        std::vector<matrix<rgb_pixel>>& testing_images,
+        std::vector<unsigned long>& testing_labels
+    );
+    /*!
+        ensures
+            - Attempts to load the CIFAR-10 dataset from the hard drive.  The CIFAR-10
+              dataset consists of 60000 32x32 colour images in 10 classes, with 6000 images
+              per class.  There are 50000 training images and 10000 test images.  It is
+              available from https://www.cs.toronto.edu/~kriz/cifar.html.  In particular,
+              the 6 files comprising the CIFAR-10 dataset should be present in the folder
+              indicated by folder_name.  These six files are:
+                - data_batch_1.bin
+                - data_batch_2.bin
+                - data_batch_3.bin
+                - data_batch_4.bin
+                - data_batch_5.bin
+                - test_batch.bin
+            - #training_images == The 50,000 training images from the dataset.
+            - #training_labels == The labels for the contents of #training_images.
+              I.e. #training_labels[i] is the label of #training_images[i].
+            - #testing_images == The 10,000 testing images from the dataset. 
+            - #testing_labels == The labels for the contents of #testing_images.
+              I.e. #testing_labels[i] is the label of #testing_images[i].
+        throws
+            - dlib::error if some problem prevents us from loading the data or the files
+              can't be found.
+    !*/
+}
+
+// ----------------------------------------------------------------------------------------
+
+#endif // DLIB_CIFAR_ABSTRACT_Hh_
+

dlib/data_io/image_dataset_metadata.cpp

+// Copyright (C) 2011  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_IMAGE_DAtASET_METADATA_CPPh_
+#define DLIB_IMAGE_DAtASET_METADATA_CPPh_
+
+#include "image_dataset_metadata.h"
+
+#include <fstream>
+#include <sstream>
+#include "../compress_stream.h"
+#include "../base64.h"
+#include "../xml_parser.h"
+#include "../string.h"
+
+// ----------------------------------------------------------------------------------------
+
+namespace dlib
+{
+    namespace image_dataset_metadata
+    {
+
+    // ------------------------------------------------------------------------------------
+
+        const std::string get_decoded_string();
+        void create_image_metadata_stylesheet_file(const std::string& main_filename)
+        {
+            std::string path;
+            std::string::size_type pos = main_filename.find_last_of("/\\");
+            if (pos != std::string::npos)
+                path = main_filename.substr(0,pos+1);
+
+            std::ofstream fout((path + "image_metadata_stylesheet.xsl").c_str());
+            if (!fout)
+                throw dlib::error("ERROR: Unable to open image_metadata_stylesheet.xsl for writing.");
+
+            fout << get_decoded_string();
+
+            if (!fout)
+                throw dlib::error("ERROR: Unable to write to image_metadata_stylesheet.xsl.");
+        }
+
+        void save_image_dataset_metadata (
+            const dataset& meta,
+            const std::string& filename
+        )
+        {
+            create_image_metadata_stylesheet_file(filename);
+
+            const std::vector<image>& images = meta.images;
+
+            std::ofstream fout(filename.c_str());
+            if (!fout)
+                throw dlib::error("ERROR: Unable to open " + filename + " for writing.");
+
+            fout << "<?xml version='1.0' encoding='ISO-8859-1'?>\n";
+            fout << "<?xml-stylesheet type='text/xsl' href='image_metadata_stylesheet.xsl'?>\n";
+            fout << "<dataset>\n";
+            fout << "<name>" << meta.name << "</name>\n";
+            fout << "<comment>" << meta.comment << "</comment>\n";
+            fout << "<images>\n";
+            for (unsigned long i = 0; i < images.size(); ++i)
+            {
+                fout << "  <image file='" << images[i].filename << "'";
+                if (images[i].width != 0 && images[i].height != 0)
+                {
+                    fout << " width='" << images[i].width << "'";
+                    fout << " height='" << images[i].height << "'";
+                }
+                fout << ">\n";
+
+                // save all the boxes
+                for (unsigned long j = 0; j < images[i].boxes.size(); ++j)
+                {
+                    const box& b = images[i].boxes[j];
+                    fout << "    <box top='" << b.rect.top() << "' "
+                                 << "left='" << b.rect.left() << "' "
+                                << "width='" << b.rect.width() << "' "
+                               << "height='" << b.rect.height() << "'";
+                    if (b.difficult)
+                        fout << " difficult='" << b.difficult << "'";
+                    if (b.truncated)
+                        fout << " truncated='" << b.truncated << "'";
+                    if (b.occluded)
+                        fout << " occluded='" << b.occluded << "'";
+                    if (b.ignore)
+                        fout << " ignore='" << b.ignore << "'";
+                    if (b.angle != 0)
+                        fout << " angle='" << b.angle << "'";
+                    if (b.age != 0)
+                        fout << " age='" << b.age << "'";
+                    if (b.gender == FEMALE)
+                        fout << " gender='female'";
+                    else if (b.gender == MALE)
+                        fout << " gender='male'";
+                    if (b.pose != 0)
+                        fout << " pose='" << b.pose << "'";
+                    if (b.detection_score != 0)
+                        fout << " detection_score='" << b.detection_score << "'";
+
+                    if (b.has_label() || b.parts.size() != 0)
+                    {
+                        fout << ">\n";
+
+                        if (b.has_label())
+                            fout << "      <label>" << b.label << "</label>\n";
+                        
+                        // save all the parts
+                        std::map<std::string,point>::const_iterator itr;
+                        for (itr = b.parts.begin(); itr != b.parts.end(); ++itr)
+                        {
+                            fout << "      <part name='"<< itr->first << "' x='"<< itr->second.x() <<"' y='"<< itr->second.y() <<"'/>\n";
+                        }
+
+                        fout << "    </box>\n";
+                    }
+                    else
+                    {
+                        fout << "/>\n";
+                    }
+                }
+
+
+
+                fout << "  </image>\n";
+
+                if (!fout)
+                    throw dlib::error("ERROR: Unable to write to " + filename + ".");
+            }
+            fout << "</images>\n";
+            fout << "</dataset>";
+        }
+
+    // ------------------------------------------------------------------------------------
+    // ------------------------------------------------------------------------------------
+    // ------------------------------------------------------------------------------------
+
+        class doc_handler : public document_handler
+        {
+            std::vector<std::string> ts;
+            image temp_image;
+            box temp_box;
+
+            dataset& meta;
+
+        public:
+
+            doc_handler(
+                dataset& metadata_
+            ):
+                meta(metadata_) 
+            {}
+
+
+            virtual void start_document (
+            )
+            {
+                meta = dataset();
+                ts.clear();
+                temp_image = image();
+                temp_box = box();
+            }
+
+            virtual void end_document (
+            )
+            {
+            }
+
+            virtual void start_element ( 
+                const unsigned long line_number,
+                const std::string& name,
+                const dlib::attribute_list& atts
+            )
+            {
+                try
+                {
+                    if (ts.size() == 0) 
+                    {
+                        if (name != "dataset")
+                        {
+                            std::ostringstream sout;
+                            sout << "Invalid XML document.  Root tag must be <dataset>.  Found <" << name << "> instead.";
+                            throw dlib::error(sout.str());
+                        }
+                        else
+                        {
+                            ts.push_back(name);
+                            return;
+                        }
+                    }
+
+
+                    if (name == "box")
+                    {
+                        if (atts.is_in_list("top")) temp_box.rect.top() = sa = atts["top"];
+                        else throw dlib::error("<box> missing required attribute 'top'");
+
+                        if (atts.is_in_list("left")) temp_box.rect.left() = sa = atts["left"];
+                        else throw dlib::error("<box> missing required attribute 'left'");
+
+                        if (atts.is_in_list("width")) temp_box.rect.right() = sa = atts["width"];
+                        else throw dlib::error("<box> missing required attribute 'width'");
+
+                        if (atts.is_in_list("height")) temp_box.rect.bottom() = sa = atts["height"];
+                        else throw dlib::error("<box> missing required attribute 'height'");
+
+                        if (atts.is_in_list("difficult")) temp_box.difficult = sa = atts["difficult"];
+                        if (atts.is_in_list("truncated")) temp_box.truncated = sa = atts["truncated"];
+                        if (atts.is_in_list("occluded"))  temp_box.occluded  = sa = atts["occluded"];
+                        if (atts.is_in_list("ignore"))  temp_box.ignore  = sa = atts["ignore"];
+                        if (atts.is_in_list("angle"))  temp_box.angle  = sa = atts["angle"];
+                        if (atts.is_in_list("age"))  temp_box.age  = sa = atts["age"];
+                        if (atts.is_in_list("gender"))  
+                        {
+                            if (atts["gender"] == "male")
+                                temp_box.gender = MALE;
+                            else if (atts["gender"] == "female")
+                                temp_box.gender = FEMALE;
+                            else if (atts["gender"] == "unknown")
+                                temp_box.gender = UNKNOWN;
+                            else
+                                throw dlib::error("Invalid gender string in box attribute.");
+                        }
+                        if (atts.is_in_list("pose"))  temp_box.pose  = sa = atts["pose"];
+                        if (atts.is_in_list("detection_score"))  temp_box.detection_score  = sa = atts["detection_score"];
+
+                        temp_box.rect.bottom() += temp_box.rect.top()-1;
+                        temp_box.rect.right() += temp_box.rect.left()-1;
+                    }
+                    else if (name == "part" && ts.back() == "box")
+                    {
+                        point temp;
+                        if (atts.is_in_list("x")) temp.x() = sa = atts["x"];
+                        else throw dlib::error("<part> missing required attribute 'x'");
+
+                        if (atts.is_in_list("y")) temp.y() = sa = atts["y"];
+                        else throw dlib::error("<part> missing required attribute 'y'");
+
+                        if (atts.is_in_list("name")) 
+                        {
+                            if (temp_box.parts.count(atts["name"])==0)
+                            {
+                                temp_box.parts[atts["name"]] = temp;
+                            }
+                            else
+                            {
+                                throw dlib::error("<part> with name '" + atts["name"] + "' is defined more than one time in a single box.");
+                            }
+                        }
+                        else 
+                        {
+                            throw dlib::error("<part> missing required attribute 'name'");
+                        }
+                    }
+                    else if (name == "image")
+                    {
+                        temp_image.boxes.clear();
+
+                        if (atts.is_in_list("file")) temp_image.filename = atts["file"];
+                        else throw dlib::error("<image> missing required attribute 'file'");
+
+                        if (atts.is_in_list("width")) temp_image.width = sa = atts["width"];
+                        if (atts.is_in_list("height")) temp_image.height = sa = atts["height"];
+                    }
+
+                    ts.push_back(name);
+                }
+                catch (error& e)
+                {
+                    throw dlib::error("Error on line " + cast_to_string(line_number) + ": " + e.what());
+                }
+            }
+
+            virtual void end_element ( 
+                const unsigned long ,
+                const std::string& name
+            )
+            {
+                ts.pop_back();
+                if (ts.size() == 0)
+                    return;
+
+                if (name == "box" && ts.back() == "image")
+                {
+                    temp_image.boxes.push_back(temp_box);
+                    temp_box = box();
+                }
+                else if (name == "image" && ts.back() == "images")
+                {
+                    meta.images.push_back(temp_image);
+                    temp_image = image();
+                }
+            }
+
+            virtual void characters ( 
+                const std::string& data
+            )
+            {
+                if (ts.size() == 2 && ts[1] == "name")
+                {
+                    meta.name = trim(data);
+                }
+                else if (ts.size() == 2 && ts[1] == "comment")
+                {
+                    meta.comment = trim(data);
+                }
+                else if (ts.size() >= 2 && ts[ts.size()-1] == "label" && 
+                                           ts[ts.size()-2] == "box")
+                {
+                    temp_box.label = trim(data);
+                }
+            }
+
+            virtual void processing_instruction (
+                const unsigned long ,
+                const std::string& ,
+                const std::string& 
+            )
+            {
+            }
+        };
+
+    // ----------------------------------------------------------------------------------------
+
+        class xml_error_handler : public error_handler
+        {
+        public:
+            virtual void error (
+                const unsigned long 
+            ) { }
+
+            virtual void fatal_error (
+                const unsigned long line_number
+            )
+            {
+                std::ostringstream sout;
+                sout << "There is a fatal error on line " << line_number << " so parsing will now halt.";
+                throw dlib::error(sout.str());
+            }
+        };
+
+    // ------------------------------------------------------------------------------------
+
+        void load_image_dataset_metadata (
+            dataset& meta,
+            const std::string& filename
+        )
+        {
+            xml_error_handler eh;
+            doc_handler dh(meta);
+
+            std::ifstream fin(filename.c_str());
+            if (!fin)
+                throw dlib::error("ERROR: unable to open " + filename + " for reading.");
+
+            xml_parser parser;
+            parser.add_document_handler(dh);
+            parser.add_error_handler(eh);
+            parser.parse(fin);
+        }
+
+    // ------------------------------------------------------------------------------------
+    // ------------------------------------------------------------------------------------
+    // ------------------------------------------------------------------------------------
+
+        // This function returns the contents of the file 'images.xsl'
+        const std::string get_decoded_string()
+        {
+            dlib::base64 base64_coder;
+            dlib::compress_stream::kernel_1ea compressor;
+            std::ostringstream sout;
+            std::istringstream sin;
+
+            // The base64 encoded data from the file 'image_metadata_stylesheet.xsl' we want to decode and return.
+            sout << "PFWfgmWfCHr1DkV63lbjjeY2dCc2FbHDOVh0Kd7dkvaOfRYrOG24f0x77/5iMVq8FtE3UBxtGwSd";
+            sout << "1ZHOHRSHgieNoeBv8ssJQ75RRxYtFKRY3OTPX5eKQoCN9jUaUnHnR4QZtEHgmKqXSs50Yrdd+2Ah";
+            sout << "gNyarPZCiR6nvqNvCjtP2MP5FxleqNf8Fylatm2KdsXmrv5K87LYVN7i7JMkmZ++cTXYSOxDmxZi";
+            sout << "OiCH8funXUdF9apDW547gCjz9HOQUI6dkz5dYUeFjfp6dFugpnaJyyprFLKq048Qk7+QiL4CNF/G";
+            sout << "7e0VpBw8dMpiyRNi2fSQGSZGfIAUQKKT6+rPwQoRH2spdjsdXVWj4XQAqBX87nmqMnqjMhn/Vd1s";
+            sout << "W5aoC0drwRGu3Xe3gn9vBL8hBkRXcJvEy6q/lb9bYnsLemhE5Zp/+nTmTBjfT9UFYLcsmgsjC+4n";
+            sout << "Bq6h9QlpuyMYqJ8RvW8pp3mFlvXc3Yg+18t5F0hSMQfaIFYAuDPU2lVzPpY+ba0B39iu9IrPCLsS";
+            sout << "+tUtSNSmQ74CtzZgKKjkTMA3nwYP2SDmZE3firq42pihT7hdU5vYkes69K8AQl8WZyLPpMww+r0z";
+            sout << "+veEHPlAuxF7kL3ZvVjdB+xABwwqDe0kSRHRZINYdUfJwJdfYLyDnYoMjj6afqIJZ7QOBPZ42tV5";
+            sout << "3hYOQTFwTNovOastzJJXQe1kxPg1AQ8ynmfjjJZqD0xKedlyeJybP919mVAA23UryHsq9TVlabou";
+            sout << "qNl3xZW/mKKktvVsd/nuH62HIv/kgomyhaEUY5HgupupBUbQFZfyljZ5bl3g3V3Y1400Z1xTM/LL";
+            sout << "LJpeLdlqoGzIe/19vAN1zUUVId9F/OLNUl3Zoar63yZERSJHcsuq/Pasisp0HIGi7rfI9EIQF7C/";
+            sout << "IhLKLZsJ+LOycreQGOJALZIEZHOqxYLSXG0qaPM5bQL/MQJ2OZfwEhQgYOrjaM7oPOHHEfTq5kcO";
+            sout << "daMwzefKfxrF2GXbUs0bYsEXsIGwENIUKMliFaAI4qKLxxb94oc+O3BRjWueZjZty2zKawQyTHNd";
+            sout << "ltFJBUzfffdZN9Wq4zbPzntkM3U6Ys4LRztx5M15dtbhFeKx5rAf2tPXT6wU01hx7EJxBJzpvoDE";
+            sout << "YwEoYVDSYulRKpgk82cHFzzUDgWXbl4paFSe1L1w8r9KHr67SYJDTUG86Lrm6LJ0rw73Xp0NAFcU";
+            sout << "MKpiG9g1cHW74HYbUb/yAbtVWt40eB7M637umdo2jWz/r/vP5WnfSMXEbkyWebsa1fFceg/TLWy6";
+            sout << "E8OTc4XKB48h1oFIlGagOiprxho3+F3TIcxDSwA=";
+
+
+
+            // Put the data into the istream sin
+            sin.str(sout.str());
+            sout.str("");
+
+            // Decode the base64 text into its compressed binary form
+            base64_coder.decode(sin,sout);
+            sin.clear();
+            sin.str(sout.str());
+            sout.str("");
+
+            // Decompress the data into its original form
+            compressor.decompress(sin,sout);
+
+            // Return the decoded and decompressed data
+            return sout.str();
+        }
+
+
+    }
+}
+
+// ----------------------------------------------------------------------------------------
+
+#endif // DLIB_IMAGE_DAtASET_METADATA_CPPh_
+
+

dlib/data_io/image_dataset_metadata.h

+// Copyright (C) 2011  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_IMAGE_DAtASET_METADATA_Hh_
+#define DLIB_IMAGE_DAtASET_METADATA_Hh_
+
+#include <string>
+#include <vector>
+#include "../geometry.h"
+
+// ----------------------------------------------------------------------------------------
+
+namespace dlib
+{
+    namespace image_dataset_metadata
+    {
+
+    // ------------------------------------------------------------------------------------
+
+        enum gender_t
+        {
+            UNKNOWN,
+            MALE,
+            FEMALE
+        };
+
+    // ------------------------------------------------------------------------------------
+
+        struct box
+        {
+            /*!
+                WHAT THIS OBJECT REPRESENTS
+                    This object represents an annotated rectangular area of an image.  
+                    It is typically used to mark the location of an object such as a 
+                    person, car, etc.
+
+                    The main variable of interest is rect.  It gives the location of 
+                    the box.  All the other variables are optional.
+            !*/
+
+            box(
+            ) : 
+                difficult(false),
+                truncated(false),
+                occluded(false),
+                ignore(false),
+                pose(0),
+                detection_score(0),
+                angle(0),
+                gender(UNKNOWN),
+                age(0)
+            {}
+
+            box (
+                const rectangle& rect_
+            ) : 
+                rect(rect_), 
+                difficult(false),
+                truncated(false),
+                occluded(false),
+                ignore(false),
+                pose(0),
+                detection_score(0),
+                angle(0),
+                gender(UNKNOWN),
+                age(0)
+            {}
+
+            rectangle rect;
+
+            std::map<std::string,point> parts;
+
+            // optional fields
+            std::string label;
+            bool difficult;
+            bool truncated;
+            bool occluded;
+            bool ignore;
+            double pose;
+            double detection_score;
+
+            // The angle of the object in radians.  Positive values indicate that the
+            // object at the center of the box is rotated clockwise by angle radians.  A
+            // value of 0 would indicate that the object is in its "standard" upright pose.
+            // Therefore, to make the object appear upright we would have to rotate the
+            // image counter-clockwise by angle radians.
+            double angle; 
+
+            gender_t gender;
+            double age;
+
+            bool has_label() const { return label.size() != 0; }
+            /*!
+                ensures
+                    - returns true if label metadata is present and false otherwise.
+            !*/
+        };
+
+    // ------------------------------------------------------------------------------------
+
+        struct image
+        {
+            /*!
+                WHAT THIS OBJECT REPRESENTS
+                    This object represents an annotated image.
+            !*/
+
+            image() {}
+            image(const std::string& f) : filename(f) {}
+
+            std::string filename;
+            std::vector<box> boxes;
+            long width = 0;
+            long height = 0;
+        };
+
+    // ------------------------------------------------------------------------------------
+
+        struct dataset
+        {
+            /*!
+                WHAT THIS OBJECT REPRESENTS
+                    This object represents a labeled set of images.  In particular, it
+                    contains the filename for each image as well as annotated boxes.
+            !*/
+
+            std::vector<image> images;
+            std::string comment;
+            std::string name;
+        };
+
+    // ------------------------------------------------------------------------------------
+
+        void save_image_dataset_metadata (
+            const dataset& meta,
+            const std::string& filename
+        );
+        /*!
+            ensures
+                - Writes the contents of the meta object to a file with the given
+                  filename.  The file will be in an XML format.
+            throws
+                - dlib::error 
+                  This exception is thrown if there is an error which prevents
+                  this function from succeeding.
+        !*/
+
+    // ------------------------------------------------------------------------------------
+
+        void load_image_dataset_metadata (
+            dataset& meta,
+            const std::string& filename
+        );
+        /*!
+            ensures
+                - Attempts to interpret filename as a file containing XML formatted data
+                  as produced by the save_image_dataset_metadata() function.  Then
+                  meta is loaded with the contents of the file.
+            throws
+                - dlib::error 
+                  This exception is thrown if there is an error which prevents
+                  this function from succeeding.
+        !*/
+
+    // ------------------------------------------------------------------------------------
+
+    }
+}
+
+// ----------------------------------------------------------------------------------------
+
+#ifdef NO_MAKEFILE
+#include "image_dataset_metadata.cpp"
+#endif
+
+#endif // DLIB_IMAGE_DAtASET_METADATA_Hh_
+

dlib/data_io/libsvm_io.h

+// Copyright (C) 2010  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_LIBSVM_iO_Hh_
+#define DLIB_LIBSVM_iO_Hh_
+
+#include "libsvm_io_abstract.h"
+
+#include <fstream>
+#include <string>
+#include <utility>
+#include "../algs.h"
+#include "../matrix.h"
+#include "../string.h"
+#include "../svm/sparse_vector.h"
+#include <vector>
+
+namespace dlib
+{
+    struct sample_data_io_error : public error
+    {
+        sample_data_io_error(const std::string& message): error(message) {}
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename sample_type, typename label_type, typename alloc1, typename alloc2>
+    void load_libsvm_formatted_data (
+        const std::string& file_name,
+        std::vector<sample_type, alloc1>& samples,
+        std::vector<label_type, alloc2>& labels
+    )
+    {
+        using namespace std;
+        typedef typename sample_type::value_type pair_type;
+        typedef typename basic_type<typename pair_type::first_type>::type key_type;
+        typedef typename pair_type::second_type value_type;
+
+        // You must use unsigned integral key types in your sparse vectors
+        COMPILE_TIME_ASSERT(is_unsigned_type<key_type>::value);
+
+        samples.clear();
+        labels.clear();
+
+        ifstream fin(file_name.c_str());
+
+        if (!fin)
+            throw sample_data_io_error("Unable to open file " + file_name);
+
+        string line;
+        istringstream sin;
+        key_type key;
+        value_type value;
+        label_type label;
+        sample_type sample;
+        long line_num = 0;
+        while (fin.peek() != EOF)
+        {
+            ++line_num;
+            getline(fin, line);
+
+            string::size_type pos = line.find_first_not_of(" \t\r\n");
+
+            // ignore empty lines or comment lines
+            if (pos == string::npos || line[pos] == '#')
+                continue;
+
+            sin.clear();
+            sin.str(line);
+            sample.clear();
+
+            sin >> label;
+
+            if (!sin)
+                throw sample_data_io_error("On line: " + cast_to_string(line_num) + ", error while reading file " + file_name );
+
+            // eat whitespace
+            sin >> ws;
+
+            while (sin.peek() != EOF && sin.peek() != '#')
+            {
+
+                sin >> key >> ws;
+
+                // ignore what should be a : character
+                if (sin.get() != ':')
+                    throw sample_data_io_error("On line: " + cast_to_string(line_num) + ", error while reading file " + file_name);
+
+                sin >> value;
+
+                if (sin && value != 0)
+                {
+                    sample.insert(sample.end(), make_pair(key, value));
+                }
+
+                sin >> ws;
+            }
+
+            samples.push_back(sample);
+            labels.push_back(label);
+        }
+
+    }
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <typename sample_type, typename alloc>
+    typename enable_if<is_const_type<typename sample_type::value_type::first_type> >::type 
+    fix_nonzero_indexing (
+        std::vector<sample_type,alloc>& samples
+    )
+    {
+        typedef typename sample_type::value_type pair_type;
+        typedef typename basic_type<typename pair_type::first_type>::type key_type;
+
+        if (samples.size() == 0)
+            return;
+
+        // figure out the min index value
+        key_type min_idx = samples[0].begin()->first;
+        for (unsigned long i = 0; i < samples.size(); ++i)
+            min_idx = std::min(min_idx, samples[i].begin()->first);
+
+        // Now adjust all the samples so that their min index value is zero.
+        if (min_idx != 0)
+        {
+            sample_type temp;
+            for (unsigned long i = 0; i < samples.size(); ++i)
+            {
+                // copy samples[i] into temp but make sure it has a min index of zero.
+                temp.clear();
+                typename sample_type::iterator j;
+                for (j = samples[i].begin(); j != samples[i].end(); ++j)
+                {
+                    temp.insert(temp.end(), std::make_pair(j->first-min_idx, j->second));
+                }
+
+                // replace the current sample with temp.
+                samples[i].swap(temp);
+            }
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+// If the "first" values in the std::pair objects are not const then we can modify them 
+// directly and that is what this version of fix_nonzero_indexing() does.
+    template <typename sample_type, typename alloc>
+    typename disable_if<is_const_type<typename sample_type::value_type::first_type> >::type 
+    fix_nonzero_indexing (
+        std::vector<sample_type,alloc>& samples
+    )
+    {
+        typedef typename sample_type::value_type pair_type;
+        typedef typename basic_type<typename pair_type::first_type>::type key_type;
+
+        if (samples.size() == 0)
+            return;
+
+        // figure out the min index value
+        key_type min_idx = samples[0].begin()->first;
+        for (unsigned long i = 0; i < samples.size(); ++i)
+            min_idx = std::min(min_idx, samples[i].begin()->first);
+
+        // Now adjust all the samples so that their min index value is zero.
+        if (min_idx != 0)
+        {
+            for (unsigned long i = 0; i < samples.size(); ++i)
+            {
+                typename sample_type::iterator j;
+                for (j = samples[i].begin(); j != samples[i].end(); ++j)
+                {
+                    j->first -= min_idx;
+                }
+            }
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+// This is an overload for sparse vectors
+    template <typename sample_type, typename label_type, typename alloc1, typename alloc2>
+    typename disable_if<is_matrix<sample_type>,void>::type save_libsvm_formatted_data (
+        const std::string& file_name,
+        const std::vector<sample_type, alloc1>& samples,
+        const std::vector<label_type, alloc2>& labels
+    )
+    {
+        typedef typename sample_type::value_type pair_type;
+        typedef typename basic_type<typename pair_type::first_type>::type key_type;
+
+        // You must use unsigned integral key types in your sparse vectors
+        COMPILE_TIME_ASSERT(is_unsigned_type<key_type>::value);
+
+        // make sure requires clause is not broken
+        DLIB_ASSERT(samples.size() == labels.size(),
+            "\t void save_libsvm_formatted_data()"
+            << "\n\t You have to have labels for each sample and vice versa"
+            << "\n\t samples.size(): " << samples.size()
+            << "\n\t labels.size():  " << labels.size()
+            );
+
+
+        using namespace std;
+        ofstream fout(file_name.c_str());
+        fout.precision(14);
+
+        if (!fout)
+            throw sample_data_io_error("Unable to open file " + file_name);
+
+        for (unsigned long i = 0; i < samples.size(); ++i)
+        {
+            fout << labels[i];
+
+            for (typename sample_type::const_iterator j = samples[i].begin(); j != samples[i].end(); ++j)
+            {
+                if (j->second != 0)
+                    fout << " " << j->first << ":" << j->second;
+            }
+            fout << "\n";
+
+            if (!fout)
+                throw sample_data_io_error("Error while writing to file " + file_name);
+        }
+
+    }
+
+// ----------------------------------------------------------------------------------------
+
+// This is an overload for dense vectors
+    template <typename sample_type, typename label_type, typename alloc1, typename alloc2>
+    typename enable_if<is_matrix<sample_type>,void>::type save_libsvm_formatted_data (
+        const std::string& file_name,
+        const std::vector<sample_type, alloc1>& samples,
+        const std::vector<label_type, alloc2>& labels
+    )
+    {
+        // make sure requires clause is not broken
+        DLIB_ASSERT(samples.size() == labels.size(),
+            "\t void save_libsvm_formatted_data()"
+            << "\n\t You have to have labels for each sample and vice versa"
+            << "\n\t samples.size(): " << samples.size()
+            << "\n\t labels.size():  " << labels.size()
+            );
+
+        using namespace std;
+        ofstream fout(file_name.c_str());
+        fout.precision(14);
+
+        if (!fout)
+            throw sample_data_io_error("Unable to open file " + file_name);
+
+        for (unsigned long i = 0; i < samples.size(); ++i)
+        {
+            fout << labels[i];
+
+            for (long j = 0; j < samples[i].size(); ++j)
+            {
+                if (samples[i](j) != 0)
+                    fout << " " << j << ":" << samples[i](j);
+            }
+            fout << "\n";
+
+            if (!fout)
+                throw sample_data_io_error("Error while writing to file " + file_name);
+        }
+
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_LIBSVM_iO_Hh_
+

dlib/data_io/libsvm_io_abstract.h

+// Copyright (C) 2010  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_LIBSVM_iO_ABSTRACT_Hh_
+#ifdef DLIB_LIBSVM_iO_ABSTRACT_Hh_
+
+#include <fstream>
+#include <string>
+#include <utility>
+#include "../algs.h"
+#include "../matrix.h"
+#include <vector>
+
+namespace dlib
+{
+    struct sample_data_io_error : public error
+    {
+        /*!
+            This is the exception class used by the file IO functions defined below.
+        !*/
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename sample_type, 
+        typename label_type, 
+        typename alloc1, 
+        typename alloc2
+        >
+    void load_libsvm_formatted_data (
+        const std::string& file_name,
+        std::vector<sample_type, alloc1>& samples,
+        std::vector<label_type, alloc2>& labels
+    );
+    /*!
+        requires
+            - sample_type must be an STL container
+            - sample_type::value_type == std::pair<T,U> where T is some kind of 
+              unsigned integral type
+        ensures
+            - attempts to read a file of the given name that should contain libsvm
+              formatted data.  We turn the data into sparse vectors and store it
+              in samples
+            - #labels.size() == #samples.size()
+            - for all valid i: #labels[i] is the label for #samples[i]
+        throws
+            - sample_data_io_error
+                This exception is thrown if there is any problem loading data from file
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename sample_type,
+        typename label_type,
+        typename alloc1,
+        typename alloc2
+        >
+    void save_libsvm_formatted_data (
+        const std::string& file_name,
+        const std::vector<sample_type, alloc1>& samples,
+        const std::vector<label_type, alloc2>& labels
+    );
+    /*!
+        requires
+            - sample_type must be an STL container
+            - sample_type::value_type == std::pair<T,U> where T is some kind of 
+              unsigned integral type
+            - samples.size() == labels.size()
+        ensures
+            - saves the data to the given file in libsvm format
+        throws
+            - sample_data_io_error
+                This exception is thrown if there is any problem saving data to file
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename sample_type, 
+        typename label_type, 
+        typename alloc1, 
+        typename alloc2
+        >
+    void save_libsvm_formatted_data (
+        const std::string& file_name,
+        const std::vector<sample_type, alloc1>& samples,
+        const std::vector<label_type, alloc2>& labels
+    );
+    /*!
+        requires
+            - sample_type == a dense matrix (i.e. dlib::matrix)
+            - for all valid i: is_vector(samples[i]) == true
+            - samples.size() == labels.size()
+        ensures
+            - saves the data to the given file in libsvm format
+        throws
+            - sample_data_io_error
+                This exception is thrown if there is any problem saving data to file
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename sample_type, typename alloc>
+    void fix_nonzero_indexing (
+        std::vector<sample_type,alloc>& samples
+    );
+    /*!
+        requires
+            - samples must only contain valid sparse vectors.  The definition of
+              a sparse vector can be found at the top of dlib/svm/sparse_vector_abstract.h
+        ensures
+            - Adjusts the sparse vectors in samples so that they are zero-indexed.  
+              Or in other words, assume the smallest used index value in any of the sparse 
+              vectors is N.  Then this function subtracts N from all the index values in 
+              samples.  This is useful, for example, if you load a libsvm formatted datafile 
+              with features indexed from 1 rather than 0 and you would like to fix this.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_LIBSVM_iO_ABSTRACT_Hh_
+

dlib/data_io/load_image_dataset.h

+// Copyright (C) 2012  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_LOAD_IMAGE_DaTASET_Hh_
+#define DLIB_LOAD_IMAGE_DaTASET_Hh_
+
+#include "load_image_dataset_abstract.h"
+#include "../misc_api.h"
+#include "../dir_nav.h"
+#include "../image_io.h"
+#include "../array.h"
+#include <vector>
+#include "../geometry.h"
+#include "image_dataset_metadata.h"
+#include <string>
+#include <set>
+#include "../image_processing/full_object_detection.h"
+#include <utility>
+#include <limits>
+#include "../image_transforms/image_pyramid.h"
+
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    class image_dataset_file
+    {
+    public:
+        image_dataset_file(const std::string& filename)
+        {
+            _skip_empty_images = false;
+            _have_parts = false;
+            _filename = filename;
+            _box_area_thresh = std::numeric_limits<double>::infinity();
+        }
+
+        image_dataset_file boxes_match_label(
+            const std::string& label
+        ) const
+        {
+            image_dataset_file temp(*this);
+            temp._labels.insert(label);
+            return temp;
+        }
+
+        image_dataset_file skip_empty_images(
+        ) const
+        {
+            image_dataset_file temp(*this);
+            temp._skip_empty_images = true;
+            return temp;
+        }
+
+        image_dataset_file boxes_have_parts(
+        ) const
+        {
+            image_dataset_file temp(*this);
+            temp._have_parts = true;
+            return temp;
+        }
+
+        image_dataset_file shrink_big_images(
+            double new_box_area_thresh = 150*150
+        ) const
+        {
+            image_dataset_file temp(*this);
+            temp._box_area_thresh = new_box_area_thresh;
+            return temp;
+        }
+
+        bool should_load_box (
+            const image_dataset_metadata::box& box
+        ) const
+        {
+            if (_have_parts && box.parts.size() == 0)
+                return false;
+            if (_labels.size() == 0)
+                return true;
+            if (_labels.count(box.label) != 0)
+                return true;
+            return false;
+        }
+
+        const std::string& get_filename() const { return _filename; }
+        bool should_skip_empty_images() const { return _skip_empty_images; }
+        bool should_boxes_have_parts() const { return _have_parts; }
+        double box_area_thresh() const { return _box_area_thresh; }
+        const std::set<std::string>& get_selected_box_labels() const { return _labels; }
+
+    private:
+        std::string _filename;
+        std::set<std::string> _labels;
+        bool _skip_empty_images;
+        bool _have_parts;
+        double _box_area_thresh;
+
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type
+        >
+    std::vector<std::vector<rectangle> > load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<rectangle> >& object_locations,
+        const image_dataset_file& source
+    )
+    {
+        images.clear();
+        object_locations.clear();
+
+        std::vector<std::vector<rectangle> > ignored_rects;
+
+        using namespace dlib::image_dataset_metadata;
+        dataset data;
+        load_image_dataset_metadata(data, source.get_filename());
+
+        // Set the current directory to be the one that contains the
+        // metadata file. We do this because the file might contain
+        // file paths which are relative to this folder.
+        locally_change_current_dir chdir(get_parent_directory(file(source.get_filename())));
+
+
+        typedef typename array_type::value_type image_type;
+
+
+        image_type img;
+        std::vector<rectangle> rects, ignored;
+        for (unsigned long i = 0; i < data.images.size(); ++i)
+        {
+            double min_rect_size = std::numeric_limits<double>::infinity();
+            rects.clear();
+            ignored.clear();
+            for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
+            {
+                if (source.should_load_box(data.images[i].boxes[j]))
+                {
+                    if (data.images[i].boxes[j].ignore)
+                    {
+                        ignored.push_back(data.images[i].boxes[j].rect);
+                    }
+                    else
+                    {
+                        rects.push_back(data.images[i].boxes[j].rect);
+                        min_rect_size = std::min<double>(min_rect_size, rects.back().area());
+                    }
+                }
+            }
+
+            if (!source.should_skip_empty_images() || rects.size() != 0)
+            {
+                load_image(img, data.images[i].filename);
+                if (rects.size() != 0)  
+                {
+                    // if shrinking the image would still result in the smallest box being
+                    // bigger than the box area threshold then shrink the image.
+                    while(min_rect_size/2/2 > source.box_area_thresh())
+                    {
+                        pyramid_down<2> pyr;
+                        pyr(img);
+                        min_rect_size *= (1.0/2.0)*(1.0/2.0);
+                        for (auto&& r : rects)
+                            r = pyr.rect_down(r);
+                        for (auto&& r : ignored)
+                            r = pyr.rect_down(r);
+                    }
+                    while(min_rect_size*(2.0/3.0)*(2.0/3.0) > source.box_area_thresh())
+                    {
+                        pyramid_down<3> pyr;
+                        pyr(img);
+                        min_rect_size *= (2.0/3.0)*(2.0/3.0);
+                        for (auto&& r : rects)
+                            r = pyr.rect_down(r);
+                        for (auto&& r : ignored)
+                            r = pyr.rect_down(r);
+                    }
+                }
+                images.push_back(img);
+                object_locations.push_back(rects);
+                ignored_rects.push_back(ignored);
+            }
+        }
+
+        return ignored_rects;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    namespace impl
+    {
+        inline size_t num_non_ignored_boxes (const std::vector<mmod_rect>& rects)
+        {
+            size_t cnt = 0;
+            for (auto& b : rects)
+            {
+                if (!b.ignore)
+                    cnt++;
+            }
+            return cnt;
+        }
+    }
+
+    template <
+        typename array_type
+        >
+    void load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<mmod_rect> >& object_locations,
+        const image_dataset_file& source
+    )
+    {
+        images.clear();
+        object_locations.clear();
+
+        using namespace dlib::image_dataset_metadata;
+        dataset data;
+        load_image_dataset_metadata(data, source.get_filename());
+
+        // Set the current directory to be the one that contains the
+        // metadata file. We do this because the file might contain
+        // file paths which are relative to this folder.
+        locally_change_current_dir chdir(get_parent_directory(file(source.get_filename())));
+
+        typedef typename array_type::value_type image_type;
+
+        image_type img;
+        std::vector<mmod_rect> rects;
+        for (unsigned long i = 0; i < data.images.size(); ++i)
+        {
+            double min_rect_size = std::numeric_limits<double>::infinity();
+            rects.clear();
+            for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
+            {
+                if (source.should_load_box(data.images[i].boxes[j]))
+                {
+                    if (data.images[i].boxes[j].ignore)
+                    {
+                        rects.push_back(ignored_mmod_rect(data.images[i].boxes[j].rect));
+                    }
+                    else
+                    {
+                        rects.push_back(mmod_rect(data.images[i].boxes[j].rect));
+                        min_rect_size = std::min<double>(min_rect_size, rects.back().rect.area());
+                    }
+                    rects.back().label = data.images[i].boxes[j].label;
+
+                }
+            }
+
+            if (!source.should_skip_empty_images() || impl::num_non_ignored_boxes(rects) != 0)
+            {
+                load_image(img, data.images[i].filename);
+                if (rects.size() != 0)  
+                {
+                    // if shrinking the image would still result in the smallest box being
+                    // bigger than the box area threshold then shrink the image.
+                    while(min_rect_size/2/2 > source.box_area_thresh())
+                    {
+                        pyramid_down<2> pyr;
+                        pyr(img);
+                        min_rect_size *= (1.0/2.0)*(1.0/2.0);
+                        for (auto&& r : rects)
+                            r.rect = pyr.rect_down(r.rect);
+                    }
+                    while(min_rect_size*(2.0/3.0)*(2.0/3.0) > source.box_area_thresh())
+                    {
+                        pyramid_down<3> pyr;
+                        pyr(img);
+                        min_rect_size *= (2.0/3.0)*(2.0/3.0);
+                        for (auto&& r : rects)
+                            r.rect = pyr.rect_down(r.rect);
+                    }
+                }
+                images.push_back(std::move(img));
+                object_locations.push_back(std::move(rects));
+            }
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+// ******* THIS FUNCTION IS DEPRECATED, you should use another version of load_image_dataset() *******
+    template <
+        typename image_type, 
+        typename MM
+        >
+    std::vector<std::vector<rectangle> > load_image_dataset (
+        array<image_type,MM>& images,
+        std::vector<std::vector<rectangle> >& object_locations,
+        const std::string& filename,
+        const std::string& label,
+        bool skip_empty_images = false
+    )
+    {
+        image_dataset_file f(filename);
+        if (label.size() != 0)
+            f = f.boxes_match_label(label);
+        if (skip_empty_images)
+            f = f.skip_empty_images();
+        return load_image_dataset(images, object_locations, f);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type
+        >
+    std::vector<std::vector<rectangle> > load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<rectangle> >& object_locations,
+        const std::string& filename
+    )
+    {
+        return load_image_dataset(images, object_locations, image_dataset_file(filename));
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type
+        >
+    void load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<mmod_rect>>& object_locations,
+        const std::string& filename
+    )
+    {
+        load_image_dataset(images, object_locations, image_dataset_file(filename));
+    }
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type
+        >
+    std::vector<std::vector<rectangle> > load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<full_object_detection> >& object_locations,
+        const image_dataset_file& source,
+        std::vector<std::string>& parts_list
+    )
+    {
+        typedef typename array_type::value_type image_type;
+        parts_list.clear();
+        images.clear();
+        object_locations.clear();
+
+        using namespace dlib::image_dataset_metadata;
+        dataset data;
+        load_image_dataset_metadata(data, source.get_filename());
+
+        // Set the current directory to be the one that contains the
+        // metadata file. We do this because the file might contain
+        // file paths which are relative to this folder.
+        locally_change_current_dir chdir(get_parent_directory(file(source.get_filename())));
+
+
+        std::set<std::string> all_parts;
+
+        // find out what parts are being used in the dataset.  Store results in all_parts.
+        for (unsigned long i = 0; i < data.images.size(); ++i)
+        {
+            for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
+            {
+                if (source.should_load_box(data.images[i].boxes[j]))
+                {
+                    const std::map<std::string,point>& parts = data.images[i].boxes[j].parts;
+                    std::map<std::string,point>::const_iterator itr;
+
+                    for (itr = parts.begin(); itr != parts.end(); ++itr)
+                    {
+                        all_parts.insert(itr->first);
+                    }
+                }
+            }
+        }
+
+        // make a mapping between part names and the integers [0, all_parts.size())
+        std::map<std::string,int> parts_idx;
+        for (std::set<std::string>::iterator i = all_parts.begin(); i != all_parts.end(); ++i)
+        {
+            parts_idx[*i] = parts_list.size();
+            parts_list.push_back(*i);
+        }
+
+        std::vector<std::vector<rectangle> > ignored_rects;
+        std::vector<rectangle> ignored;
+        image_type img;
+        std::vector<full_object_detection> object_dets;
+        for (unsigned long i = 0; i < data.images.size(); ++i)
+        {
+            double min_rect_size = std::numeric_limits<double>::infinity();
+            object_dets.clear();
+            ignored.clear();
+            for (unsigned long j = 0; j < data.images[i].boxes.size(); ++j)
+            {
+                if (source.should_load_box(data.images[i].boxes[j]))
+                {
+                    if (data.images[i].boxes[j].ignore)
+                    {
+                        ignored.push_back(data.images[i].boxes[j].rect);
+                    }
+                    else
+                    {
+                        std::vector<point> partlist(parts_idx.size(), OBJECT_PART_NOT_PRESENT);
+
+                        // populate partlist with all the parts present in this box.
+                        const std::map<std::string,point>& parts = data.images[i].boxes[j].parts;
+                        std::map<std::string,point>::const_iterator itr;
+                        for (itr = parts.begin(); itr != parts.end(); ++itr)
+                        {
+                            partlist[parts_idx[itr->first]] = itr->second;
+                        }
+
+                        object_dets.push_back(full_object_detection(data.images[i].boxes[j].rect, partlist));
+                        min_rect_size = std::min<double>(min_rect_size, object_dets.back().get_rect().area());
+                    }
+                }
+            }
+
+            if (!source.should_skip_empty_images() || object_dets.size() != 0)
+            {
+                load_image(img, data.images[i].filename);
+                if (object_dets.size() != 0)  
+                {
+                    // if shrinking the image would still result in the smallest box being
+                    // bigger than the box area threshold then shrink the image.
+                    while(min_rect_size/2/2 > source.box_area_thresh())
+                    {
+                        pyramid_down<2> pyr;
+                        pyr(img);
+                        min_rect_size *= (1.0/2.0)*(1.0/2.0);
+                        for (auto&& r : object_dets)
+                        {
+                            r.get_rect() = pyr.rect_down(r.get_rect());
+                            for (unsigned long k = 0; k < r.num_parts(); ++k)
+                                r.part(k) = pyr.point_down(r.part(k));
+                        }
+                        for (auto&& r : ignored)
+                        {
+                            r = pyr.rect_down(r);
+                        }
+                    }
+                    while(min_rect_size*(2.0/3.0)*(2.0/3.0) > source.box_area_thresh())
+                    {
+                        pyramid_down<3> pyr;
+                        pyr(img);
+                        min_rect_size *= (2.0/3.0)*(2.0/3.0);
+                        for (auto&& r : object_dets)
+                        {
+                            r.get_rect() = pyr.rect_down(r.get_rect());
+                            for (unsigned long k = 0; k < r.num_parts(); ++k)
+                                r.part(k) = pyr.point_down(r.part(k));
+                        }
+                        for (auto&& r : ignored)
+                        {
+                            r = pyr.rect_down(r);
+                        }
+                    }
+                }
+                images.push_back(img);
+                object_locations.push_back(object_dets);
+                ignored_rects.push_back(ignored);
+            }
+        }
+
+
+        return ignored_rects;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type
+        >
+    std::vector<std::vector<rectangle> > load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<full_object_detection> >& object_locations,
+        const image_dataset_file& source 
+    )
+    {
+        std::vector<std::string> parts_list;
+        return load_image_dataset(images, object_locations, source, parts_list);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type 
+        >
+    std::vector<std::vector<rectangle> > load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<full_object_detection> >& object_locations,
+        const std::string& filename
+    )
+    {
+        std::vector<std::string> parts_list;
+        return load_image_dataset(images, object_locations, image_dataset_file(filename), parts_list);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_LOAD_IMAGE_DaTASET_Hh_
+

dlib/data_io/load_image_dataset_abstract.h

+// Copyright (C) 2012  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_LOAD_IMAGE_DaTASET_ABSTRACT_Hh_
+#ifdef DLIB_LOAD_IMAGE_DaTASET_ABSTRACT_Hh_
+
+#include "image_dataset_metadata.h"
+#include "../array/array_kernel_abstract.h"
+#include <string>
+#include <vector>
+#include "../image_processing/full_object_detection_abstract.h"
+
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    class image_dataset_file
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This object is a tool used to tell the load_image_dataset() functions which
+                boxes and images to load from an XML based image dataset file.  By default,
+                this object tells load_image_dataset() to load all images and object boxes.
+        !*/
+
+    public:
+        image_dataset_file(
+            const std::string& filename
+        );
+        /*!
+            ensures
+                - #get_filename() == filename
+                - #should_skip_empty_images() == false
+                - #get_selected_box_labels().size() == 0
+                  This means that, initially, all boxes will be loaded.  Therefore, for all
+                  possible boxes B we have:
+                    - #should_load_box(B) == true
+                - #box_area_thresh() == infinity
+        !*/
+
+        const std::string& get_filename(
+        ) const;
+        /*!
+            ensures
+                - returns the name of the XML image dataset metadata file given to this
+                  object's constructor.
+        !*/
+        
+        bool should_skip_empty_images(
+        ) const;
+        /*!
+            ensures
+                - returns true if we are supposed to skip images that don't have any
+                  non-ignored boxes to load when loading an image dataset using
+                  load_image_dataset().
+        !*/
+
+        image_dataset_file boxes_match_label(
+            const std::string& label
+        ) const;
+        /*!
+            ensures
+                - returns a copy of *this that is identical in all respects to *this except
+                  that label will be included in the labels set (i.e. the set returned by
+                  get_selected_box_labels()).
+        !*/
+
+        const std::set<std::string>& get_selected_box_labels(
+        ) const;
+        /*!
+            ensures
+                - returns the set of box labels currently selected by the should_load_box()
+                  method.  Note that if the set is empty then we select all boxes.
+        !*/
+
+        image_dataset_file skip_empty_images(
+        ) const;
+        /*!
+            ensures
+                - returns a copy of *this that is identical in all respects to *this except
+                  that #should_skip_empty_images() == true.
+        !*/
+
+        bool should_boxes_have_parts(
+        ) const; 
+        /*!
+            ensures
+                - returns true if boxes must have some parts defined for them to be loaded.
+        !*/
+
+        image_dataset_file boxes_have_parts(
+        ) const;
+        /*!
+            ensures
+                - returns a copy of *this that is identical in all respects to *this except
+                  that #should_boxes_have_parts() == true.
+        !*/
+
+        bool should_load_box (
+            const image_dataset_metadata::box& box
+        ) const;
+        /*!
+            ensures
+                - returns true if we are supposed to load the given box from an image
+                  dataset XML file.  In particular, if should_load_box() returns false then
+                  the load_image_dataset() routines will not return the box at all, neither
+                  in the ignore rectangles list or in the primary object_locations vector.
+                  The behavior of this function is defined as follows:
+                    - if (should_boxes_have_parts() && boxes.parts.size() == 0) then
+                        - returns false
+                    - else if (get_selected_box_labels().size() == 0) then
+                        - returns true
+                    - else if (get_selected_box_labels().count(box.label) != 0) then
+                        - returns true
+                    - else
+                        - returns false
+        !*/
+
+        image_dataset_file shrink_big_images(
+            double new_box_area_thresh = 150*150
+        ) const;
+        /*!
+            ensures
+                - returns a copy of *this that is identical in all respects to *this except
+                  that #box_area_thresh() == new_box_area_thresh
+        !*/
+
+        double box_area_thresh(
+        ) const;
+        /*!
+            ensures
+                - If the smallest non-ignored rectangle in an image has an area greater
+                  than box_area_thresh() then we will shrink the image until the area of
+                  the box is about equal to box_area_thresh().  This is useful if you have
+                  a dataset containing very high resolution images and you don't want to
+                  load it in its native high resolution.  Setting the box_area_thresh()
+                  allows you to control the resolution of the loaded images.
+        !*/
+    };
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type 
+        >
+    std::vector<std::vector<rectangle> > load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<rectangle> >& object_locations,
+        const image_dataset_file& source
+    );
+    /*!
+        requires
+            - array_type == An array of images.  This is anything with an interface that
+              looks like std::vector<some generic image type> where a "generic image" is
+              anything that implements the generic image interface defined in
+              dlib/image_processing/generic_image.h.
+        ensures
+            - This routine loads the images and their associated object boxes from the
+              image metadata file indicated by source.get_filename().  This metadata file
+              should be in the XML format used by the save_image_dataset_metadata() routine.
+            - #images.size() == The number of images loaded from the metadata file.  This
+              is all the images listed in the file unless source.should_skip_empty_images()
+              is set to true.
+            - #images.size() == #object_locations.size()
+            - This routine is capable of loading any image format which can be read by the
+              load_image() routine.
+            - let IGNORED_RECTS denote the vector returned from this function.
+            - IGNORED_RECTS.size() == #object_locations.size()
+            - IGNORED_RECTS == a list of the rectangles which have the "ignore" flag set to
+              true in the input XML file.
+            - for all valid i:  
+                - #images[i] == a copy of the i-th image from the dataset.
+                - #object_locations[i] == a vector of all the rectangles associated with
+                  #images[i].  These are the rectangles for which source.should_load_box()
+                  returns true and are also not marked as "ignore" in the XML file.
+                - IGNORED_RECTS[i] == A vector of all the rectangles associated with #images[i] 
+                  that are marked as "ignore" but not discarded by source.should_load_box().
+                - if (source.should_skip_empty_images() == true) then
+                    - #object_locations[i].size() != 0
+                      (i.e. we won't load images that don't end up having any object locations)
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type 
+        >
+    std::vector<std::vector<rectangle> > load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<rectangle> >& object_locations,
+        const std::string& filename
+    );
+    /*!
+        requires
+            - array_type == An array of images.  This is anything with an interface that
+              looks like std::vector<some generic image type> where a "generic image" is
+              anything that implements the generic image interface defined in
+              dlib/image_processing/generic_image.h.
+        ensures
+            - performs: return load_image_dataset(images, object_locations, image_dataset_file(filename));
+              (i.e. it ignores box labels and therefore loads all the boxes in the dataset)
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type
+        >
+    void load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<mmod_rect> >& object_locations,
+        const image_dataset_file& source
+    );
+    /*!
+        requires
+            - array_type == An array of images.  This is anything with an interface that
+              looks like std::vector<some generic image type> where a "generic image" is
+              anything that implements the generic image interface defined in
+              dlib/image_processing/generic_image.h.
+        ensures
+            - This function has essentially the same behavior as the above
+              load_image_dataset() routines, except here we output to a vector of
+              mmod_rects instead of rectangles.  In this case, both ignore and non-ignore
+              rectangles go into object_locations since mmod_rect has an ignore boolean
+              field that records the ignored/non-ignored state of each rectangle.  We also store 
+              a each box's string label into the mmod_rect::label field as well.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type 
+        >
+    void load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<mmod_rect> >& object_locations,
+        const std::string& filename
+    );
+    /*!
+        requires
+            - array_type == An array of images.  This is anything with an interface that
+              looks like std::vector<some generic image type> where a "generic image" is
+              anything that implements the generic image interface defined in
+              dlib/image_processing/generic_image.h.
+        ensures
+            - performs: load_image_dataset(images, object_locations, image_dataset_file(filename));
+              (i.e. it ignores box labels and therefore loads all the boxes in the dataset)
+    !*/
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type
+        >
+    std::vector<std::vector<rectangle> > load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<full_object_detection> >& object_locations,
+        const image_dataset_file& source,
+        std::vector<std::string>& parts_list
+    );
+    /*!
+        requires
+            - array_type == An array of images.  This is anything with an interface that
+              looks like std::vector<some generic image type> where a "generic image" is
+              anything that implements the generic image interface defined in
+              dlib/image_processing/generic_image.h.
+        ensures
+            - This routine loads the images and their associated object locations from the
+              image metadata file indicated by source.get_filename().  This metadata file
+              should be in the XML format used by the save_image_dataset_metadata() routine.
+            - The difference between this function and the version of load_image_dataset()
+              defined above is that this version will also load object part information and
+              thus fully populates the full_object_detection objects.
+            - #images.size() == The number of images loaded from the metadata file.  This
+              is all the images listed in the file unless source.should_skip_empty_images()
+              is set to true.
+            - #images.size() == #object_locations.size()
+            - This routine is capable of loading any image format which can be read
+              by the load_image() routine.
+            - #parts_list == a vector that contains the list of object parts found in the
+              input file and loaded into object_locations.
+            - #parts_list is in lexicographic sorted order.
+            - let IGNORED_RECTS denote the vector returned from this function.
+            - IGNORED_RECTS.size() == #object_locations.size()
+            - IGNORED_RECTS == a list of the rectangles which have the "ignore" flag set to
+              true in the input XML file.
+            - for all valid i:  
+                - #images[i] == a copy of the i-th image from the dataset.
+                - #object_locations[i] == a vector of all the rectangles associated with
+                  #images[i].  These are the rectangles for which source.should_load_box()
+                  returns true and are also not marked as "ignore" in the XML file.
+                - IGNORED_RECTS[i] == A vector of all the rectangles associated with #images[i] 
+                  that are marked as "ignore" but not discarded by source.should_load_box().
+                - if (source.should_skip_empty_images() == true) then
+                    - #object_locations[i].size() != 0
+                      (i.e. we won't load images that don't end up having any object locations)
+                - for all valid j:
+                    - #object_locations[i][j].num_parts() == #parts_list.size()
+                    - for all valid k:
+                        - #object_locations[i][j].part(k) == the location of the part
+                          with name #parts_list[k] or OBJECT_PART_NOT_PRESENT if the
+                          part was not indicated for object #object_locations[i][j].
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type 
+        >
+    std::vector<std::vector<rectangle> > load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<full_object_detection> >& object_locations,
+        const image_dataset_file& source 
+    );
+    /*!
+        requires
+            - array_type == An array of images.  This is anything with an interface that
+              looks like std::vector<some generic image type> where a "generic image" is
+              anything that implements the generic image interface defined in
+              dlib/image_processing/generic_image.h.
+        ensures
+            - performs: return load_image_dataset(images, object_locations, source, parts_list);
+              (i.e. this function simply calls the above function and discards the output
+              parts_list.  So it is just a convenience function you can call if you don't
+              care about getting the parts list.)
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename array_type 
+        >
+    std::vector<std::vector<rectangle> > load_image_dataset (
+        array_type& images,
+        std::vector<std::vector<full_object_detection> >& object_locations,
+        const std::string& filename
+    );
+    /*!
+        requires
+            - array_type == An array of images.  This is anything with an interface that
+              looks like std::vector<some generic image type> where a "generic image" is
+              anything that implements the generic image interface defined in
+              dlib/image_processing/generic_image.h.
+        ensures
+            - performs: return load_image_dataset(images, object_locations, image_dataset_file(filename));
+              (i.e. it ignores box labels and therefore loads all the boxes in the dataset)
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_LOAD_IMAGE_DaTASET_ABSTRACT_Hh_
+
+

dlib/data_io/mnist.cpp

+// Copyright (C) 2015  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_MNIST_CPp_
+#define DLIB_MNIST_CPp_
+
+#include "mnist.h"
+#include <fstream>
+#include "../byte_orderer.h"
+#include "../uintn.h"
+
+// ----------------------------------------------------------------------------------------
+
+namespace dlib
+{
+    void load_mnist_dataset (
+        const std::string& folder_name,
+        std::vector<matrix<unsigned char> >& training_images,
+        std::vector<unsigned long>& training_labels,
+        std::vector<matrix<unsigned char> >& testing_images,
+        std::vector<unsigned long>& testing_labels
+    )
+    {
+        using namespace std;
+        ifstream fin1((folder_name+"/train-images-idx3-ubyte").c_str(), ios::binary);
+        if (!fin1)
+        {
+            fin1.open((folder_name + "/train-images.idx3-ubyte").c_str(), ios::binary);
+        }
+
+        ifstream fin2((folder_name+"/train-labels-idx1-ubyte").c_str(), ios::binary);
+        if (!fin2)
+        {
+            fin2.open((folder_name + "/train-labels.idx1-ubyte").c_str(), ios::binary);
+        }
+
+        ifstream fin3((folder_name+"/t10k-images-idx3-ubyte").c_str(), ios::binary);
+        if (!fin3)
+        {
+            fin3.open((folder_name + "/t10k-images.idx3-ubyte").c_str(), ios::binary);
+        }
+
+        ifstream fin4((folder_name+"/t10k-labels-idx1-ubyte").c_str(), ios::binary);
+        if (!fin4)
+        {
+            fin4.open((folder_name + "/t10k-labels.idx1-ubyte").c_str(), ios::binary);
+        }
+
+        if (!fin1) throw error("Unable to open file train-images-idx3-ubyte or train-images.idx3-ubyte");
+        if (!fin2) throw error("Unable to open file train-labels-idx1-ubyte or train-labels.idx1-ubyte");
+        if (!fin3) throw error("Unable to open file t10k-images-idx3-ubyte or t10k-images.idx3-ubyte");
+        if (!fin4) throw error("Unable to open file t10k-labels-idx1-ubyte or t10k-labels.idx1-ubyte");
+		
+        byte_orderer bo;
+
+        // make sure the files have the contents we expect.
+        uint32 magic, num, nr, nc, num2, num3, num4;
+        fin1.read((char*)&magic, sizeof(magic));  bo.big_to_host(magic);
+        fin1.read((char*)&num, sizeof(num));  bo.big_to_host(num);
+        fin1.read((char*)&nr, sizeof(nr));  bo.big_to_host(nr);
+        fin1.read((char*)&nc, sizeof(nc));  bo.big_to_host(nc);
+        if (magic != 2051 || num != 60000 || nr != 28 || nc != 28)
+            throw error("mnist dat files are corrupted.");
+
+        fin2.read((char*)&magic, sizeof(magic));  bo.big_to_host(magic);
+        fin2.read((char*)&num2, sizeof(num2));  bo.big_to_host(num2);
+        if (magic != 2049 || num2 != 60000)
+            throw error("mnist dat files are corrupted.");
+
+        fin3.read((char*)&magic, sizeof(magic));  bo.big_to_host(magic);
+        fin3.read((char*)&num3, sizeof(num3));  bo.big_to_host(num3);
+        fin3.read((char*)&nr, sizeof(nr));  bo.big_to_host(nr);
+        fin3.read((char*)&nc, sizeof(nc));  bo.big_to_host(nc);
+        if (magic != 2051 || num3 != 10000 || nr != 28 || nc != 28)
+            throw error("mnist dat files are corrupted.");
+
+        fin4.read((char*)&magic, sizeof(magic));  bo.big_to_host(magic);
+        fin4.read((char*)&num4, sizeof(num4));  bo.big_to_host(num4);
+        if (magic != 2049 || num4 != 10000)
+            throw error("mnist dat files are corrupted.");
+
+        if (!fin1) throw error("Unable to read train-images-idx3-ubyte");
+        if (!fin2) throw error("Unable to read train-labels-idx1-ubyte");
+        if (!fin3) throw error("Unable to read t10k-images-idx3-ubyte");
+        if (!fin4) throw error("Unable to read t10k-labels-idx1-ubyte");
+
+
+        training_images.resize(60000);
+        training_labels.resize(60000);
+        testing_images.resize(10000);
+        testing_labels.resize(10000);
+
+        for (size_t i = 0; i < training_images.size(); ++i)
+        {
+            training_images[i].set_size(nr,nc);
+            fin1.read((char*)&training_images[i](0,0), nr*nc);
+        }
+        for (size_t i = 0; i < training_labels.size(); ++i)
+        {
+            char l;
+            fin2.read(&l, 1);
+            training_labels[i] = l;
+        }
+
+        for (size_t i = 0; i < testing_images.size(); ++i)
+        {
+            testing_images[i].set_size(nr,nc);
+            fin3.read((char*)&testing_images[i](0,0), nr*nc);
+        }
+        for (size_t i = 0; i < testing_labels.size(); ++i)
+        {
+            char l;
+            fin4.read(&l, 1);
+            testing_labels[i] = l;
+        }
+
+        if (!fin1) throw error("Unable to read train-images-idx3-ubyte");
+        if (!fin2) throw error("Unable to read train-labels-idx1-ubyte");
+        if (!fin3) throw error("Unable to read t10k-images-idx3-ubyte");
+        if (!fin4) throw error("Unable to read t10k-labels-idx1-ubyte");
+
+        if (fin1.get() != EOF) throw error("Unexpected bytes at end of train-images-idx3-ubyte");
+        if (fin2.get() != EOF) throw error("Unexpected bytes at end of train-labels-idx1-ubyte");
+        if (fin3.get() != EOF) throw error("Unexpected bytes at end of t10k-images-idx3-ubyte");
+        if (fin4.get() != EOF) throw error("Unexpected bytes at end of t10k-labels-idx1-ubyte");
+    }
+}
+
+// ----------------------------------------------------------------------------------------
+
+#endif // DLIB_MNIST_CPp_
+
+
+

dlib/data_io/mnist.h

+// Copyright (C) 2015  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_MNIST_Hh_
+#define DLIB_MNIST_Hh_
+
+#include "mnist_abstract.h"
+#include <string>
+#include <vector>
+#include "../matrix.h"
+
+// ----------------------------------------------------------------------------------------
+
+namespace dlib
+{
+    void load_mnist_dataset (
+        const std::string& folder_name,
+        std::vector<matrix<unsigned char> >& training_images,
+        std::vector<unsigned long>& training_labels,
+        std::vector<matrix<unsigned char> >& testing_images,
+        std::vector<unsigned long>& testing_labels
+    );
+}
+
+// ----------------------------------------------------------------------------------------
+
+#ifdef NO_MAKEFILE
+#include "mnist.cpp"
+#endif
+
+#endif // DLIB_MNIST_Hh_
+
+

dlib/data_io/mnist_abstract.h

+// Copyright (C) 2015  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_MNIST_ABSTRACT_Hh_
+#ifdef DLIB_MNIST_ABSTRACT_Hh_
+
+#include <string>
+#include <vector>
+#include "../matrix.h"
+
+// ----------------------------------------------------------------------------------------
+
+namespace dlib
+{
+    void load_mnist_dataset (
+        const std::string& folder_name,
+        std::vector<matrix<unsigned char> >& training_images,
+        std::vector<unsigned long>& training_labels,
+        std::vector<matrix<unsigned char> >& testing_images,
+        std::vector<unsigned long>& testing_labels
+    );
+    /*!
+        ensures
+            - Attempts to load the MNIST dataset from the hard drive.  This is the dataset
+              of handwritten digits available from http://yann.lecun.com/exdb/mnist/. In
+              particular, the 4 files comprising the MNIST dataset should be present in the
+              folder indicated by folder_name.  These four files are:
+                - train-images-idx3-ubyte
+                - train-labels-idx1-ubyte
+                - t10k-images-idx3-ubyte
+                - t10k-labels-idx1-ubyte
+            - #training_images == The 60,000 training images from the dataset. 
+            - #training_labels == The labels for the contents of #training_images.  
+              I.e. #training_labels[i] is the label of #training_images[i].
+            - #testing_images == The 10,000 testing images from the dataset. 
+            - #testing_labels == The labels for the contents of #testing_images.  
+              I.e. #testing_labels[i] is the label of #testing_images[i].
+        throws
+            - dlib::error if some problem prevents us from loading the data or the files
+              can't be found.
+    !*/
+}
+
+// ----------------------------------------------------------------------------------------
+
+#endif // DLIB_MNIST_ABSTRACT_Hh_
+

dlib/dir_nav.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DIR_NAv_
+#define DLIB_DIR_NAv_
+
+
+#include "platform.h"
+
+
+#ifdef WIN32
+#include "dir_nav/windows.h"
+#endif
+
+#ifndef WIN32
+#include "dir_nav/posix.h"
+#endif
+
+#include "dir_nav/dir_nav_extensions.h"
+
+#endif // DLIB_DIR_NAv_
+

dlib/dir_nav/dir_nav_extensions.cpp

+// Copyright (C) 2009  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DIR_NAV_EXTENSIONs_CPP_
+#define DLIB_DIR_NAV_EXTENSIONs_CPP_
+
+#include "dir_nav_extensions.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    namespace implementation_details
+    {
+        void get_all_sub_dirs (
+            const directory& top_of_tree,
+            unsigned long max_depth,
+            std::vector<directory>& result,
+            std::vector<directory>& temp
+        )
+        {
+            if (max_depth > 0)
+            {
+                top_of_tree.get_dirs(temp);
+                const unsigned long start = result.size();
+                result.insert(result.end(), temp.begin(), temp.end());
+                const unsigned long end = start + temp.size();
+
+                for (unsigned long i = start; i < end; ++i)
+                {
+                    get_all_sub_dirs(result[i], max_depth-1, result, temp);
+                }
+            }
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    bool file_exists (
+        const std::string& filename
+    )
+    {
+        try
+        {
+            dlib::file temp(filename);
+            return true;
+        }
+        catch (file::file_not_found&)
+        {
+            return false;
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    bool directory_exists (
+        const std::string& dirname
+    )
+    {
+        try
+        {
+            dlib::directory temp(dirname);
+            return true;
+        }
+        catch (directory::dir_not_found&)
+        {
+            return false;
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    directory get_parent_directory (
+        const directory& dir
+    )
+    {
+        return dir.get_parent();
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    directory get_parent_directory (
+        const file& f
+    )
+    {
+        if (f.full_name().size() == 0)
+            return directory();
+
+        std::string::size_type pos = f.full_name().find_last_of("\\/");
+
+        if (pos == std::string::npos)
+            return directory();
+
+        return directory(f.full_name().substr(0,pos));
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    std::string select_oldest_file (
+        const std::string& filename1,
+        const std::string& filename2
+    )
+    {
+        file f1, f2;
+        try{f1 = file(filename1);} catch(file::file_not_found&) { return filename1; }
+        try{f2 = file(filename2);} catch(file::file_not_found&) { return filename2; }
+
+        if (f1.last_modified() < f2.last_modified())
+            return filename1;
+        else
+            return filename2;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    std::string select_newest_file (
+        const std::string& filename1,
+        const std::string& filename2
+    )
+    {
+        file f1, f2;
+        try{f1 = file(filename1);} catch(file::file_not_found&) { return filename2; }
+        try{f2 = file(filename2);} catch(file::file_not_found&) { return filename1; }
+
+        if (f1.last_modified() > f2.last_modified())
+            return filename1;
+        else
+            return filename2;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_DIR_NAV_EXTENSIONs_CPP_

dlib/dir_nav/dir_nav_extensions.h

+// Copyright (C) 2009  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DIR_NAV_EXTENSIONs_H_
+#define DLIB_DIR_NAV_EXTENSIONs_H_
+
+#include <string>
+#include <vector>
+#include <algorithm>
+#include "dir_nav_extensions_abstract.h"
+#include "../dir_nav.h"
+#include "../string.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    bool file_exists (
+        const std::string& filename
+    );
+
+// ----------------------------------------------------------------------------------------
+
+    bool directory_exists (
+        const std::string& dirname
+    );
+
+// ----------------------------------------------------------------------------------------
+
+    namespace implementation_details
+    {
+        void get_all_sub_dirs (
+            const directory& top_of_tree,
+            unsigned long max_depth,
+            std::vector<directory>& result,
+            std::vector<directory>& temp
+        );
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename T>
+    const std::vector<file> get_files_in_directory_tree (
+        const directory& top_of_tree,
+        const T& add_file,
+        unsigned long max_depth = 30
+    )
+    {
+        std::vector<file> result, temp;
+        std::vector<directory> dirs, dirs_temp;
+        dirs.push_back(top_of_tree);
+
+        // get all the directories in the tree first
+        implementation_details::get_all_sub_dirs(top_of_tree, max_depth, dirs, dirs_temp);
+
+        // now just loop over all the directories and pick out the files we want to keep
+        for (unsigned long d = 0; d < dirs.size(); ++d)
+        {
+            dirs[d].get_files(temp);
+
+            // pick out the members of temp that we should keep
+            for (unsigned long i = 0; i < temp.size(); ++i)
+            {
+                if (add_file(temp[i]))
+                    result.push_back(temp[i]);
+            }
+        }
+
+        return result;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    class match_ending
+    {
+
+    public:
+        match_ending ( 
+            const std::string& ending_
+        ) : ending(ending_) {}
+
+        bool operator() (
+            const file& f
+        ) const
+        {
+            // if the ending is bigger than f's name then it obviously doesn't match
+            if (ending.size() > f.name().size())
+                return false;
+
+            // now check if the actual characters that make up the end of the file name 
+            // matches what is in ending.
+            return std::equal(ending.begin(), ending.end(), f.name().end()-ending.size());
+        }
+
+    private:
+        std::string ending;
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    class match_endings
+    {
+
+    public:
+        match_endings ( 
+            const std::string& endings_
+        ) 
+        {
+            const std::vector<std::string>& s = split(endings_);
+            for (unsigned long i = 0; i < s.size(); ++i)
+            {
+                endings.push_back(match_ending(s[i]));
+            }
+        }
+
+        bool operator() (
+            const file& f
+        ) const
+        {
+            for (unsigned long i = 0; i < endings.size(); ++i)
+            {
+                if (endings[i](f))
+                    return true;
+            }
+
+            return false;
+        }
+
+    private:
+        std::vector<match_ending> endings;
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    class match_all
+    {
+    public:
+        bool operator() (
+            const file& 
+        ) const { return true; }
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    directory get_parent_directory (
+        const directory& dir
+    );
+
+// ----------------------------------------------------------------------------------------
+
+    directory get_parent_directory (
+        const file& f
+    );
+
+// ----------------------------------------------------------------------------------------
+
+    std::string select_oldest_file (
+        const std::string& filename1,
+        const std::string& filename2
+    );
+
+// ----------------------------------------------------------------------------------------
+
+    std::string select_newest_file (
+        const std::string& filename1,
+        const std::string& filename2
+    );
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#ifdef NO_MAKEFILE
+#include "dir_nav_extensions.cpp"
+#endif
+
+#endif // DLIB_DIR_NAV_EXTENSIONs_H_
+

dlib/dir_nav/dir_nav_extensions_abstract.h

+// Copyright (C) 2009  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_DIR_NAV_EXTENSIONs_ABSTRACT_
+#ifdef DLIB_DIR_NAV_EXTENSIONs_ABSTRACT_
+
+#include <string>
+#include <vector>
+#include "dir_nav_kernel_abstract.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    bool file_exists (
+        const std::string& filename
+    );
+    /*!
+        ensures
+            - if (a file with the given filename exists) then
+                - returns true
+            - else
+                - returns false
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    bool directory_exists (
+        const std::string& dirname
+    );
+    /*!
+        ensures
+            - if (a directory with the given dirname exists) then
+                - returns true
+            - else
+                - returns false
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename T>
+    const std::vector<file> get_files_in_directory_tree (
+        const directory& top_of_tree,
+        const T& add_file,
+        unsigned long max_depth = 30
+    );
+    /*!
+        requires
+            - add_file must be a function object with the following prototype:
+                bool add_file (file f);
+        ensures
+            - performs a recursive search through the directory top_of_tree and all
+              its sub-directories (up to the given max depth).  All files in these
+              directories are examined by passing them to add_file() and if it 
+              returns true then they will be included in the returned std::vector<file>
+              object.
+            - Note that a max_depth of 0 means that only the files in the directory
+              top_of_tree will be considered.  A depth of 1 means that only files in 
+              top_of_tree and its immediate sub-directories will be considered.  And
+              so on...
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    class match_ending
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This is a simple function object that can be used with the
+                above get_files_in_directory_tree() function.  This object
+                just looks for files with a certain ending.
+        !*/
+
+    public:
+        match_ending ( 
+            const std::string& ending
+        );
+        /*!
+            ensures
+                - this object will be a function that checks if a file has a 
+                  name that ends with the given ending string.
+        !*/
+
+        bool operator() (
+            const file& f
+        ) const;
+        /*!
+            ensures
+                - if (the file f has a name that ends with the ending string given
+                  to this object's constructor) then
+                    - returns true
+                - else
+                    - returns false
+        !*/
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    class match_endings
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This is a simple function object that can be used with the
+                above get_files_in_directory_tree() function.  This object
+                allows you to look for files with a number of different 
+                endings.
+        !*/
+
+    public:
+        match_endings ( 
+            const std::string& ending_list
+        );
+        /*!
+            ensures
+                - ending_list is interpreted as a whitespace separated list
+                  of file endings. 
+                - this object will be a function that checks if a file has a 
+                  name that ends with one of the strings in ending_list.
+        !*/
+
+        bool operator() (
+            const file& f
+        ) const;
+        /*!
+            ensures
+                - if (the file f has a name that ends with one of the ending strings 
+                  given to this object's constructor) then
+                    - returns true
+                - else
+                    - returns false
+        !*/
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    class match_all
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This is a simple function object that can be used with the
+                above get_files_in_directory_tree() function.  This object
+                matches all files. 
+        !*/
+
+    public:
+        bool operator() (
+            const file& f
+        ) const;
+        /*!
+            ensures
+                - returns true
+                  (i.e. this function doesn't do anything.  It just says it
+                  matches all files no matter what)
+        !*/
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    directory get_parent_directory (
+        const directory& dir
+    );
+    /*!
+        ensures
+            - returns the parent directory of dir.  In particular, this
+              function returns the value of dir.get_parent()
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    directory get_parent_directory (
+        const file& f
+    );
+    /*!
+        ensures
+            - if (f.full_name() != "") then
+                - returns the directory which contains the given file
+            - else
+                - returns a default initialized directory (i.e. directory())
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    std::string select_oldest_file (
+        const std::string& filename1,
+        const std::string& filename2
+    );
+    /*!
+        ensures
+            - Checks the last modification times of the two given files and returns the
+              filename of the oldest file, i.e., the file that has gone longest since being
+              modified.  Ties are broken arbitrarily. 
+            - For the purpose of comparison, a file that doesn't exist is presumed to have
+              a last modification time of -infinity (i.e. very far in the past).
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    std::string select_newest_file (
+        const std::string& filename1,
+        const std::string& filename2
+    );
+    /*!
+        ensures
+            - Checks the last modification times of the two given files and returns the
+              filename that was most recently modified.  Ties are broken arbitrarily. 
+            - For the purpose of comparison, a file that doesn't exist is presumed to have
+              a last modification time of -infinity (i.e. very far in the past).
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_DIR_NAV_EXTENSIONs_ABSTRACT_
+
+

dlib/dir_nav/dir_nav_kernel_1.cpp

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DIR_NAV_KERNEL_1_CPp_
+#define DLIB_DIR_NAV_KERNEL_1_CPp_
+#include "../platform.h"
+
+#ifdef WIN32
+
+#include "dir_nav_kernel_1.h"
+#include "../string.h"
+
+
+#ifdef __BORLANDC__
+// Apparently the borland compiler doesn't define this.
+#define INVALID_FILE_ATTRIBUTES ((DWORD)-1)
+#endif
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // file object implementation
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    void file::
+    init (
+        const std::string& name
+    )
+    {
+        using namespace std;
+
+
+        char buf[3000];
+        char* str;
+        if (GetFullPathNameA(name.c_str(),sizeof(buf),buf,&str) == 0)
+        {
+            // the file was not found
+            throw file_not_found("Unable to find file " + name);
+        }
+        state.full_name = buf;
+        
+
+        string::size_type pos = state.full_name.find_last_of(directory::get_separator());
+        if (pos == string::npos)
+        {
+            // no valid full path has no separator characters.  
+            throw file_not_found("Unable to find file " + name);
+        }
+        state.name = state.full_name.substr(pos+1);
+
+
+        // now find the size of this file
+        WIN32_FIND_DATAA data;
+        HANDLE ffind = FindFirstFileA(state.full_name.c_str(), &data);
+        if (ffind == INVALID_HANDLE_VALUE ||
+            (data.dwFileAttributes&FILE_ATTRIBUTE_DIRECTORY) != 0)
+        {
+            throw file_not_found("Unable to find file " + name);                
+        }
+        else
+        {
+            uint64 temp = data.nFileSizeHigh;            
+            temp <<= 32;
+            temp |= data.nFileSizeLow;
+            state.file_size = temp;
+            FindClose(ffind);
+
+            ULARGE_INTEGER ull;
+            ull.LowPart = data.ftLastWriteTime.dwLowDateTime;
+            ull.HighPart = data.ftLastWriteTime.dwHighDateTime;
+            std::chrono::nanoseconds epoch(100 * (ull.QuadPart - 116444736000000000));            
+            state.last_modified = std::chrono::time_point<std::chrono::system_clock>(std::chrono::duration_cast<std::chrono::system_clock::duration>(epoch));
+        } 
+
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    bool file::
+    operator == (
+        const file& rhs
+    ) const 
+    { 
+        using namespace std;
+
+        if (state.full_name.size() != rhs.state.full_name.size())
+            return false;
+        
+        // compare the strings but ignore the case because file names
+        // are not case sensitive on windows
+        return tolower(state.full_name) == tolower(rhs.state.full_name);
+    }
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // directory object implementation
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    void directory::
+    init (
+        const std::string& name
+    )
+    {
+        using namespace std;
+
+        
+        char buf[3000];
+        char* str;
+        if (GetFullPathNameA(name.c_str(),sizeof(buf),buf,&str) == 0)
+        {
+            // the directory was not found
+            throw dir_not_found("Unable to find directory " + name);
+        }
+        state.full_name = buf;
+  
+        
+        const char sep = get_separator();
+        if (is_root_path(state.full_name) == false)
+        {
+            // ensure that thre is not a trialing separator
+            if (state.full_name[state.full_name.size()-1] == sep)
+                state.full_name.erase(state.full_name.size()-1);
+
+            // pick out the directory name
+            string::size_type pos = state.full_name.find_last_of(sep);
+            state.name = state.full_name.substr(pos+1);
+        }
+        else
+        {
+            // ensure that there is a trailing separator
+            if (state.full_name[state.full_name.size()-1] != sep)
+                state.full_name += sep;
+        }
+
+
+        // now check that this is actually a valid directory
+        DWORD attribs = GetFileAttributesA(state.full_name.c_str());
+        if (attribs == INVALID_FILE_ATTRIBUTES ||
+            (attribs&FILE_ATTRIBUTE_DIRECTORY) == 0)
+        {
+            // the directory was not found
+            throw dir_not_found("Unable to find directory " + name);
+        }
+
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    char directory::
+    get_separator (
+    ) 
+    {
+        return '\\';
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    bool directory::
+    operator == (
+        const directory& rhs
+    ) const 
+    { 
+        using namespace std;
+
+        if (state.full_name.size() != rhs.state.full_name.size())
+            return false;
+
+        // compare the strings but ignore the case because file names
+        // are not case sensitive on windows
+        return tolower(state.full_name) == tolower(rhs.state.full_name);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    const directory directory::
+    get_parent (
+    ) const
+    {
+        using namespace std;
+        // if *this is the root then just return *this
+        if (is_root())
+        {
+            return *this;
+        }
+        else
+        {
+            directory temp;
+
+            const char sep = get_separator();
+
+            string::size_type pos = state.full_name.find_last_of(sep);
+            temp.state.full_name = state.full_name.substr(0,pos);
+
+            if ( is_root_path(temp.state.full_name))
+            {
+                temp.state.full_name += sep;
+            }
+            else
+            {
+                pos = temp.state.full_name.find_last_of(sep);
+                if (pos != string::npos)
+                {
+                    temp.state.name = temp.state.full_name.substr(pos+1);
+                }
+                else
+                {
+                    temp.state.full_name += sep;
+                }
+            }
+            return temp;
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    bool directory::
+    is_root_path (
+        const std::string& path
+    ) const
+    {
+        using namespace std;
+        const char sep = get_separator();
+        bool root_path = false;
+        if (path.size() > 2 && path[0] == sep && path[1] == sep)
+        {
+            // in this case this is a windows share path
+            string::size_type pos = path.find_first_of(sep,2);
+            if (pos != string::npos)
+            {                
+                pos = path.find_first_of(sep,pos+1);
+
+                if (pos == string::npos && path[path.size()-1] != sep)
+                    root_path = true;
+                else if (pos == path.size()-1)
+                    root_path = true;
+            }
+
+        }
+        else if ( (path.size() == 2 || path.size() == 3) && path[1] == ':')
+        {
+            // if this is a valid windows path then it must be a root path
+            root_path = true;
+        }
+
+        return root_path;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // WIN32
+
+#endif // DLIB_DIR_NAV_KERNEL_1_CPp_
+

dlib/dir_nav/dir_nav_kernel_1.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DIR_NAV_KERNEl_1_
+#define DLIB_DIR_NAV_KERNEl_1_
+
+#ifdef DLIB_ISO_CPP_ONLY
+#error "DLIB_ISO_CPP_ONLY is defined so you can't use this OS dependent code.  Turn DLIB_ISO_CPP_ONLY off if you want to use it."
+#endif
+
+#include "../platform.h"
+
+
+#include "dir_nav_kernel_abstract.h"
+#include <string>
+#include "../uintn.h"
+#include "../algs.h"
+
+#include "../windows_magic.h"
+#include <windows.h>
+#include <vector>
+#include "../stl_checked.h"
+#include "../enable_if.h"
+#include "../queue.h"
+#include <chrono>
+
+namespace dlib
+{
+
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // file object    
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    
+    class file
+    {
+        /*!
+            INITIAL VALUES
+                state.name        == name()
+                state.full_name   == full_name()
+                state.file_size   == size()
+                state.last_modified == last_modified()
+
+            CONVENTION
+                state.name        == name()
+                state.full_name   == full_name()
+                state.file_size   == size()
+                state.last_modified == last_modified()
+
+        !*/
+
+        friend class directory;
+
+        struct data
+        {
+            uint64 file_size;
+            std::string name;
+            std::string full_name;
+            std::chrono::time_point<std::chrono::system_clock> last_modified;
+        };
+
+
+        void init ( const std::string& name);
+
+    public:
+
+        struct private_constructor{};
+        inline file (
+            const std::string& name,
+            const std::string& full_name,
+            const uint64 file_size,
+            const std::chrono::time_point<std::chrono::system_clock>& last_modified,
+            private_constructor
+        )
+        {
+            state.file_size = file_size;
+            state.name = name;
+            state.full_name = full_name;
+            state.last_modified = last_modified;
+        }
+
+
+
+
+        class file_not_found : public error { 
+            public: file_not_found(const std::string& s): error(s){}
+        };
+        
+        inline file (
+        )
+        {
+            state.file_size = 0;
+        }
+
+        file (
+            const std::string& name
+        ) { init(name); }
+
+        file (
+            const char* name
+        ) { init(name); }
+
+        inline const std::string& name (
+        ) const { return state.name; }
+
+        inline  const std::string& full_name (
+        ) const { return state.full_name; }
+
+        operator std::string (
+        ) const { return full_name(); }
+
+        inline uint64 size (
+        ) const { return state.file_size; }
+
+        inline std::chrono::time_point<std::chrono::system_clock> last_modified (
+        ) const { return state.last_modified; }
+
+        bool operator == (
+            const file& rhs
+        ) const;
+
+        bool operator != (
+            const file& rhs
+        ) const { return !(*this == rhs); }
+
+        inline bool operator < (
+            const file& item
+        ) const { return full_name() < item.full_name(); }
+
+        inline void swap (
+            file& item
+        ) 
+        { 
+            exchange(state,item.state); 
+        }
+
+    private:
+
+        data state;
+
+    };
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // directory object    
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+       
+    class directory
+    {
+        /*!
+            INITIAL VALUES
+                state.name        == name()
+                state.full_name   == full_name()
+
+            CONVENTION
+                state.name        == name()
+                state.full_name   == full_name()
+                is_root()          == state.name.size() == 0
+
+        !*/
+
+        void init (const std::string& name);
+
+    public:
+
+        struct data
+        {
+            std::string name;
+            std::string full_name;
+        };
+
+
+        /*
+            The reason we don't just make this constructor actually
+            private is because doing it this way avoids a bug that 
+            sometimes occurs in visual studio 7.1.  The bug has 
+            something to do with templated friend functions 
+            such as the get_filesystem_roots() function below if 
+            it was declared as a friend template of this class. 
+        */
+        struct private_constructor{};
+        inline directory (
+            const std::string& name,
+            const std::string& full_name,
+            private_constructor 
+        )
+        {
+            state.name = name;
+            state.full_name = full_name;
+        }
+
+
+        class dir_not_found : public error {
+            public: dir_not_found(const std::string& s):error(s){}
+        };
+        class listing_error : public error {
+            public: listing_error(const std::string& s):error(s){}
+        };
+        
+        inline directory (
+        )
+        {
+        }
+
+        directory (
+            const std::string& name
+        ) { init(name); }
+
+        directory (
+            const char* name
+        ) { init(name); }
+
+
+        static char get_separator (
+        );
+
+
+        template <
+            typename queue_of_files
+            >
+        void get_files (
+            queue_of_files& files
+        ) const;
+
+        template <
+            typename queue_of_dirs
+            >
+        void get_dirs (
+            queue_of_dirs& dirs
+        ) const;
+
+        std::vector<file> get_files (
+        ) const
+        {
+            std::vector<file> temp_vector;
+            get_files(temp_vector);
+            return temp_vector;
+        }
+
+        std::vector<directory> get_dirs (
+        ) const
+        {
+            std::vector<directory> temp_vector;
+            get_dirs(temp_vector);
+            return temp_vector;
+        }
+
+        const directory get_parent (
+        ) const;
+       
+        inline bool is_root (
+        ) const { return state.name.size() == 0; }
+
+        inline const std::string& name (
+        ) const { return state.name; }
+
+        inline const std::string& full_name (
+        ) const { return state.full_name; }
+
+        operator std::string (
+        ) const { return full_name(); }
+
+        bool operator == (
+            const directory& rhs
+        ) const;
+
+        bool operator != (
+            const directory& rhs
+        ) const { return !(*this == rhs); }
+
+        inline bool operator < (
+            const directory& item
+        ) const { return full_name() < item.full_name(); }
+
+        inline void swap (
+            directory& item
+        ) 
+        { 
+            exchange(state,item.state); 
+        }
+
+    private:
+
+        // member data
+        data state;
+
+        bool is_root_path (
+            const std::string& path
+        ) const;
+        /*!
+            ensures
+                - returns true if path is a root path.  
+                  Note that this function considers root paths that don't
+                  have a trailing separator to also be valid.
+        !*/
+
+
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    inline std::ostream& operator<< (
+        std::ostream& out,
+        const directory& item
+    ) { out << (std::string)item; return out; }
+
+    inline std::ostream& operator<< (
+        std::ostream& out,
+        const file& item
+    ) { out << (std::string)item; return out; }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_dir
+        >
+    typename disable_if<is_std_vector<queue_of_dir>,void>::type get_filesystem_roots (
+        queue_of_dir& roots
+    )
+    {
+        roots.clear();
+        const DWORD mask = GetLogicalDrives();
+        DWORD bit = 1;
+        char buf[] = "A:\\";
+
+        do
+        {
+            if (mask & bit)
+            {
+                directory dir("",buf,directory::private_constructor());
+                roots.enqueue(dir);
+            }
+            bit <<= 1;
+            ++buf[0];
+        } while (buf[0] != 'Z');
+    }
+
+    template <
+        typename queue_of_dir
+        >
+    typename enable_if<is_std_vector<queue_of_dir>,void>::type get_filesystem_roots (
+        queue_of_dir& roots
+    )
+    {
+        roots.clear();
+        const DWORD mask = GetLogicalDrives();
+        DWORD bit = 1;
+        char buf[] = "A:\\";
+
+        do
+        {
+            if (mask & bit)
+            {
+                directory dir("",buf,directory::private_constructor());
+                roots.push_back(dir);
+            }
+            bit <<= 1;
+            ++buf[0];
+        } while (buf[0] != 'Z');
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    inline void swap (
+        file& a, 
+        file& b 
+    ) { a.swap(b); }   
+
+// ----------------------------------------------------------------------------------------
+
+    inline void swap (
+        directory& a, 
+        directory& b 
+    ) { a.swap(b); }  
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // templated member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_files
+        >
+    typename disable_if<is_std_vector<queue_of_files>,void>::type 
+    directory_helper_get_files (
+        const directory::data& state,
+        queue_of_files& files
+    ) 
+    {
+        using namespace std;
+        typedef directory::listing_error listing_error;
+        typedef file::private_constructor private_constructor;
+
+        files.clear();
+        if (state.full_name.size() == 0)
+            throw listing_error("This directory object currently doesn't represent any directory.");
+
+        HANDLE ffind = INVALID_HANDLE_VALUE;
+        try
+        {
+            WIN32_FIND_DATAA data;
+            string path = state.full_name;
+            // ensure that the path ends with a separator
+            if (path[path.size()-1] != directory::get_separator())
+                path += directory::get_separator();
+            
+            ffind = FindFirstFileA((path+"*").c_str(), &data);
+            if (ffind == INVALID_HANDLE_VALUE)
+            {
+                throw listing_error("Unable to list the contents of " + state.full_name);
+            }
+
+
+            bool no_more_files = false;
+            do
+            {
+                if ((data.dwFileAttributes&FILE_ATTRIBUTE_DIRECTORY) == 0)
+                {
+                    uint64 file_size = data.nFileSizeHigh;                                   
+                    file_size <<= 32;
+                    file_size |= data.nFileSizeLow;
+
+                    ULARGE_INTEGER ull;
+                    ull.LowPart = data.ftLastWriteTime.dwLowDateTime;
+                    ull.HighPart = data.ftLastWriteTime.dwHighDateTime;
+                    std::chrono::nanoseconds epoch(100 * (ull.QuadPart - 116444736000000000));
+                    auto last_modified = std::chrono::time_point<std::chrono::system_clock>(std::chrono::duration_cast<std::chrono::system_clock::duration>(epoch));
+                    
+                    // this is a file so add it to the queue
+                    file temp(data.cFileName,path+data.cFileName,file_size, last_modified, private_constructor());
+                    files.enqueue(temp);
+                }
+
+                if (FindNextFileA(ffind,&data) == 0)
+                {
+                    // an error occurred
+                    if ( GetLastError() == ERROR_NO_MORE_FILES)
+                    {
+                        // there are no more files
+                        no_more_files = true;
+                    }
+                    else
+                    {
+                        // there was an error
+                        throw listing_error("Unable to list the contents of " + state.full_name);
+                    }  
+                }
+            } while (no_more_files == false);
+
+            FindClose(ffind); 
+            ffind = INVALID_HANDLE_VALUE;
+        }
+        catch (...)
+        {
+            if (ffind != INVALID_HANDLE_VALUE)
+                FindClose(ffind);    
+            files.clear();
+            throw;
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_files
+        >
+    typename enable_if<is_std_vector<queue_of_files>,void>::type 
+    directory_helper_get_files (
+        const directory::data& state,
+        queue_of_files& files
+    ) 
+    {
+        queue<file>::kernel_2a temp_files;
+        directory_helper_get_files(state,temp_files);
+
+        files.clear();
+
+        // copy the queue of files into the vector
+        temp_files.reset();
+        while (temp_files.move_next())
+        {
+            files.push_back(temp_files.element());
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_files
+        >
+    void directory::
+    get_files (
+        queue_of_files& files
+    ) const
+    {
+        // the reason for this indirection here is because it avoids a bug in
+        // the mingw version of gcc
+        directory_helper_get_files(state,files);
+    }
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_dirs
+        >
+    typename disable_if<is_std_vector<queue_of_dirs>,void>::type 
+    directory_helper_get_dirs (
+        const directory::data& state,
+        queue_of_dirs& dirs
+    ) 
+    {
+        using namespace std;
+        typedef directory::listing_error listing_error;
+        typedef directory::private_constructor private_constructor;
+
+        dirs.clear();
+        if (state.full_name.size() == 0)
+            throw listing_error("This directory object currently doesn't represent any directory.");
+
+        HANDLE dfind = INVALID_HANDLE_VALUE;
+        try
+        {
+            WIN32_FIND_DATAA data;
+            string path = state.full_name;
+            // ensure that the path ends with a separator
+            if (path[path.size()-1] != directory::get_separator())
+                path += directory::get_separator();
+            
+            dfind = FindFirstFileA((path+"*").c_str(), &data);
+            if (dfind == INVALID_HANDLE_VALUE)
+            {
+                throw listing_error("Unable to list the contents of " + state.full_name);
+            }
+
+
+            bool no_more_files = false;
+            do
+            {
+                string tname(data.cFileName);
+                if ((data.dwFileAttributes&FILE_ATTRIBUTE_DIRECTORY) != 0 &&
+                    tname != "." &&
+                    tname != "..")
+                {
+                    // this is a directory so add it to the queue
+                    directory temp(tname,path+tname,private_constructor());
+                    dirs.enqueue(temp);
+                }
+
+                if (FindNextFileA(dfind,&data) == 0)
+                {
+                    // an error occurred
+                    if ( GetLastError() == ERROR_NO_MORE_FILES)
+                    {
+                        // there are no more files
+                        no_more_files = true;
+                    }
+                    else
+                    {
+                        // there was an error
+                        throw listing_error("Unable to list the contents of " + state.full_name);
+                    }  
+                }
+            } while (no_more_files == false);
+
+            FindClose(dfind);  
+            dfind = INVALID_HANDLE_VALUE;
+        }
+        catch (...)
+        {         
+            if (dfind != INVALID_HANDLE_VALUE)
+                FindClose(dfind);
+            dirs.clear();
+            throw;
+        }
+
+    }
+ 
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_dirs
+        >
+    typename enable_if<is_std_vector<queue_of_dirs>,void>::type 
+    directory_helper_get_dirs (
+        const directory::data& state,
+        queue_of_dirs& dirs
+    ) 
+    {
+        queue<directory>::kernel_2a temp_dirs;
+        directory_helper_get_dirs(state,temp_dirs);
+
+        dirs.clear();
+
+        // copy the queue of dirs into the vector
+        temp_dirs.reset();
+        while (temp_dirs.move_next())
+        {
+            dirs.push_back(temp_dirs.element());
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_dirs
+        >
+    void directory::
+    get_dirs (
+        queue_of_dirs& dirs
+    ) const
+    {
+        // the reason for this indirection here is because it avoids a bug in
+        // the mingw version of gcc
+        directory_helper_get_dirs(state,dirs);
+    }
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+}
+
+
+#ifdef NO_MAKEFILE
+#include "dir_nav_kernel_1.cpp"
+#endif
+
+#endif // DLIB_DIR_NAV_KERNEl_1_
+

dlib/dir_nav/dir_nav_kernel_2.cpp

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DIR_NAV_KERNEL_2_CPp_
+#define DLIB_DIR_NAV_KERNEL_2_CPp_
+
+#include "../platform.h"
+
+#ifdef DLIB_POSIX
+
+
+#include "dir_nav_kernel_2.h"
+
+
+
+
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // file object implementation
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    void file::
+    init (
+        const std::string& name
+    )
+    {
+        using namespace std;
+
+
+
+        char buf[PATH_MAX];
+        if (realpath(name.c_str(),buf) == 0)
+        {
+            // the file was not found
+            throw file_not_found("Unable to find file " + name);
+        }
+        state.full_name = buf;
+
+
+        string::size_type pos = state.full_name.find_last_of(directory::get_separator());
+        if (pos == string::npos)
+        {
+            // no valid full path has no separtor characters.  
+            throw file_not_found("Unable to find file " + name);
+        }
+        state.name = state.full_name.substr(pos+1);
+
+
+        // now find the size of this file
+        struct stat64 buffer;
+        if (::stat64(state.full_name.c_str(), &buffer) ||
+            S_ISDIR(buffer.st_mode))
+        {
+            // there was an error during the call to stat64 or
+            // name is actually a directory
+            throw file_not_found("Unable to find file " + name);
+        }
+        else
+        {
+            state.file_size = static_cast<uint64>(buffer.st_size);
+
+
+            state.last_modified = std::chrono::system_clock::from_time_t(buffer.st_mtime);
+#ifdef _BSD_SOURCE 
+            state.last_modified += std::chrono::duration_cast<std::chrono::system_clock::duration>(std::chrono::nanoseconds(buffer.st_atim.tv_nsec));
+#endif
+        }
+
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    bool file::
+    operator == (
+        const file& rhs
+    ) const 
+    { 
+        using namespace std;
+        if (state.full_name.size() == 0 && rhs.state.full_name.size() == 0)
+            return true;
+
+        // These files might have different names but actually represent the same
+        // file due to the presence of symbolic links.
+        char buf[PATH_MAX];
+        string left, right;
+        if (realpath(state.full_name.c_str(),buf) == 0)
+            return false;    
+        left = buf;
+
+        if (realpath(rhs.state.full_name.c_str(),buf) == 0)
+            return false;    
+        right = buf;
+
+        return (left == right);
+    }
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // directory object implementation
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    void directory::
+    init (
+        const std::string& name
+    )
+    {
+        using namespace std;
+
+        
+        char buf[PATH_MAX];
+        if (realpath(name.c_str(),buf) == 0)
+        {
+            // the directory was not found
+            throw dir_not_found("Unable to find directory " + name);
+        }
+        state.full_name = buf;
+  
+        
+        const char sep = get_separator();
+        if (is_root_path(state.full_name) == false)
+        {
+            // ensure that thre is not a trialing separator
+            if (state.full_name[state.full_name.size()-1] == sep)
+                state.full_name.erase(state.full_name.size()-1);
+
+            // pick out the directory name
+            string::size_type pos = state.full_name.find_last_of(sep);
+            state.name = state.full_name.substr(pos+1);
+        }
+        else
+        {
+            // ensure that there is a trailing separator
+            if (state.full_name[state.full_name.size()-1] != sep)
+                state.full_name += sep;
+        }
+
+
+        struct stat64 buffer;
+        // now check that this is actually a valid directory
+        if (::stat64(state.full_name.c_str(),&buffer))
+        {
+            // the directory was not found
+            throw dir_not_found("Unable to find directory " + name);
+        }
+        else if (S_ISDIR(buffer.st_mode) == 0)
+        {
+            // It is not a directory
+            throw dir_not_found("Unable to find directory " + name);
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    char directory::
+    get_separator (
+    ) 
+    {
+        return '/';
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    bool directory::
+    operator == (
+        const directory& rhs
+    ) const 
+    { 
+        using namespace std;
+        if (state.full_name.size() == 0 && rhs.state.full_name.size() == 0)
+            return true;
+
+        // These directories might have different names but actually represent the same
+        // directory due to the presence of symbolic links.
+        char buf[PATH_MAX];
+        string left, right;
+        if (realpath(state.full_name.c_str(),buf) == 0)
+            return false;    
+        left = buf;
+
+        if (realpath(rhs.state.full_name.c_str(),buf) == 0)
+            return false;    
+        right = buf;
+
+        return (left == right);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    const directory directory::
+    get_parent (
+    ) const
+    {
+        using namespace std;
+        // if *this is the root then just return *this
+        if (is_root())
+        {
+            return *this;
+        }
+        else
+        {
+            directory temp;
+
+            const char sep = get_separator();
+
+            string::size_type pos = state.full_name.find_last_of(sep);
+            temp.state.full_name = state.full_name.substr(0,pos);
+
+            if ( is_root_path(temp.state.full_name))
+            {
+                temp.state.full_name += sep;
+            }
+            else
+            {
+                pos = temp.state.full_name.find_last_of(sep);
+                if (pos != string::npos)
+                {
+                    temp.state.name = temp.state.full_name.substr(pos+1);
+                }
+                else
+                {
+                    temp.state.full_name += sep;
+                }
+            }
+            return temp;
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    bool directory::
+    is_root_path (
+        const std::string& path
+    ) const
+    {
+        const char sep = get_separator();
+        if (path.size() == 1 && path[0] == sep)
+            return true;
+        else
+            return false;   
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_POSIX
+
+#endif // DLIB_DIR_NAV_KERNEL_2_CPp_
+

dlib/dir_nav/dir_nav_kernel_2.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DIR_NAV_KERNEl_2_
+#define DLIB_DIR_NAV_KERNEl_2_
+
+#ifdef DLIB_ISO_CPP_ONLY
+#error "DLIB_ISO_CPP_ONLY is defined so you can't use this OS dependent code.  Turn DLIB_ISO_CPP_ONLY off if you want to use it."
+#endif
+
+
+#include "dir_nav_kernel_abstract.h"
+
+#include <string>
+#include "../uintn.h"
+#include "../algs.h"
+
+#include <sys/types.h>
+#include <dirent.h>
+#include <libgen.h>
+#include <limits.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <chrono>
+
+#if !defined(__USE_LARGEFILE64 ) && !defined(_LARGEFILE64_SOURCE)
+#define stat64 stat
+#endif
+
+#include <vector>
+#include "../stl_checked.h"
+#include "../enable_if.h"
+#include "../queue.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // file object    
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    
+    class file
+    {
+        /*!
+            INITIAL VALUES
+                state.name        == name()
+                state.full_name   == full_name()
+                state.file_size   == size()
+                state.last_modified == last_modified()
+
+            CONVENTION
+                state.name        == name()
+                state.full_name   == full_name()
+                state.file_size   == size()
+                state.last_modified == last_modified()
+
+        !*/
+
+        friend class directory;
+
+        struct data
+        {
+            uint64 file_size;
+            std::string name;
+            std::string full_name;
+            std::chrono::time_point<std::chrono::system_clock> last_modified;
+        };
+
+        void init(const std::string& name);
+
+    public:
+
+        struct private_constructor{};
+        inline file (
+            const std::string& name,
+            const std::string& full_name,
+            const uint64 file_size,
+            const std::chrono::time_point<std::chrono::system_clock>& last_modified,
+            private_constructor
+        )
+        {
+            state.file_size = file_size;
+            state.name = name;
+            state.full_name = full_name;
+            state.last_modified = last_modified;
+        }
+
+
+        class file_not_found : public error { 
+            public: file_not_found(const std::string& s): error(s){}
+        };
+        
+        inline file (
+        )
+        {
+            state.file_size = 0;
+        }
+
+        file (
+            const std::string& name
+        ) { init(name); }
+
+        file (
+            const char* name
+        ) { init(name); }
+
+        inline const std::string& name (
+        ) const { return state.name; }
+
+        inline  const std::string& full_name (
+        ) const { return state.full_name; }
+
+        inline uint64 size (
+        ) const { return state.file_size; }
+
+        inline std::chrono::time_point<std::chrono::system_clock> last_modified (
+        ) const { return state.last_modified; }
+
+        operator std::string (
+        ) const { return full_name(); }
+
+        bool operator == (
+            const file& rhs
+        ) const;
+
+        bool operator != (
+            const file& rhs
+        ) const { return !(*this == rhs); }
+
+        inline bool operator < (
+            const file& item
+        ) const { return full_name() < item.full_name(); }
+
+        inline void swap (
+            file& item
+        ) 
+        { 
+            exchange(state,item.state); 
+        }
+
+    private:
+
+        // member data
+        data state;
+
+    };
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // directory object    
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+       
+    class directory
+    {
+        /*!
+            INITIAL VALUES
+                state.name        == name()
+                state.full_name   == full_name()
+
+            CONVENTION
+                state.name        == name()
+                state.full_name   == full_name()
+                is_root()          == state.name.size() == 0
+
+        !*/
+
+        void init(const std::string& name);
+
+    public:
+        struct private_constructor{};
+        inline directory (
+            const std::string& name,
+            const std::string& full_name,
+            private_constructor
+        )
+        {
+            state.name = name;
+            state.full_name = full_name;
+        }
+
+        struct data
+        {
+            std::string name;
+            std::string full_name;
+        };
+
+        class dir_not_found : public error {
+            public: dir_not_found(const std::string& s):error(s){}
+        };
+        class listing_error : public error {
+            public: listing_error(const std::string& s):error(s){}
+        };
+        
+        inline directory (
+        )
+        {
+        }
+
+        directory (
+            const std::string& name
+        ) { init(name); }
+
+        directory (
+            const char* name
+        ) { init(name); }
+
+        static char get_separator (
+        );
+
+        template <
+            typename queue_of_files
+            >
+        void get_files (
+            queue_of_files& files
+        ) const;
+
+        template <
+            typename queue_of_dirs
+            >
+        void get_dirs (
+            queue_of_dirs& dirs
+        ) const;
+
+        std::vector<file> get_files (
+        ) const
+        {
+            std::vector<file> temp_vector;
+            get_files(temp_vector);
+            return temp_vector;
+        }
+
+        std::vector<directory> get_dirs (
+        ) const
+        {
+            std::vector<directory> temp_vector;
+            get_dirs(temp_vector);
+            return temp_vector;
+        }
+
+        const directory get_parent (
+        ) const;
+       
+        inline bool is_root (
+        ) const { return state.name.size() == 0; }
+
+        inline const std::string& name (
+        ) const { return state.name; }
+
+        inline const std::string& full_name (
+        ) const { return state.full_name; }
+
+        operator std::string (
+        ) const { return full_name(); }
+
+        bool operator == (
+            const directory& rhs
+        ) const;
+
+        bool operator != (
+            const directory& rhs
+        ) const { return !(*this == rhs); }
+
+        inline bool operator < (
+            const directory& item
+        ) const { return full_name() < item.full_name(); }
+
+        inline void swap (
+            directory& item
+        ) 
+        { 
+            exchange(state,item.state); 
+        }
+
+    private:
+
+        // member data
+        data state;
+
+        bool is_root_path (
+            const std::string& path
+        ) const;
+        /*!
+            ensures
+                - returns true if path is a root path.  
+                  Note that this function considers root paths that don't
+                  have a trailing separator to also be valid.
+        !*/
+
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    inline std::ostream& operator<< (
+        std::ostream& out,
+        const directory& item
+    ) { out << (std::string)item; return out; }
+
+    inline std::ostream& operator<< (
+        std::ostream& out,
+        const file& item
+    ) { out << (std::string)item; return out; }
+
+// ----------------------------------------------------------------------------------------
+
+    inline void swap (
+        file& a, 
+        file& b 
+    ) { a.swap(b); }   
+
+// ----------------------------------------------------------------------------------------
+
+    inline void swap (
+        directory& a, 
+        directory& b 
+    ) { a.swap(b); }  
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // templated member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_files
+        >
+    typename disable_if<is_std_vector<queue_of_files>,void>::type
+    directory_helper_get_files (
+        const directory::data& state,
+        queue_of_files& files
+    ) 
+    {
+        using namespace std;
+
+        files.clear();
+        if (state.full_name.size() == 0)
+            throw directory::listing_error("This directory object currently doesn't represent any directory.");
+
+        DIR* ffind = 0;
+        struct dirent* data;
+        struct stat64 buffer;
+
+        try
+        {
+            string path = state.full_name;
+            // ensure that the path ends with a separator
+            if (path[path.size()-1] != directory::get_separator())
+                path += directory::get_separator();
+
+            // get a handle to something we can search with
+            ffind = opendir(state.full_name.c_str());
+            if (ffind == 0)
+            {
+                throw directory::listing_error("Unable to list the contents of " + state.full_name);
+            }
+
+            while(true)
+            {
+                errno = 0;
+                if ( (data = readdir(ffind)) == 0)
+                {                    
+                    // there was an error or no more files
+                    if ( errno == 0)
+                    {
+                        // there are no more files
+                        break;
+                    }
+                    else
+                    {
+                        // there was an error
+                        throw directory::listing_error("Unable to list the contents of " + state.full_name);
+                    }                
+                }
+
+                uint64 file_size;
+                // get a stat64 structure so we can see if this is a file
+                if (::stat64((path+data->d_name).c_str(), &buffer) != 0)
+                {
+                    // this might be a broken symbolic link.  We can check by calling
+                    // readlink and seeing if it finds anything.  
+                    char buf[PATH_MAX];
+                    ssize_t temp = readlink((path+data->d_name).c_str(),buf,sizeof(buf));
+                    if (temp == -1)                    
+                        throw directory::listing_error("Unable to list the contents of " + state.full_name);
+                    else
+                        file_size = static_cast<uint64>(temp);
+                }
+                else
+                {
+                    file_size = static_cast<uint64>(buffer.st_size);
+                }
+                auto last_modified = std::chrono::system_clock::from_time_t(buffer.st_mtime);
+#ifdef _BSD_SOURCE 
+                last_modified += std::chrono::duration_cast<std::chrono::system_clock::duration>(std::chrono::nanoseconds(buffer.st_atim.tv_nsec));
+#endif
+
+                if (S_ISDIR(buffer.st_mode) == 0)
+                {
+                    // this is actually a file
+                    file temp(
+                        data->d_name,
+                        path+data->d_name,
+                        file_size,
+                        last_modified,
+                        file::private_constructor()
+                        );
+                    files.enqueue(temp);
+                }
+            } // while (true)
+
+            if (ffind != 0)
+            {
+                while (closedir(ffind))
+                {
+                    if (errno != EINTR)
+                        break;
+                }
+                ffind = 0;
+            }
+
+        }
+        catch (...)
+        {
+            if (ffind != 0)
+            {
+                while (closedir(ffind))
+                {
+                    if (errno != EINTR)
+                        break;
+                }
+                ffind = 0;
+            }
+            files.clear();
+            throw;
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_files
+        >
+    typename enable_if<is_std_vector<queue_of_files>,void>::type 
+    directory_helper_get_files (
+        const directory::data& state,
+        queue_of_files& files
+    ) 
+    {
+        queue<file>::kernel_2a temp_files;
+        directory_helper_get_files(state,temp_files);
+
+        files.clear();
+
+        // copy the queue of files into the vector
+        temp_files.reset();
+        while (temp_files.move_next())
+        {
+            files.push_back(temp_files.element());
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_files
+        >
+    void directory::
+    get_files (
+        queue_of_files& files
+    ) const
+    {
+        // the reason for this indirection here is because it avoids a bug in
+        // the cygwin version of gcc
+        directory_helper_get_files(state,files);
+    }
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_dirs
+        >
+    typename disable_if<is_std_vector<queue_of_dirs>,void>::type 
+    directory_helper_get_dirs (
+        const directory::data& state,
+        queue_of_dirs& dirs
+    ) 
+    {
+        using namespace std;
+
+        dirs.clear();
+        if (state.full_name.size() == 0)
+            throw directory::listing_error("This directory object currently doesn't represent any directory.");
+
+        DIR* ffind = 0;
+        struct dirent* data;
+        struct stat64 buffer;
+
+        try
+        {
+            string path = state.full_name;
+            // ensure that the path ends with a separator
+            if (path[path.size()-1] != directory::get_separator())
+                path += directory::get_separator();
+
+            // get a handle to something we can search with
+            ffind = opendir(state.full_name.c_str());
+            if (ffind == 0)
+            {
+                throw directory::listing_error("Unable to list the contents of " + state.full_name);
+            }
+
+            while(true)
+            {
+                errno = 0;
+                if ( (data = readdir(ffind)) == 0)
+                {                    
+                    // there was an error or no more files
+                    if ( errno == 0)
+                    {
+                        // there are no more files
+                        break;
+                    }
+                    else
+                    {
+                        // there was an error
+                        throw directory::listing_error("Unable to list the contents of " + state.full_name);
+                    }                
+                }
+
+                // get a stat64 structure so we can see if this is a file
+                if (::stat64((path+data->d_name).c_str(), &buffer) != 0)
+                {
+                    // just assume this isn't a directory.  It is probably a broken
+                    // symbolic link.
+                    continue;
+                }
+
+                string dtemp(data->d_name);
+                if (S_ISDIR(buffer.st_mode) &&
+                    dtemp != "." &&
+                    dtemp != ".." )
+                {
+                    // this is a directory so add it to dirs
+                    directory temp(dtemp,path+dtemp, directory::private_constructor());
+                    dirs.enqueue(temp);
+                }
+            } // while (true)
+
+            if (ffind != 0)
+            {
+                while (closedir(ffind))
+                {
+                    if (errno != EINTR)
+                        break;
+                }
+                ffind = 0;
+            }
+
+        }
+        catch (...)
+        {
+            if (ffind != 0)
+            {
+                while (closedir(ffind))
+                {
+                    if (errno != EINTR)
+                        break;
+                }
+                ffind = 0;
+            }
+            dirs.clear();
+            throw;
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_dirs
+        >
+    typename enable_if<is_std_vector<queue_of_dirs>,void>::type 
+    directory_helper_get_dirs (
+        const directory::data& state,
+        queue_of_dirs& dirs
+    ) 
+    {
+        queue<directory>::kernel_2a temp_dirs;
+        directory_helper_get_dirs(state,temp_dirs);
+
+        dirs.clear();
+
+        // copy the queue of dirs into the vector
+        temp_dirs.reset();
+        while (temp_dirs.move_next())
+        {
+            dirs.push_back(temp_dirs.element());
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+
+    template <
+        typename queue_of_dirs
+        >
+    void directory::
+    get_dirs (
+        queue_of_dirs& dirs
+    ) const
+    {
+        // the reason for this indirection here is because it avoids a bug in
+        // the cygwin version of gcc
+        directory_helper_get_dirs(state,dirs);
+    }
+ 
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_dir
+        >
+    typename disable_if<is_std_vector<queue_of_dir>,void>::type get_filesystem_roots (
+        queue_of_dir& roots
+    )
+    {
+        roots.clear();
+        directory dir("/");
+        roots.enqueue(dir);
+    }
+
+    template <
+        typename queue_of_dir
+        >
+    typename enable_if<is_std_vector<queue_of_dir>,void>::type get_filesystem_roots (
+        std::vector<directory>& roots
+    )
+    {
+        roots.clear();
+        directory dir("/");
+        roots.push_back(dir);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+
+#ifdef NO_MAKEFILE
+#include "dir_nav_kernel_2.cpp"
+#endif
+
+#endif // DLIB_DIR_NAV_KERNEl_2_
+

dlib/dir_nav/dir_nav_kernel_abstract.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_DIR_NAV_KERNEl_ABSTRACT_
+#ifdef DLIB_DIR_NAV_KERNEl_ABSTRACT_
+
+#include <string>
+#include <vector>
+#include "../uintn.h"
+#include "../algs.h"
+#include <chrono>
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    /*!
+        GENERAL WARNING
+            Don't call any of these functions or make any of these objects 
+            before main() has been entered.   That means no instances
+            of file or directory at the global scope.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename queue_of_dir
+        >
+    void get_filesystem_roots (
+        queue_of_dir& roots
+    );
+    /*!
+        requires
+            - queue_of_dirs == an implementation of queue/queue_kernel_abstract.h with T 
+              set to directory or a std::vector<directory> or dlib::std_vector_c<directory>.
+        ensures
+            - #roots == a queue containing directories that represent all the roots 
+              of the filesystem on this machine.   (e.g. in windows you have c:\, d:\ 
+              etc.)
+        throws
+            - std::bad_alloc
+    !*/
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // file object    
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    
+    class file
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This object represents a file.
+
+                Note that the size of a file is determined at the time the file
+                object is constructed.  Thus if a file changes sizes after its
+                file object has been created its file object's size() method
+                will not reflect the new file size.    
+        !*/
+
+    public:
+
+        class file_not_found : public error {};
+        
+        file (
+        );
+        /*!
+            ensures
+                - #*this has been properly initialized
+                - #name() == ""
+                - #full_name() == ""
+                - #size() == 0
+                - #*this does not represent any file
+            throws  
+                - std::bad_alloc
+        !*/
+
+        file (
+            const std::string& name
+        );
+        /*!
+            ensures
+                - #*this has been properly initialized 
+                - #*this represents the file given by name
+                  Note that name can be a fully qualified path or just a path
+                  relative to the current working directory.  Also, any symbolic 
+                  links in name will be resolved.
+            throws  
+                - std::bad_alloc
+                - file_not_found
+                    This exception is thrown if the file can not be found or
+                    accessed.                    
+        !*/
+
+        file (
+            const char* name
+        );
+        /*!
+            ensures
+                - this function is identical to file(const std::string& name)
+        !*/
+
+        file (
+            const file& item
+        );
+        /*!
+            ensures
+                - #*this == item
+            throws  
+                - std::bad_alloc
+        !*/
+
+        ~file (
+        );
+        /*!
+            ensures
+                - all resources associated with *this have been released
+        !*/
+
+        const std::string& name (
+        ) const;
+        /*!
+            ensures
+                - returns the name of the file.  This is full_name() minus 
+                  the path to the file.            
+        !*/
+
+        const std::string& full_name (
+        ) const;
+        /*!
+            ensures
+                - returns the fully qualified name for the file represented by *this 
+        !*/
+
+        uint64 size (
+        ) const;
+        /*!
+            ensures
+                - returns the size of this file in bytes.
+        !*/
+
+        std::chrono::time_point<std::chrono::system_clock> last_modified (
+        ) const;
+        /*!
+            ensures
+                - returns the time the file was last modified.
+        !*/
+
+        operator std::string (
+        ) const; 
+        /*!
+            ensures
+                - returns full_name()
+                  (i.e. provides an implicit conversion to string from dlib::file)
+        !*/
+
+        file& operator= (
+            const file& rhs
+        );
+        /*!
+            ensures
+                - #*this == rhs
+        !*/
+
+        bool operator == (
+            const file& rhs
+        ) const;
+        /*!
+            ensures
+                - if (*this and rhs represent the same file) then
+                    - returns true
+                - else
+                    - returns false
+        !*/
+
+        bool operator != (
+            const file& rhs
+        ) const;
+        /*!
+            ensures
+                - if (*this and rhs represent the same file) then
+                    - returns false 
+                - else
+                    - returns true 
+        !*/
+
+        bool operator < (
+            const file& item
+        ) const;
+        /*!
+            ensures
+                - if (full_name() < item.full_name()) then
+                    - returns true
+                - else
+                    - returns false
+        !*/
+
+        void swap (
+            file& item
+        );
+        /*!
+            ensures
+                - swaps *this and item
+        !*/ 
+
+    };
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // directory object    
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    
+    class directory
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This object represents a directory in a file system.  It gives
+                the ability to traverse a directory tree.  
+
+                Note that the directories . and .. are not returned by get_dirs() 
+        !*/
+
+    public:
+
+        class dir_not_found : public error {};
+        class listing_error : public error {};
+        
+        directory (
+        );
+        /*!
+            ensures
+                - #*this has been properly initialized
+                - #full_name() == ""
+                - #name() == ""
+                - #is_root() == true
+                - #*this does not represent any directory
+            throws  
+                - std::bad_alloc
+        !*/
+
+        directory (
+            const std::string& name
+        );
+        /*!
+            ensures
+                - #*this has been properly initialized 
+                - #*this represents the directory given by name.
+                  Note that name can be a fully qualified path or just a path
+                  relative to the current working directory. Also, any symbolic 
+                  links in name will be resolved.
+            throws  
+                - std::bad_alloc
+                - dir_not_found
+                    This exception is thrown if the directory can not be found or
+                    accessed.    
+        !*/
+
+        directory (
+            const char* name
+        );
+        /*!
+            ensures
+                - this function is identical to directory(const std::string& name)
+        !*/
+
+        directory (
+            const directory& item
+        );
+        /*!
+            ensures
+                - #*this == item
+            throws  
+                - std::bad_alloc
+        !*/
+
+        ~directory (
+        );
+        /*!
+            ensures
+                - all resources associated with *this have been released
+        !*/
+
+        static char get_separator (
+        );
+        /*!
+            ensures
+                - returns the character used to separate directories and file names in a
+                  path.  (i.e.  \ on windows and / in unix)
+        !*/
+
+        template <
+            typename queue_of_files
+            >
+        void get_files (
+            queue_of_files& files
+        ) const;
+        /*!
+            requires
+                - queue_of_files == an implementation of queue/queue_kernel_abstract.h with T 
+                  set to file or a std::vector<file> or dlib::std_vector_c<file>.
+            ensures
+                - #files == A queue containing all the files present in this directory.
+                  (Note that symbolic links will not have been resolved in the names 
+                  of the returned files.)
+                - #files.size() == the number of files in this directory
+            throws 
+                - bad_alloc
+                    If this exception is thrown then the call to get_files() has
+                    no effect on *this and #files is unusable until files.clear()
+                    is called and succeeds.
+                - listing_error
+                    This exception is thrown if listing access has been denied to this
+                    directory or if some error occurred that prevented us from successfully
+                    getting the contents of this directory.       
+                    If this exception is thrown then the call to get_files() has
+                    no effect on *this and #files.size()==0.         
+        !*/
+
+        std::vector<file> get_files (
+        ) const;
+        /*!
+            ensures
+                - This function simply calls get_files(temp_vector) and then returns temp_vector.
+        !*/
+
+        template <
+            typename queue_of_dirs
+            >
+        void get_dirs (
+            queue_of_dirs& dirs
+        ) const;
+        /*!
+            requires
+                - queue_of_dirs == an implementation of queue/queue_kernel_abstract.h with T 
+                  set to directory or a std::vector<directory> or dlib::std_vector_c<directory>.
+            ensures
+                - #dirs == a queue containing all the directories present in this directory.
+                  (note that symbolic links will not have been resolved in the names 
+                  of the returned directories.)
+                - #dirs.size() == the number of subdirectories in this directory
+            throws 
+                - bad_alloc
+                    If this exception is thrown then the call to get_files() has
+                    no effect on *this and #files is unusable until files.clear()
+                    is called and succeeds.
+                - listing_error
+                    This exception is thrown if listing access has been denied to this
+                    directory or if some error occurred that prevented us from successfully
+                    getting the contents of this directory.
+                    If this exception is thrown then the call to get_dirs() has
+                    no effect on *this and #dirs.size()==0.
+        !*/
+
+        std::vector<directory> get_dirs (
+        ) const;
+        /*!
+            ensures
+                - This function simply calls get_dirs(temp_vector) and then returns temp_vector.
+        !*/
+
+        bool is_root (
+        ) const;
+        /*!
+            ensures
+                - if (*this represents the root of this directory tree) then
+                    - returns true
+                - else
+                    - returns false
+        !*/
+
+        const directory get_parent (
+        ) const;
+        /*!
+            ensures
+                - if (is_root()) then
+                    - returns a copy of *this                    
+                - else
+                    - returns the parent directory of *this                    
+            throws
+                - bad_alloc
+                    If this exception is thrown then the call to get_parent() will
+                    have no effect.
+        !*/
+
+        const std::string& name (
+        ) const;
+        /*!
+            ensures
+                - if (is_root()) then
+                    - returns ""
+                - else
+                    - returns the name of the directory.  This is full_name() minus 
+                      the path to the directory.           
+        !*/
+
+        const std::string& full_name (
+        ) const;
+        /*!
+            ensures
+                - returns the fully qualified directory name for *this 
+                - if (is_root()) then
+                    - the last character of #full_name() is get_separator()
+                - else
+                    - the last character of #full_name() is NOT get_separator()
+        !*/
+
+        operator std::string (
+        ) const; 
+        /*!
+            ensures
+                - returns full_name()
+                  (i.e. provides an implicit conversion to string from dlib::directory)
+        !*/
+
+        directory& operator= (
+            const directory& rhs
+        );
+        /*!
+            ensures
+                - #*this == rhs
+        !*/
+
+        bool operator == (
+            const directory& rhs
+        ) const;
+        /*!
+            ensures
+                - if (*this and rhs represent the same directory) then
+                    - returns true
+                - else
+                    - returns false
+        !*/
+
+        bool operator != (
+            const directory& rhs
+        ) const;
+        /*!
+            ensures
+                - if (*this and rhs represent the same directory) then
+                    - returns false 
+                - else
+                    - returns true 
+        !*/
+
+        bool operator < (
+            const directory& item
+        ) const;
+        /*!
+            ensures
+                - if (full_name() < item.full_name()) then
+                    - returns true
+                - else
+                    - returns false
+        !*/
+
+        void swap (
+            directory& item
+        );
+        /*!
+            ensures
+                - swaps *this and item
+        !*/ 
+
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    inline std::ostream& operator<< (
+        std::ostream& out,
+        const directory& item
+    );
+    /*!
+        ensures
+            - performs: out << item.full_name()
+            - returns out
+    !*/
+
+    inline std::ostream& operator<< (
+        std::ostream& out,
+        const file& item
+    );
+    /*!
+        ensures
+            - performs: out << item.full_name()
+            - returns out
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    inline void swap (
+        file& a, 
+        file& b 
+    ) { a.swap(b); }   
+    /*!
+        provides a global swap function for file objects
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    inline void swap (
+        directory& a, 
+        directory& b 
+    ) { a.swap(b); }   
+    /*!
+        provides a global swap function for directory objects
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_DIR_NAV_KERNEl_ABSTRACT_
+