dlib from github, version=19.24

dlib/cmd_line_parser/cmd_line_parser_print_1.h

+// Copyright (C) 2005  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_CMD_LINE_PARSER_PRINt_1_
+#define DLIB_CMD_LINE_PARSER_PRINt_1_
+
+#include "cmd_line_parser_kernel_abstract.h"
+#include "../algs.h"
+#include "../string.h"
+#include <iostream>
+#include <string>
+#include <sstream>
+#include <map>
+#include <memory>
+
+namespace dlib
+{
+
+    template <
+        typename clp_base 
+        >
+    class cmd_line_parser_print_1 : public clp_base
+    {
+
+        public:
+
+            void print_options (
+                std::basic_ostream<typename clp_base::char_type>& out
+            ) const;
+
+            void print_options (
+            ) const
+            {
+                print_options(std::cout);
+            }
+
+    };
+
+    template <
+        typename clp_base
+        >
+    inline void swap (
+        cmd_line_parser_print_1<clp_base>& a, 
+        cmd_line_parser_print_1<clp_base>& b 
+    ) { a.swap(b); }   
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename clp_base
+        >
+    void cmd_line_parser_print_1<clp_base>::
+    print_options (
+        std::basic_ostream<typename clp_base::char_type>& out
+    ) const
+    {
+        typedef typename clp_base::char_type ct;
+        typedef std::basic_string<ct> string;
+        typedef typename string::size_type size_type;
+
+        typedef std::basic_ostringstream<ct> ostringstream;
+
+        try
+        {
+
+
+            size_type max_len = 0; 
+            this->reset();
+
+            // this loop here is just the bottom loop but without the print statements.
+            // I'm doing this to figure out what len should be.
+            while (this->move_next())
+            {
+                size_type len = 0; 
+                len += 3;
+                if (this->element().name().size() > 1)
+                {
+                    ++len;
+                }
+                len += this->element().name().size();
+
+                if (this->element().number_of_arguments() == 1)
+                {
+                    len += 6;
+                }
+                else
+                {
+                    for (unsigned long i = 0; i < this->element().number_of_arguments(); ++i)
+                    {
+                        len += 7;
+                        if (i+1 > 9)
+                            ++len;
+                    }
+                }
+
+                len += 3;
+                if (len < 33)
+                    max_len = std::max(max_len,len);
+            }
+
+
+            // Make a separate ostringstream for each option group.  We are going to write
+            // the output for each group to a separate ostringstream so that we can keep
+            // them grouped together in the final output.
+            std::map<string,std::shared_ptr<ostringstream> > groups;
+            this->reset();
+            while(this->move_next())
+            {
+                if (!groups[this->element().group_name()])
+                    groups[this->element().group_name()].reset(new ostringstream);
+            }
+
+
+
+
+            this->reset();
+
+            while (this->move_next())
+            {
+                ostringstream& sout = *groups[this->element().group_name()];
+
+                size_type len = 0; 
+                sout << _dT(ct,"\n  -");
+                len += 3;
+                if (this->element().name().size() > 1)
+                {
+                    sout << _dT(ct,"-");
+                    ++len;
+                }
+                sout << this->element().name();
+                len += this->element().name().size();
+
+                if (this->element().number_of_arguments() == 1)
+                {
+                    sout << _dT(ct," <arg>");
+                    len += 6;
+                }
+                else
+                {
+                    for (unsigned long i = 0; i < this->element().number_of_arguments(); ++i)
+                    {
+                        sout << _dT(ct," <arg") << i+1 << _dT(ct,">");
+                        len += 7;
+                        if (i+1 > 9)
+                            ++len;
+                    }
+                }
+
+                sout << _dT(ct,"   ");
+                len += 3;
+
+                while (len < max_len)
+                {
+                    ++len;
+                    sout << _dT(ct," ");
+                }
+
+                const unsigned long ml = static_cast<unsigned long>(max_len);
+                // now print the description but make it wrap around nicely if it 
+                // is to long to fit on one line.
+                if (len <= max_len)
+                    sout << wrap_string(this->element().description(),0,ml);
+                else
+                    sout << _dT(ct,"\n") << wrap_string(this->element().description(),ml,ml);
+            }
+
+            // Only print out a generic Options: group name if there is an unnamed option
+            // present.
+            if (groups.count(string()) == 1)
+                out << _dT(ct,"Options:");
+
+            // Now print everything out
+            typename std::map<string,std::shared_ptr<ostringstream> >::iterator i;
+            for (i = groups.begin(); i != groups.end(); ++i)
+            {
+                // print the group name if we have one
+                if (i->first.size() != 0)
+                {
+                    if (i != groups.begin())
+                        out << _dT(ct,"\n\n");
+                    out << i->first << _dT(ct,":");
+                }
+
+                // print the options in the group
+                out << i->second->str();
+            }
+            out << _dT(ct,"\n\n");
+            this->reset();
+        }
+        catch (...)
+        {
+            this->reset();
+            throw;
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_CMD_LINE_PARSER_PRINt_1_
+

dlib/cmd_line_parser/get_option.h

+// Copyright (C) 2012  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_GET_OPTiON_Hh_
+#define DLIB_GET_OPTiON_Hh_
+
+#include "get_option_abstract.h"
+#include "../string.h"
+#include "../is_kind.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    class option_parse_error : public error
+    {
+    public:
+        option_parse_error(const std::string& option_string, const std::string& str):
+            error(EOPTION_PARSE,"Error parsing argument for option '" + option_string + "', offending string is '" + str + "'.") {}
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename config_reader_type, typename T>
+    T impl_config_reader_get_option (
+        const config_reader_type& cr,
+        const std::string& option_name,
+        const std::string& full_option_name,
+        T default_value
+    )
+    {
+        std::string::size_type pos = option_name.find_first_of(".");
+        if (pos == std::string::npos)
+        {
+            if (cr.is_key_defined(option_name))
+            {
+                try{ return string_cast<T>(cr[option_name]); }
+                catch (string_cast_error&) { throw option_parse_error(full_option_name, cr[option_name]); }
+            }
+        }
+        else
+        {
+            std::string block_name = option_name.substr(0,pos);
+            if (cr.is_block_defined(block_name))
+            {
+                return impl_config_reader_get_option(cr.block(block_name), 
+                                                     option_name.substr(pos+1),
+                                                     full_option_name,
+                                                     default_value);
+            }
+        }
+
+        return default_value;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename cr_type, typename T>
+    typename enable_if<is_config_reader<cr_type>,T>::type get_option (
+        const cr_type& cr,
+        const std::string& option_name,
+        T default_value
+    )
+    {
+        return impl_config_reader_get_option(cr, option_name, option_name, default_value);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename parser_type, typename T>
+    typename disable_if<is_config_reader<parser_type>,T>::type get_option (
+        const parser_type& parser,
+        const std::string& option_name,
+        T default_value
+    )
+    {
+        // make sure requires clause is not broken
+        DLIB_ASSERT( parser.option_is_defined(option_name) == true &&
+                     parser.option(option_name).number_of_arguments() == 1,
+            "\t T get_option()"
+            << "\n\t option_name: " << option_name
+            << "\n\t parser.option_is_defined(option_name):            " << parser.option_is_defined(option_name)
+            << "\n\t parser.option(option_name).number_of_arguments(): " << parser.option(option_name).number_of_arguments()
+            );
+
+        if (parser.option(option_name))
+        {
+            try
+            {
+                default_value = string_cast<T>(parser.option(option_name).argument()); 
+            }
+            catch (string_cast_error&) 
+            { 
+                throw option_parse_error(option_name, parser.option(option_name).argument()); 
+            }
+        }
+        return default_value;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename parser_type, typename cr_type, typename T>
+    typename disable_if<is_config_reader<parser_type>,T>::type get_option (
+        const parser_type& parser,
+        const cr_type& cr,
+        const std::string& option_name,
+        T default_value
+    )
+    {
+        // make sure requires clause is not broken
+        DLIB_ASSERT( parser.option_is_defined(option_name) == true &&
+                     parser.option(option_name).number_of_arguments() == 1,
+            "\t T get_option()"
+            << "\n\t option_name: " << option_name
+            << "\n\t parser.option_is_defined(option_name):            " << parser.option_is_defined(option_name)
+            << "\n\t parser.option(option_name).number_of_arguments(): " << parser.option(option_name).number_of_arguments()
+            );
+
+        if (parser.option(option_name))
+            return get_option(parser, option_name, default_value); 
+        else
+            return get_option(cr, option_name, default_value);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename parser_type, typename cr_type, typename T>
+    typename disable_if<is_config_reader<parser_type>,T>::type get_option (
+        const cr_type& cr,
+        const parser_type& parser,
+        const std::string& option_name,
+        T default_value
+    )
+    {
+        // make sure requires clause is not broken
+        DLIB_ASSERT( parser.option_is_defined(option_name) == true &&
+                     parser.option(option_name).number_of_arguments() == 1,
+            "\t T get_option()"
+            << "\n\t option_name: " << option_name
+            << "\n\t parser.option_is_defined(option_name):            " << parser.option_is_defined(option_name)
+            << "\n\t parser.option(option_name).number_of_arguments(): " << parser.option(option_name).number_of_arguments()
+            );
+
+        if (parser.option(option_name))
+            return get_option(parser, option_name, default_value); 
+        else
+            return get_option(cr, option_name, default_value);
+    }
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <typename T>
+    inline std::string get_option (
+        const T& cr,
+        const std::string& option_name,
+        const char* default_value
+    )
+    {
+        return get_option(cr, option_name, std::string(default_value));
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename T, typename U>
+    inline std::string get_option (
+        const T& parser,
+        const U& cr,
+        const std::string& option_name,
+        const char* default_value
+    )
+    {
+        return get_option(parser, cr, option_name, std::string(default_value));
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_GET_OPTiON_Hh_
+

dlib/cmd_line_parser/get_option_abstract.h

+// Copyright (C) 2012  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_GET_OPTiON_ABSTRACT_Hh_
+#ifdef DLIB_GET_OPTiON_ABSTRACT_Hh_
+
+#inclue <string>
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    class option_parse_error : public error 
+    { 
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This is the exception thrown by the get_option() functions.  It is
+                thrown when the option string given by a command line parser or
+                config reader can't be converted into the type T.
+        !*/
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename config_reader_type, 
+        typename T
+        >
+    T get_option (
+        const config_reader_type& cr,
+        const std::string& option_name,
+        T default_value
+    );
+    /*!
+        requires
+            - T is a type which can be read from an input stream
+            - config_reader_type == an implementation of config_reader/config_reader_kernel_abstract.h
+        ensures
+            - option_name is used to index into the given config_reader.  
+            - if (cr contains an entry corresponding to option_name) then
+                - converts the string value in cr corresponding to option_name into
+                  an object of type T and returns it.
+            - else
+                - returns default_value
+            - The scheme for indexing into cr based on option_name is best
+              understood by looking at a few examples:
+                - an option name of "name" corresponds to cr["name"]
+                - an option name of "block1.name" corresponds to cr.block("block1")["name"]
+                - an option name of "block1.block2.name" corresponds to cr.block("block1").block("block2")["name"]
+        throws
+            - option_parse_error
+              This exception is thrown if we attempt but fail to convert the string value
+              in cr into an object of type T.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename command_line_parser_type,
+        typename T
+        >
+    T get_option (
+        const command_line_parser_type& parser,
+        const std::string& option_name,
+        T default_value
+    );
+    /*!
+        requires
+            - parser.option_is_defined(option_name) == true
+            - parser.option(option_name).number_of_arguments() == 1   
+            - T is a type which can be read from an input stream
+            - command_line_parser_type == an implementation of cmd_line_parser/cmd_line_parser_kernel_abstract.h
+        ensures
+            - if (parser.option(option_name)) then 
+                - converts parser.option(option_name).argument() into an object
+                  of type T and returns it.  That is, the string argument to this
+                  command line option is converted into a T and returned.
+            - else
+                - returns default_value
+        throws
+            - option_parse_error
+              This exception is thrown if we attempt but fail to convert the string
+              argument into an object of type T.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename command_line_parser_type, 
+        typename config_reader_type, 
+        typename T
+        >
+    T get_option (
+        const command_line_parser_type& parser,
+        const config_reader_type& cr,
+        const std::string& option_name,
+        T default_value
+    );
+    /*!
+        requires
+            - parser.option_is_defined(option_name) == true
+            - parser.option(option_name).number_of_arguments() == 1   
+            - T is a type which can be read from an input stream
+            - command_line_parser_type == an implementation of cmd_line_parser/cmd_line_parser_kernel_abstract.h
+            - config_reader_type == an implementation of config_reader/config_reader_kernel_abstract.h
+        ensures
+            - if (parser.option(option_name)) then 
+                - returns get_option(parser, option_name, default_value)
+            - else
+                - returns get_option(cr, option_name, default_value)
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename command_line_parser_type, 
+        typename config_reader_type, 
+        typename T
+        >
+    T get_option (
+        const config_reader_type& cr,
+        const command_line_parser_type& parser,
+        const std::string& option_name,
+        T default_value
+    );
+    /*!
+        requires
+            - parser.option_is_defined(option_name) == true
+            - parser.option(option_name).number_of_arguments() == 1   
+            - T is a type which can be read from an input stream
+            - command_line_parser_type == an implementation of cmd_line_parser/cmd_line_parser_kernel_abstract.h
+            - config_reader_type == an implementation of config_reader/config_reader_kernel_abstract.h
+        ensures
+            - if (parser.option(option_name)) then 
+                - returns get_option(parser, option_name, default_value)
+            - else
+                - returns get_option(cr, option_name, default_value)
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_GET_OPTiON_ABSTRACT_Hh_
+
+

dlib/compress_stream.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_COMPRESS_STREAm_
+#define DLIB_COMPRESS_STREAm_
+
+#include "compress_stream/compress_stream_kernel_1.h"
+#include "compress_stream/compress_stream_kernel_2.h"
+#include "compress_stream/compress_stream_kernel_3.h"
+
+#include "conditioning_class.h"
+#include "entropy_encoder.h"
+#include "entropy_decoder.h"
+
+#include "entropy_encoder_model.h"
+#include "entropy_decoder_model.h"
+#include "lz77_buffer.h"
+#include "sliding_buffer.h"
+#include "lzp_buffer.h"
+#include "crc32.h"
+
+
+namespace dlib
+{
+
+    class compress_stream
+    {
+        compress_stream() {}
+
+        typedef entropy_encoder_model<257,entropy_encoder::kernel_2a>::kernel_1b fce1;
+        typedef entropy_decoder_model<257,entropy_decoder::kernel_2a>::kernel_1b fcd1;
+
+        typedef entropy_encoder_model<257,entropy_encoder::kernel_2a>::kernel_2b fce2;
+        typedef entropy_decoder_model<257,entropy_decoder::kernel_2a>::kernel_2b fcd2;
+
+        typedef entropy_encoder_model<257,entropy_encoder::kernel_2a>::kernel_3b fce3;
+        typedef entropy_decoder_model<257,entropy_decoder::kernel_2a>::kernel_3b fcd3;
+
+        typedef entropy_encoder_model<257,entropy_encoder::kernel_2a>::kernel_4a fce4a;
+        typedef entropy_decoder_model<257,entropy_decoder::kernel_2a>::kernel_4a fcd4a;
+        typedef entropy_encoder_model<257,entropy_encoder::kernel_2a>::kernel_4b fce4b;
+        typedef entropy_decoder_model<257,entropy_decoder::kernel_2a>::kernel_4b fcd4b;
+
+        typedef entropy_encoder_model<257,entropy_encoder::kernel_2a>::kernel_5a fce5a;
+        typedef entropy_decoder_model<257,entropy_decoder::kernel_2a>::kernel_5a fcd5a;
+        typedef entropy_encoder_model<257,entropy_encoder::kernel_2a>::kernel_5b fce5b;
+        typedef entropy_decoder_model<257,entropy_decoder::kernel_2a>::kernel_5b fcd5b;
+        typedef entropy_encoder_model<257,entropy_encoder::kernel_2a>::kernel_5c fce5c;
+        typedef entropy_decoder_model<257,entropy_decoder::kernel_2a>::kernel_5c fcd5c;
+
+        typedef entropy_encoder_model<257,entropy_encoder::kernel_2a>::kernel_6a fce6;
+        typedef entropy_decoder_model<257,entropy_decoder::kernel_2a>::kernel_6a fcd6;
+
+
+        typedef entropy_encoder_model<257,entropy_encoder::kernel_2a>::kernel_2d fce2d;
+        typedef entropy_decoder_model<257,entropy_decoder::kernel_2a>::kernel_2d fcd2d;
+
+        typedef sliding_buffer<unsigned char>::kernel_1a sliding_buffer1;
+        typedef lz77_buffer::kernel_2a lz77_buffer2a;
+
+
+        typedef lzp_buffer::kernel_1a lzp_buf_1;
+        typedef lzp_buffer::kernel_2a lzp_buf_2;
+
+
+        typedef entropy_encoder_model<513,entropy_encoder::kernel_2a>::kernel_1b fce_length;
+        typedef entropy_decoder_model<513,entropy_decoder::kernel_2a>::kernel_1b fcd_length;
+
+        typedef entropy_encoder_model<65534,entropy_encoder::kernel_2a>::kernel_1b fce_length_2;
+        typedef entropy_decoder_model<65534,entropy_decoder::kernel_2a>::kernel_1b fcd_length_2;
+
+
+        typedef entropy_encoder_model<32257,entropy_encoder::kernel_2a>::kernel_1b fce_index;
+        typedef entropy_decoder_model<32257,entropy_decoder::kernel_2a>::kernel_1b fcd_index;
+
+    public:
+        
+        //----------- kernels ---------------
+
+        // kernel_1a        
+        typedef      compress_stream_kernel_1 <fce1,fcd1,crc32::kernel_1a>
+                     kernel_1a;
+   
+        // kernel_1b        
+        typedef      compress_stream_kernel_1 <fce2,fcd2,crc32::kernel_1a>
+                     kernel_1b;
+
+        // kernel_1c        
+        typedef      compress_stream_kernel_1 <fce3,fcd3,crc32::kernel_1a>
+                     kernel_1c;
+
+        // kernel_1da        
+        typedef      compress_stream_kernel_1 <fce4a,fcd4a,crc32::kernel_1a>
+                     kernel_1da;
+
+        // kernel_1ea        
+        typedef      compress_stream_kernel_1 <fce5a,fcd5a,crc32::kernel_1a>
+                     kernel_1ea;
+
+        // kernel_1db        
+        typedef      compress_stream_kernel_1 <fce4b,fcd4b,crc32::kernel_1a>
+                     kernel_1db;
+
+        // kernel_1eb        
+        typedef      compress_stream_kernel_1 <fce5b,fcd5b,crc32::kernel_1a>
+                     kernel_1eb;
+
+        // kernel_1ec        
+        typedef      compress_stream_kernel_1 <fce5c,fcd5c,crc32::kernel_1a>
+                     kernel_1ec;
+
+
+
+
+        // kernel_2a        
+        typedef      compress_stream_kernel_2 <fce2,fcd2,lz77_buffer2a,sliding_buffer1,fce_length,fcd_length,fce_index,fcd_index,crc32::kernel_1a>
+                     kernel_2a;
+
+
+
+
+        // kernel_3a        
+        typedef      compress_stream_kernel_3 <lzp_buf_1,crc32::kernel_1a,16>
+                     kernel_3a;
+        // kernel_3b        
+        typedef      compress_stream_kernel_3 <lzp_buf_2,crc32::kernel_1a,16>
+                     kernel_3b;
+   
+
+    };
+}
+
+#endif // DLIB_COMPRESS_STREAm_
+

dlib/compress_stream/compress_stream_kernel_1.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_COMPRESS_STREAM_KERNEl_1_
+#define DLIB_COMPRESS_STREAM_KERNEl_1_
+
+#include "../algs.h"
+#include <iostream>
+#include <streambuf>
+#include <cstdio>
+#include "compress_stream_kernel_abstract.h"
+
+namespace dlib
+{
+
+    template <
+        typename fce,
+        typename fcd,
+        typename crc32
+        >
+    class compress_stream_kernel_1
+    {
+        /*!
+            REQUIREMENTS ON fce
+                is an implementation of entropy_encoder_model/entropy_encoder_model_kernel_abstract.h
+                the alphabet_size of fce must be 257.
+                fce and fcd share the same kernel number.
+
+            REQUIREMENTS ON fcd
+                is an implementation of entropy_decoder_model/entropy_decoder_model_kernel_abstract.h
+                the alphabet_size of fcd must be 257.
+                fce and fcd share the same kernel number.
+
+            REQUIREMENTS ON crc32
+                is an implementation of crc32/crc32_kernel_abstract.h
+
+
+
+            INITIAL VALUE
+                this object has no state
+
+            CONVENTION
+                this object has no state
+        !*/
+
+        const static unsigned long eof_symbol = 256;
+
+    public:
+
+        class decompression_error : public dlib::error 
+        { 
+            public: 
+                decompression_error(
+                    const char* i
+                ) :
+                    dlib::error(std::string(i))
+                {}
+
+                decompression_error(
+                    const std::string& i
+                ) :
+                    dlib::error(i)
+                {}
+        };
+
+
+        compress_stream_kernel_1 (
+        )
+        {}
+
+        ~compress_stream_kernel_1 (
+        )
+        {}
+
+        void compress (
+            std::istream& in,
+            std::ostream& out
+        ) const;
+
+        void decompress (
+            std::istream& in,
+            std::ostream& out
+        ) const;
+
+    private:
+
+        // restricted functions
+        compress_stream_kernel_1(compress_stream_kernel_1&);        // copy constructor
+        compress_stream_kernel_1& operator=(compress_stream_kernel_1&);    // assignment operator
+
+    };
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename fce,
+        typename fcd,
+        typename crc32
+        >
+    void compress_stream_kernel_1<fce,fcd,crc32>::
+    compress (
+        std::istream& in_,
+        std::ostream& out_
+    ) const
+    {
+        std::streambuf::int_type temp;
+
+        std::streambuf& in = *in_.rdbuf();
+
+        typename fce::entropy_encoder_type coder;
+        coder.set_stream(out_);
+
+        fce model(coder);
+
+        crc32 crc;
+
+        unsigned long count = 0;
+
+        while (true)
+        {
+            // write out a known value every 20000 symbols
+            if (count == 20000)
+            {
+                count = 0;
+                coder.encode(1500,1501,8000);
+            }
+            ++count;
+
+            // get the next character
+            temp = in.sbumpc();
+
+            // if we have hit EOF then encode the marker symbol
+            if (temp != EOF)  
+            {
+                // encode the symbol
+                model.encode(static_cast<unsigned long>(temp));
+                crc.add(static_cast<unsigned char>(temp));
+                continue;
+            }
+            else
+            {
+                model.encode(eof_symbol);
+
+                // now write the checksum
+                unsigned long checksum = crc.get_checksum();
+                unsigned char byte1 = static_cast<unsigned char>((checksum>>24)&0xFF);
+                unsigned char byte2 = static_cast<unsigned char>((checksum>>16)&0xFF);
+                unsigned char byte3 = static_cast<unsigned char>((checksum>>8)&0xFF);
+                unsigned char byte4 = static_cast<unsigned char>((checksum)&0xFF);
+
+                model.encode(byte1);
+                model.encode(byte2);
+                model.encode(byte3);
+                model.encode(byte4);
+
+                break;
+            }
+        }      
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename fce,
+        typename fcd,
+        typename crc32
+        >
+    void compress_stream_kernel_1<fce,fcd,crc32>::
+    decompress (
+        std::istream& in_,
+        std::ostream& out_
+    ) const
+    {
+
+        std::streambuf& out = *out_.rdbuf();
+
+        typename fcd::entropy_decoder_type coder;
+        coder.set_stream(in_);
+
+        fcd model(coder);
+
+        unsigned long symbol;
+        unsigned long count = 0;
+
+        crc32 crc;
+
+        // decode until we hit the marker symbol
+        while (true)
+        {
+            // make sure this is the value we expect
+            if (count == 20000)
+            {
+                if (coder.get_target(8000) != 1500)
+                {
+                    throw decompression_error("Error detected in compressed data stream.");
+                }
+                count = 0;
+                coder.decode(1500,1501);
+            }
+            ++count;
+
+            // decode the next symbol
+            model.decode(symbol);
+            if (symbol != eof_symbol)
+            {
+                crc.add(static_cast<unsigned char>(symbol));
+                // write this symbol to out
+                if (out.sputc(static_cast<char>(symbol)) != static_cast<int>(symbol))
+                {
+                    throw std::ios::failure("error occurred in compress_stream_kernel_1::decompress");
+                }
+                continue;
+            }
+            else
+            {
+                // we read eof from the encoded data.  now we just have to check the checksum and we are done.
+                unsigned char byte1;
+                unsigned char byte2;
+                unsigned char byte3;
+                unsigned char byte4;
+
+                model.decode(symbol); byte1 = static_cast<unsigned char>(symbol);
+                model.decode(symbol); byte2 = static_cast<unsigned char>(symbol);
+                model.decode(symbol); byte3 = static_cast<unsigned char>(symbol);
+                model.decode(symbol); byte4 = static_cast<unsigned char>(symbol);
+
+                unsigned long checksum = byte1;
+                checksum <<= 8;
+                checksum |= byte2;
+                checksum <<= 8;
+                checksum |= byte3;
+                checksum <<= 8;
+                checksum |= byte4;
+
+                if (checksum != crc.get_checksum())
+                    throw decompression_error("Error detected in compressed data stream.");
+
+                break;
+            }
+        } // while (true)
+
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_COMPRESS_STREAM_KERNEl_1_
+

dlib/compress_stream/compress_stream_kernel_2.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_COMPRESS_STREAM_KERNEl_2_
+#define DLIB_COMPRESS_STREAM_KERNEl_2_
+
+#include "../algs.h"
+#include <iostream>
+#include <streambuf>
+#include "compress_stream_kernel_abstract.h"
+
+namespace dlib
+{
+
+    template <
+        typename fce,
+        typename fcd,
+        typename lz77_buffer,
+        typename sliding_buffer,
+        typename fce_length,
+        typename fcd_length,
+        typename fce_index,
+        typename fcd_index,
+        typename crc32
+        >
+    class compress_stream_kernel_2
+    {
+        /*!
+            REQUIREMENTS ON fce
+                is an implementation of entropy_encoder_model/entropy_encoder_model_kernel_abstract.h
+                the alphabet_size of fce must be 257.
+                fce and fcd share the same kernel number.
+
+            REQUIREMENTS ON fcd
+                is an implementation of entropy_decoder_model/entropy_decoder_model_kernel_abstract.h
+                the alphabet_size of fcd must be 257.
+                fce and fcd share the same kernel number.
+
+            REQUIREMENTS ON lz77_buffer
+                is an implementation of lz77_buffer/lz77_buffer_kernel_abstract.h
+
+            REQUIREMENTS ON sliding_buffer
+                is an implementation of sliding_buffer/sliding_buffer_kernel_abstract.h
+                is instantiated with T = unsigned char
+
+            REQUIREMENTS ON fce_length
+                is an implementation of entropy_encoder_model/entropy_encoder_model_kernel_abstract.h
+                the alphabet_size of fce must be 513.  This will be used to encode the length of lz77 matches.
+                fce_length and fcd share the same kernel number.
+
+            REQUIREMENTS ON fcd_length
+                is an implementation of entropy_decoder_model/entropy_decoder_model_kernel_abstract.h
+                the alphabet_size of fcd must be 513.  This will be used to decode the length of lz77 matches.
+                fce_length and fcd share the same kernel number.
+
+            REQUIREMENTS ON fce_index
+                is an implementation of entropy_encoder_model/entropy_encoder_model_kernel_abstract.h
+                the alphabet_size of fce must be 32257.  This will be used to encode the index of lz77 matches.
+                fce_index and fcd share the same kernel number.
+
+            REQUIREMENTS ON fcd_index
+                is an implementation of entropy_decoder_model/entropy_decoder_model_kernel_abstract.h
+                the alphabet_size of fcd must be 32257.  This will be used to decode the index of lz77 matches.
+                fce_index and fcd share the same kernel number.
+
+            REQUIREMENTS ON crc32
+                is an implementation of crc32/crc32_kernel_abstract.h
+
+            INITIAL VALUE
+                this object has no state
+
+            CONVENTION
+                this object has no state
+        !*/
+
+        const static unsigned long eof_symbol = 256;
+
+    public:
+
+        class decompression_error : public dlib::error 
+        { 
+            public: 
+                decompression_error(
+                    const char* i
+                ) :
+                    dlib::error(std::string(i))
+                {}
+
+                decompression_error(
+                    const std::string& i
+                ) :
+                    dlib::error(i)
+                {}
+        };
+
+
+        compress_stream_kernel_2 (
+        )
+        {}
+
+        ~compress_stream_kernel_2 (
+        )
+        {}
+
+        void compress (
+            std::istream& in,
+            std::ostream& out
+        ) const;
+
+        void decompress (
+            std::istream& in,
+            std::ostream& out
+        ) const;
+
+    private:
+
+        // restricted functions
+        compress_stream_kernel_2(compress_stream_kernel_2&);        // copy constructor
+        compress_stream_kernel_2& operator=(compress_stream_kernel_2&);    // assignment operator
+
+    };
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename fce,
+        typename fcd,
+        typename lz77_buffer,
+        typename sliding_buffer,
+        typename fce_length,
+        typename fcd_length,
+        typename fce_index,
+        typename fcd_index,
+        typename crc32
+        >
+    void compress_stream_kernel_2<fce,fcd,lz77_buffer,sliding_buffer,fce_length,fcd_length,fce_index,fcd_index,crc32>::
+    compress (
+        std::istream& in_,
+        std::ostream& out_
+    ) const
+    {
+        std::streambuf::int_type temp;
+
+        std::streambuf& in = *in_.rdbuf();
+
+        typename fce::entropy_encoder_type coder;
+        coder.set_stream(out_);
+
+        fce model(coder);
+        fce_length model_length(coder);
+        fce_index model_index(coder);
+
+        const unsigned long LOOKAHEAD_LIMIT = 512; 
+        lz77_buffer buffer(15,LOOKAHEAD_LIMIT);
+        
+        crc32 crc;
+      
+
+        unsigned long count = 0;
+
+        unsigned long lz77_count = 1;  // number of times we used lz77 to encode
+        unsigned long ppm_count = 1;   // number of times we used ppm to encode
+
+
+        while (true)
+        {
+            // write out a known value every 20000 symbols
+            if (count == 20000)
+            {
+                count = 0;
+                coder.encode(150,151,400);
+            }
+            ++count;
+
+            // try to fill the lookahead buffer
+            if (buffer.get_lookahead_buffer_size() < buffer.get_lookahead_buffer_limit())
+            {
+                temp = in.sbumpc();
+                while (temp != EOF)
+                {
+                    crc.add(static_cast<unsigned char>(temp));
+                    buffer.add(static_cast<unsigned char>(temp));
+                    if (buffer.get_lookahead_buffer_size() == buffer.get_lookahead_buffer_limit())
+                        break;
+                    temp = in.sbumpc();
+                }
+            }
+
+            // compute the sum of ppm_count and lz77_count but make sure
+            // it is less than 65536
+            unsigned long sum = ppm_count + lz77_count;
+            if (sum >= 65536)
+            {
+                ppm_count >>= 1;                    
+                lz77_count >>= 1;
+                ppm_count |= 1;
+                lz77_count |= 1;
+                sum = ppm_count+lz77_count;                    
+            }
+
+            // if there are still more symbols in the lookahead buffer to encode
+            if (buffer.get_lookahead_buffer_size() > 0)  
+            {
+                unsigned long match_index, match_length;
+                buffer.find_match(match_index,match_length,6);
+                if (match_length != 0)
+                {
+                  
+                    // signal the decoder that we are using lz77
+                    coder.encode(0,lz77_count,sum);
+                    ++lz77_count;
+                    
+                    // encode the index and length pair
+                    model_index.encode(match_index);                   
+                    model_length.encode(match_length);                   
+
+                }
+                else
+                {
+
+                    // signal the decoder that we are using ppm 
+                    coder.encode(lz77_count,sum,sum);
+                    ++ppm_count;
+
+                    // encode the symbol using the ppm model
+                    model.encode(buffer.lookahead_buffer(0));
+                    buffer.shift_buffers(1);                    
+                }
+            }
+            else
+            {
+                // signal the decoder that we are using ppm 
+                coder.encode(lz77_count,sum,sum);
+                
+
+                model.encode(eof_symbol);
+                // now write the checksum
+                unsigned long checksum = crc.get_checksum();
+                unsigned char byte1 = static_cast<unsigned char>((checksum>>24)&0xFF);
+                unsigned char byte2 = static_cast<unsigned char>((checksum>>16)&0xFF);
+                unsigned char byte3 = static_cast<unsigned char>((checksum>>8)&0xFF);
+                unsigned char byte4 = static_cast<unsigned char>((checksum)&0xFF);
+
+                model.encode(byte1);
+                model.encode(byte2);
+                model.encode(byte3);
+                model.encode(byte4);
+
+                break;      
+            }
+        } // while (true)        
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename fce,
+        typename fcd,
+        typename lz77_buffer,
+        typename sliding_buffer,
+        typename fce_length,
+        typename fcd_length,
+        typename fce_index,
+        typename fcd_index,
+        typename crc32
+        >
+    void compress_stream_kernel_2<fce,fcd,lz77_buffer,sliding_buffer,fce_length,fcd_length,fce_index,fcd_index,crc32>::
+    decompress (
+        std::istream& in_,
+        std::ostream& out_
+    ) const
+    {
+
+        std::streambuf& out = *out_.rdbuf();
+
+        typename fcd::entropy_decoder_type coder;
+        coder.set_stream(in_);
+
+        fcd model(coder);
+        fcd_length model_length(coder);
+        fcd_index model_index(coder);
+
+        unsigned long symbol;
+        unsigned long count = 0;
+
+        sliding_buffer buffer;
+        buffer.set_size(15);
+
+        // Initialize the buffer to all zeros.  There is no algorithmic reason to
+        // do this.  But doing so avoids a warning from valgrind so that is why
+        // I'm doing this.
+        for (unsigned long i = 0; i < buffer.size(); ++i)
+              buffer[i] = 0;
+
+        crc32 crc;
+        
+        unsigned long lz77_count = 1;  // number of times we used lz77 to encode
+        unsigned long ppm_count = 1;   // number of times we used ppm to encode
+        bool next_block_lz77;
+
+
+        // decode until we hit the marker symbol
+        while (true)
+        {
+            // make sure this is the value we expect
+            if (count == 20000)
+            {
+                if (coder.get_target(400) != 150)
+                {
+                    throw decompression_error("Error detected in compressed data stream.");
+                }
+                count = 0;
+                coder.decode(150,151);
+            }
+            ++count;
+
+
+            // compute the sum of ppm_count and lz77_count but make sure
+            // it is less than 65536
+            unsigned long sum = ppm_count + lz77_count;
+            if (sum >= 65536)
+            {
+                ppm_count >>= 1;                    
+                lz77_count >>= 1;
+                ppm_count |= 1;
+                lz77_count |= 1;
+                sum = ppm_count+lz77_count;                    
+            }
+
+            // check if we are decoding a lz77 or ppm block
+            if (coder.get_target(sum) < lz77_count)
+            {
+                coder.decode(0,lz77_count);
+                next_block_lz77 = true;
+                ++lz77_count;
+            }
+            else
+            {
+                coder.decode(lz77_count,sum);
+                next_block_lz77 = false;
+                ++ppm_count;
+            }
+
+
+            if (next_block_lz77)
+            {
+                
+                unsigned long match_length, match_index;
+                // decode the match index
+                model_index.decode(match_index);
+
+                // decode the match length
+                model_length.decode(match_length);
+
+                
+                match_index += match_length;
+                buffer.rotate_left(match_length);
+                for (unsigned long i = 0; i < match_length; ++i)
+                {
+                    unsigned char ch = buffer[match_index-i];
+                    buffer[match_length-i-1] = ch;
+
+                    crc.add(ch);
+                    // write this ch to out
+                    if (out.sputc(static_cast<char>(ch)) != static_cast<int>(ch))
+                    {
+                        throw std::ios::failure("error occurred in compress_stream_kernel_2::decompress");
+                    }
+                }
+                
+            }
+            else
+            {
+
+                // decode the next symbol
+                model.decode(symbol);
+                if (symbol != eof_symbol)
+                {
+                    buffer.rotate_left(1);
+                    buffer[0] = static_cast<unsigned char>(symbol);
+                    
+
+                    crc.add(static_cast<unsigned char>(symbol));
+                    // write this symbol to out
+                    if (out.sputc(static_cast<char>(symbol)) != static_cast<int>(symbol))
+                    {
+                        throw std::ios::failure("error occurred in compress_stream_kernel_2::decompress");
+                    }
+                }
+                else
+                {
+                    // this was the eof marker symbol so we are done.  now check the checksum
+
+                    // now get the checksum and make sure it matches
+                    unsigned char byte1;
+                    unsigned char byte2;
+                    unsigned char byte3;
+                    unsigned char byte4;
+
+                    model.decode(symbol); byte1 = static_cast<unsigned char>(symbol);
+                    model.decode(symbol); byte2 = static_cast<unsigned char>(symbol);
+                    model.decode(symbol); byte3 = static_cast<unsigned char>(symbol);
+                    model.decode(symbol); byte4 = static_cast<unsigned char>(symbol);
+
+                    unsigned long checksum = byte1;
+                    checksum <<= 8;
+                    checksum |= byte2;
+                    checksum <<= 8;
+                    checksum |= byte3;
+                    checksum <<= 8;
+                    checksum |= byte4;
+
+                    if (checksum != crc.get_checksum())
+                        throw decompression_error("Error detected in compressed data stream.");
+
+                    break;
+                }
+            }
+
+        } // while (true)
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_COMPRESS_STREAM_KERNEl_2_
+

dlib/compress_stream/compress_stream_kernel_3.h

+// Copyright (C) 2005  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_COMPRESS_STREAM_KERNEl_3_
+#define DLIB_COMPRESS_STREAM_KERNEl_3_
+
+#include "../algs.h"
+#include "compress_stream_kernel_abstract.h"
+#include "../assert.h"
+
+namespace dlib
+{
+
+    template <
+        typename lzp_buf,
+        typename crc32,
+        unsigned long buffer_size
+        >
+    class compress_stream_kernel_3
+    {
+        /*!
+            REQUIREMENTS ON lzp_buf
+                is an implementation of lzp_buffer/lzp_buffer_kernel_abstract.h
+
+            REQUIREMENTS ON buffer_size
+                10 < buffer_size < 32
+
+            REQUIREMENTS ON crc32
+                is an implementation of crc32/crc32_kernel_abstract.h
+
+
+            INITIAL VALUE
+                this object has no state
+
+            CONVENTION
+                this object has no state
+
+
+                This implementation uses the lzp_buffer and writes out matches
+                in a byte aligned format.
+
+        !*/
+
+
+    public:
+
+        class decompression_error : public dlib::error 
+        { 
+            public: 
+                decompression_error(
+                    const char* i
+                ) :
+                    dlib::error(std::string(i))
+                {}
+
+                decompression_error(
+                    const std::string& i
+                ) :
+                    dlib::error(i)
+                {}
+        };
+
+
+        compress_stream_kernel_3 (
+        )
+        {
+            COMPILE_TIME_ASSERT(10 < buffer_size && buffer_size < 32);
+        }
+
+        ~compress_stream_kernel_3 (
+        )
+        {}
+
+        void compress (
+            std::istream& in,
+            std::ostream& out
+        ) const;
+
+        void decompress (
+            std::istream& in,
+            std::ostream& out
+        ) const;
+
+
+
+    private:
+
+        inline void write (
+            unsigned char symbol
+        ) const
+        { 
+            if (out->sputn(reinterpret_cast<char*>(&symbol),1)==0)
+                throw std::ios_base::failure("error writing to output stream in compress_stream_kernel_3");        
+        }
+
+        inline void decode (
+            unsigned char& symbol,
+            unsigned char& flag
+        ) const
+        { 
+            if (count == 0)
+            {
+                if (((size_t)in->sgetn(reinterpret_cast<char*>(buffer),sizeof(buffer)))!=sizeof(buffer))
+                    throw decompression_error("Error detected in compressed data stream.");
+                count = 8;
+            }
+            --count;
+            symbol = buffer[8-count];
+            flag = buffer[0] >> 7; 
+            buffer[0] <<= 1;
+        }
+
+        inline void encode (
+            unsigned char symbol,
+            unsigned char flag
+        ) const
+        /*!
+            requires
+                - 0 <= flag <= 1
+            ensures
+                - writes symbol with the given one bit flag
+        !*/
+        { 
+            // add this symbol and flag to the buffer            
+            ++count;
+            buffer[0] <<= 1;
+            buffer[count] = symbol;
+            buffer[0] |= flag;
+
+            if (count == 8)
+            {
+                if (((size_t)out->sputn(reinterpret_cast<char*>(buffer),sizeof(buffer)))!=sizeof(buffer))
+                    throw std::ios_base::failure("error writing to output stream in compress_stream_kernel_3");        
+                count = 0;
+                buffer[0] = 0;
+            }
+        }
+
+        void clear (
+        ) const
+        /*!
+            ensures
+                - resets the buffers
+        !*/
+        {
+            count = 0;
+        }
+
+        void flush (
+        ) const
+        /*!
+            ensures
+                - flushes any data in the buffers to out
+        !*/
+        {
+            if (count != 0)
+            {
+                buffer[0] <<= (8-count);
+                if (((size_t)out->sputn(reinterpret_cast<char*>(buffer),sizeof(buffer)))!=sizeof(buffer))
+                    throw std::ios_base::failure("error writing to output stream in compress_stream_kernel_3");        
+            }
+        }
+
+        mutable unsigned int count;
+        // count tells us how many bytes are buffered in buffer and how many flag
+        // bit are currently in buffer[0]
+        mutable unsigned char buffer[9];  
+        // buffer[0] holds the flag bits to be writen.
+        // the rest of the buffer holds the bytes to be writen.
+
+        mutable std::streambuf* in;
+        mutable std::streambuf* out;
+
+        // restricted functions
+        compress_stream_kernel_3(compress_stream_kernel_3<lzp_buf,crc32,buffer_size>&);        // copy constructor
+        compress_stream_kernel_3<lzp_buf,crc32,buffer_size>& operator=(compress_stream_kernel_3<lzp_buf,crc32,buffer_size>&);    // assignment operator
+
+    };
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename lzp_buf,
+        typename crc32,
+        unsigned long buffer_size
+        >
+    void compress_stream_kernel_3<lzp_buf,crc32,buffer_size>::
+    compress (
+        std::istream& in_,
+        std::ostream& out_
+    ) const
+    {
+        in = in_.rdbuf();
+        out = out_.rdbuf();
+        clear();
+
+        crc32 crc;
+     
+        lzp_buf buffer(buffer_size);
+
+        std::streambuf::int_type temp = in->sbumpc();
+        unsigned long index;
+        unsigned char symbol;
+        unsigned char length;
+
+        while (temp != EOF)
+        {
+            symbol = static_cast<unsigned char>(temp);
+            if (buffer.predict_match(index))
+            {
+                if (buffer[index] == symbol)
+                {
+                    // this is a match so we must find out how long it is
+                    length = 1;
+                                        
+                    buffer.add(symbol);
+                    crc.add(symbol);
+
+                    temp = in->sbumpc();
+                    while (length < 255)
+                    {
+                        if (temp == EOF)
+                        {                          
+                            break;
+                        }
+                        else if (static_cast<unsigned long>(length) >= index)
+                        {
+                            break;
+                        }
+                        else if (static_cast<unsigned char>(temp) == buffer[index])
+                        {
+                            ++length;
+                            buffer.add(static_cast<unsigned char>(temp));
+                            crc.add(static_cast<unsigned char>(temp));
+                            temp = in->sbumpc();
+                        }
+                        else
+                        {
+                            break;
+                        }
+                    }                        
+
+                    encode(length,1);
+                }
+                else
+                {
+                    // this is also not a match
+                    encode(symbol,0);
+                    buffer.add(symbol);
+                    crc.add(symbol);
+
+                    // get the next symbol
+                    temp = in->sbumpc();
+                }
+            }
+            else
+            {
+                // there wasn't a match so just write this symbol
+                encode(symbol,0);
+                buffer.add(symbol);
+                crc.add(symbol);
+
+                // get the next symbol
+                temp = in->sbumpc();
+            }
+        }
+
+        // use a match of zero length to indicate EOF
+        encode(0,1);
+
+        // now write the checksum
+        unsigned long checksum = crc.get_checksum();
+        unsigned char byte1 = static_cast<unsigned char>((checksum>>24)&0xFF);
+        unsigned char byte2 = static_cast<unsigned char>((checksum>>16)&0xFF);
+        unsigned char byte3 = static_cast<unsigned char>((checksum>>8)&0xFF);
+        unsigned char byte4 = static_cast<unsigned char>((checksum)&0xFF);
+
+        encode(byte1,0);
+        encode(byte2,0);
+        encode(byte3,0);
+        encode(byte4,0);
+
+        flush();
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename lzp_buf,
+        typename crc32,
+        unsigned long buffer_size
+        >
+    void compress_stream_kernel_3<lzp_buf,crc32,buffer_size>::
+    decompress (
+        std::istream& in_,
+        std::ostream& out_
+    ) const
+    { 
+        in = in_.rdbuf();
+        out = out_.rdbuf();
+        clear();
+     
+        crc32 crc;
+
+        lzp_buf buffer(buffer_size);
+
+
+        unsigned long index = 0;
+        unsigned char symbol;
+        unsigned char length;
+        unsigned char flag;
+
+        decode(symbol,flag);
+        while (flag == 0 || symbol != 0)
+        {
+            buffer.predict_match(index);
+
+            if (flag == 1)
+            {
+                length = symbol;
+                do 
+                {
+                    --length;
+                    symbol = buffer[index];
+                    write(symbol);
+                    buffer.add(symbol);   
+                    crc.add(symbol);                    
+                } while (length != 0);
+            }
+            else
+            {
+                // this is just a literal
+                write(symbol);
+                buffer.add(symbol);
+                crc.add(symbol);
+            }
+            decode(symbol,flag);
+        }
+
+
+        // now get the checksum and make sure it matches
+        unsigned char byte1;
+        unsigned char byte2;
+        unsigned char byte3;
+        unsigned char byte4;
+
+        decode(byte1,flag);
+        if (flag != 0)
+            throw decompression_error("Error detected in compressed data stream.");
+        decode(byte2,flag);
+        if (flag != 0)
+            throw decompression_error("Error detected in compressed data stream.");
+        decode(byte3,flag);
+        if (flag != 0)
+            throw decompression_error("Error detected in compressed data stream.");
+        decode(byte4,flag);
+        if (flag != 0)
+            throw decompression_error("Error detected in compressed data stream.");
+
+        unsigned long checksum = byte1;
+        checksum <<= 8;
+        checksum |= byte2;
+        checksum <<= 8;
+        checksum |= byte3;
+        checksum <<= 8;
+        checksum |= byte4;
+
+        if (checksum != crc.get_checksum())
+            throw decompression_error("Error detected in compressed data stream.");
+ 
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_COMPRESS_STREAM_KERNEl_3_
+

dlib/compress_stream/compress_stream_kernel_abstract.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_COMPRESS_STREAM_KERNEl_ABSTRACT_
+#ifdef DLIB_COMPRESS_STREAM_KERNEl_ABSTRACT_
+
+#include "../algs.h"
+#include <iosfwd>
+
+namespace dlib
+{
+
+    class compress_stream 
+    {
+        /*!
+            INITIAL VALUE
+                This object does not have any state associated with it.
+
+            WHAT THIS OBJECT REPRESENTS
+                This object consists of the two functions compress and decompress.
+                These functions allow you to compress and decompress data.
+        !*/
+
+    public:
+
+        class decompression_error : public dlib::error {};
+
+        compress_stream (
+        );
+        /*!
+            ensures
+                - #*this is properly initialized
+            throws
+                - std::bad_alloc
+        !*/
+
+        virtual ~compress_stream (
+        );
+        /*!
+            ensures
+                - all memory associated with *this has been released
+        !*/
+
+
+        void compress (
+            std::istream& in,
+            std::ostream& out
+        ) const;
+        /*!
+            ensures
+                - reads all data from in (until EOF is reached) and compresses it
+                  and writes it to out
+            throws
+                - std::ios_base::failure
+                    if there was a problem writing to out then this exception will 
+                    be thrown.                      
+                - any other exception
+                    this exception may be thrown if there is any other problem                    
+        !*/
+
+
+        void decompress (
+            std::istream& in,
+            std::ostream& out
+        ) const;
+        /*!
+            ensures
+                - reads data from in, decompresses it and writes it to out.  note that
+                  it stops reading data from in when it encounters the end of the 
+                  compressed data, not when it encounters EOF. 
+            throws
+                - std::ios_base::failure
+                    if there was a problem writing to out then this exception will 
+                    be thrown.           
+                - decompression_error
+                    if an error was detected in the compressed data that prevented
+                    it from being correctly decompressed then this exception is 
+                    thrown.  
+                - any other exception
+                    this exception may be thrown if there is any other problem                    
+        !*/
+
+
+    private:
+
+        // restricted functions
+        compress_stream(compress_stream&);        // copy constructor
+        compress_stream& operator=(compress_stream&);    // assignment operator
+
+    };   
+   
+}
+
+#endif // DLIB_COMPRESS_STREAM_KERNEl_ABSTRACT_
+

dlib/conditioning_class.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_CONDITIONING_CLASs_
+#define DLIB_CONDITIONING_CLASs_
+
+#include "conditioning_class/conditioning_class_kernel_1.h"
+#include "conditioning_class/conditioning_class_kernel_2.h"
+#include "conditioning_class/conditioning_class_kernel_3.h"
+#include "conditioning_class/conditioning_class_kernel_4.h"
+#include "conditioning_class/conditioning_class_kernel_c.h"
+
+
+#include "memory_manager.h"
+
+namespace dlib
+{
+
+    template <
+        unsigned long alphabet_size
+        >
+    class conditioning_class
+    {
+        conditioning_class() {}
+
+        typedef memory_manager<char>::kernel_2b mm;
+
+    public:
+        
+        //----------- kernels ---------------
+
+        // kernel_1a        
+        typedef      conditioning_class_kernel_1<alphabet_size>
+                     kernel_1a;
+        typedef      conditioning_class_kernel_c<kernel_1a>
+                     kernel_1a_c;
+
+        // kernel_2a        
+        typedef      conditioning_class_kernel_2<alphabet_size>
+                     kernel_2a;
+        typedef      conditioning_class_kernel_c<kernel_2a>
+                     kernel_2a_c;
+          
+        // kernel_3a        
+        typedef      conditioning_class_kernel_3<alphabet_size>
+                     kernel_3a;
+        typedef      conditioning_class_kernel_c<kernel_3a>
+                     kernel_3a_c;
+          
+
+        // -------- kernel_4 ---------
+
+        // kernel_4a        
+        typedef      conditioning_class_kernel_4<alphabet_size,10000,mm>
+                     kernel_4a;
+        typedef      conditioning_class_kernel_c<kernel_4a>
+                     kernel_4a_c;
+
+        // kernel_4b        
+        typedef      conditioning_class_kernel_4<alphabet_size,100000,mm>
+                     kernel_4b;
+        typedef      conditioning_class_kernel_c<kernel_4b>
+                     kernel_4b_c;
+
+        // kernel_4c        
+        typedef      conditioning_class_kernel_4<alphabet_size,1000000,mm>
+                     kernel_4c;
+        typedef      conditioning_class_kernel_c<kernel_4c>
+                     kernel_4c_c;
+
+        // kernel_4d        
+        typedef      conditioning_class_kernel_4<alphabet_size,10000000,mm>
+                     kernel_4d;
+        typedef      conditioning_class_kernel_c<kernel_4d>
+                     kernel_4d_c;
+
+    };
+}
+
+#endif // DLIB_CONDITIONING_CLASS_
+

dlib/conditioning_class/conditioning_class_kernel_1.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_CONDITIONING_CLASS_KERNEl_1_
+#define DLIB_CONDITIONING_CLASS_KERNEl_1_
+
+#include "conditioning_class_kernel_abstract.h"
+#include "../assert.h"
+#include "../algs.h"
+
+namespace dlib
+{
+
+    template <
+        unsigned long alphabet_size
+        >
+    class conditioning_class_kernel_1 
+    {
+        /*!
+            INITIAL VALUE
+                total == 1
+                counts == pointer to an array of alphabet_size unsigned shorts
+                for all i except i == alphabet_size-1: counts[i] == 0
+                counts[alphabet_size-1] == 1
+
+            CONVENTION
+                counts == pointer to an array of alphabet_size unsigned shorts
+                get_total() == total
+                get_count(symbol) == counts[symbol]
+
+                LOW_COUNT(symbol) == sum of counts[0] though counts[symbol-1]
+                                     or 0 if symbol == 0
+
+                get_memory_usage() == global_state.memory_usage
+        !*/
+
+    public:
+
+        class global_state_type
+        {
+        public:
+            global_state_type () : memory_usage(0) {}
+        private:
+            unsigned long memory_usage;
+
+            friend class conditioning_class_kernel_1<alphabet_size>;
+        };
+
+        conditioning_class_kernel_1 (
+            global_state_type& global_state_
+        );
+
+        ~conditioning_class_kernel_1 (
+        );
+
+        void clear(
+        );
+
+        bool increment_count (
+            unsigned long symbol,
+            unsigned short amount = 1
+        );
+
+        unsigned long get_count (
+            unsigned long symbol
+        ) const;
+
+        unsigned long get_total (
+        ) const;
+        
+        unsigned long get_range (
+            unsigned long symbol,
+            unsigned long& low_count,
+            unsigned long& high_count,
+            unsigned long& total_count
+        ) const;
+
+        void get_symbol (
+            unsigned long target,
+            unsigned long& symbol,            
+            unsigned long& low_count,
+            unsigned long& high_count
+        ) const;
+
+        unsigned long get_memory_usage (
+        ) const;
+
+        global_state_type& get_global_state (
+        );
+
+        static unsigned long get_alphabet_size (
+        );
+
+
+    private:
+
+        // restricted functions
+        conditioning_class_kernel_1(conditioning_class_kernel_1<alphabet_size>&);        // copy constructor
+        conditioning_class_kernel_1& operator=(conditioning_class_kernel_1<alphabet_size>&);    // assignment operator
+
+        // data members
+        unsigned short total;
+        unsigned short* counts;
+        global_state_type& global_state;
+
+    };   
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    conditioning_class_kernel_1<alphabet_size>::
+    conditioning_class_kernel_1 (
+        global_state_type& global_state_
+    ) :
+        total(1),
+        counts(new unsigned short[alphabet_size]),
+        global_state(global_state_)
+    {
+        COMPILE_TIME_ASSERT( 1 < alphabet_size && alphabet_size < 65536 );
+
+        unsigned short* start = counts;
+        unsigned short* end = counts+alphabet_size-1;
+        while (start != end)
+        {
+            *start = 0;
+            ++start;
+        }
+        *start = 1;
+
+        // update memory usage
+        global_state.memory_usage += sizeof(unsigned short)*alphabet_size + 
+                                     sizeof(conditioning_class_kernel_1);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    conditioning_class_kernel_1<alphabet_size>::
+    ~conditioning_class_kernel_1 (
+    )
+    {
+        delete [] counts;
+        // update memory usage
+        global_state.memory_usage -= sizeof(unsigned short)*alphabet_size + 
+                                     sizeof(conditioning_class_kernel_1);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    void conditioning_class_kernel_1<alphabet_size>::
+    clear(
+    )
+    {
+        total = 1;
+        unsigned short* start = counts;
+        unsigned short* end = counts+alphabet_size-1;
+        while (start != end)
+        {
+            *start = 0;
+            ++start;
+        }
+        *start = 1;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_1<alphabet_size>::
+    get_memory_usage(
+    ) const
+    {
+        return global_state.memory_usage;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    typename conditioning_class_kernel_1<alphabet_size>::global_state_type& conditioning_class_kernel_1<alphabet_size>::
+    get_global_state(
+    )
+    {
+        return global_state;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    bool conditioning_class_kernel_1<alphabet_size>::
+    increment_count (
+        unsigned long symbol,
+        unsigned short amount
+    )
+    {        
+        // if we are going over a total of 65535 then scale down all counts by 2
+        if (static_cast<unsigned long>(total)+static_cast<unsigned long>(amount) >= 65536)
+        {
+            total = 0;
+            unsigned short* start = counts;
+            unsigned short* end = counts+alphabet_size;
+            while (start != end)
+            {
+                *start >>= 1;
+                total += *start;
+                ++start;
+            }    
+            // make sure it is at least one
+            if (counts[alphabet_size-1]==0)
+            {
+                ++total;
+                counts[alphabet_size-1] = 1;
+            }
+        }
+        counts[symbol] += amount;
+        total += amount;
+        return true;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_1<alphabet_size>::
+    get_count (
+        unsigned long symbol
+    ) const
+    {
+        return counts[symbol];
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_1<alphabet_size>::
+    get_alphabet_size (        
+    ) 
+    {
+        return alphabet_size;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_1<alphabet_size>::
+    get_total (
+    ) const
+    {
+        return total;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_1<alphabet_size>::
+    get_range (
+        unsigned long symbol,
+        unsigned long& low_count,
+        unsigned long& high_count,
+        unsigned long& total_count
+    ) const
+    {
+        if (counts[symbol] == 0)
+            return 0;
+
+        total_count = total;
+        
+        const unsigned short* start = counts;
+        const unsigned short* end = counts+symbol;
+        unsigned short high_count_temp = *start;
+        while (start != end)
+        {
+            ++start;
+            high_count_temp += *start;            
+        }  
+        low_count = high_count_temp - *start;
+        high_count = high_count_temp;
+        return *start;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    void conditioning_class_kernel_1<alphabet_size>::
+    get_symbol (
+        unsigned long target,
+        unsigned long& symbol,            
+        unsigned long& low_count,
+        unsigned long& high_count
+    ) const
+    {
+        unsigned long high_count_temp = *counts;
+        const unsigned short* start = counts;        
+        while (target >= high_count_temp)
+        {
+            ++start;
+            high_count_temp += *start;            
+        } 
+
+        low_count = high_count_temp - *start;
+        high_count = high_count_temp;
+        symbol = static_cast<unsigned long>(start-counts);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_CONDITIONING_CLASS_KERNEl_1_
+

dlib/conditioning_class/conditioning_class_kernel_2.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_CONDITIONING_CLASS_KERNEl_2_
+#define DLIB_CONDITIONING_CLASS_KERNEl_2_
+
+#include "conditioning_class_kernel_abstract.h"
+#include "../assert.h"
+#include "../algs.h"
+
+namespace dlib
+{
+
+    template <
+        unsigned long alphabet_size
+        >
+    class conditioning_class_kernel_2 
+    {
+        /*!
+            INITIAL VALUE
+                total == 1
+                symbols == pointer to array of alphabet_size data structs
+                for all i except i == alphabet_size-1: symbols[i].count == 0
+                                                       symbols[i].left_count == 0
+
+                symbols[alphabet_size-1].count == 1
+                symbols[alpahbet_size-1].left_count == 0
+
+            CONVENTION
+                symbols == pointer to array of alphabet_size data structs
+                get_total() == total
+                get_count(symbol) == symbols[symbol].count
+
+                symbols is organized as a tree with symbols[0] as the root.
+
+                the left subchild of symbols[i] is symbols[i*2+1] and
+                the right subchild is symbols[i*2+2].
+                the partent of symbols[i] == symbols[(i-1)/2]
+
+                symbols[i].left_count == the sum of the counts of all the
+                                         symbols to the left of symbols[i]
+
+                get_memory_usage() == global_state.memory_usage                                         
+        !*/
+
+    public:
+
+        class global_state_type
+        {
+        public:
+            global_state_type () : memory_usage(0) {}
+        private:
+            unsigned long memory_usage;
+
+            friend class conditioning_class_kernel_2<alphabet_size>;
+        };
+
+        conditioning_class_kernel_2 (
+            global_state_type& global_state_
+        );
+
+        ~conditioning_class_kernel_2 (
+        );
+
+        void clear(
+        );
+
+        bool increment_count (
+            unsigned long symbol,
+            unsigned short amount = 1
+        );
+
+        unsigned long get_count (
+            unsigned long symbol
+        ) const;
+
+        inline unsigned long get_total (
+        ) const;
+        
+        unsigned long get_range (
+            unsigned long symbol,
+            unsigned long& low_count,
+            unsigned long& high_count,
+            unsigned long& total_count
+        ) const;
+
+        void get_symbol (
+            unsigned long target,
+            unsigned long& symbol,            
+            unsigned long& low_count,
+            unsigned long& high_count
+        ) const;
+
+        unsigned long get_memory_usage (
+        ) const;
+
+        global_state_type& get_global_state (
+        );
+
+        static unsigned long get_alphabet_size (
+        );
+
+    private:
+
+        // restricted functions
+        conditioning_class_kernel_2(conditioning_class_kernel_2<alphabet_size>&);        // copy constructor
+        conditioning_class_kernel_2& operator=(conditioning_class_kernel_2<alphabet_size>&);    // assignment operator
+
+        // data members
+        unsigned short total;
+        struct data
+        {
+            unsigned short count;
+            unsigned short left_count;
+        };
+
+        data* symbols;
+        global_state_type& global_state;
+
+    };   
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    conditioning_class_kernel_2<alphabet_size>::
+    conditioning_class_kernel_2 (
+        global_state_type& global_state_
+    ) :
+        total(1),
+        symbols(new data[alphabet_size]),
+        global_state(global_state_)
+    {
+        COMPILE_TIME_ASSERT( 1 < alphabet_size && alphabet_size < 65536 );
+
+        data* start = symbols;
+        data* end = symbols + alphabet_size-1;
+        
+        while (start != end)
+        {            
+            start->count = 0;            
+            start->left_count = 0;
+            ++start;
+        }
+
+        start->count = 1;
+        start->left_count = 0;
+
+
+        // update the left_counts for the symbol alphabet_size-1
+        unsigned short temp;
+        unsigned long symbol = alphabet_size-1;
+        while (symbol != 0)
+        {
+            // temp will be 1 if symbol is odd, 0 if it is even
+            temp = static_cast<unsigned short>(symbol&0x1);
+
+            // set symbol to its parent
+            symbol = (symbol-1)>>1;
+            
+            // note that all left subchidren are odd and also that 
+            // if symbol was a left subchild then we want to increment
+            // its parents left_count 
+            if (temp)
+                ++symbols[symbol].left_count;
+        }
+
+        global_state.memory_usage += sizeof(data)*alphabet_size + 
+                                     sizeof(conditioning_class_kernel_2);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    conditioning_class_kernel_2<alphabet_size>::
+    ~conditioning_class_kernel_2 (
+    )
+    {
+        delete [] symbols;
+        global_state.memory_usage -= sizeof(data)*alphabet_size + 
+                                     sizeof(conditioning_class_kernel_2);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    void conditioning_class_kernel_2<alphabet_size>::
+    clear(
+    )
+    {
+        data* start = symbols;
+        data* end = symbols + alphabet_size-1;
+        
+        total = 1;
+
+        while (start != end)
+        {            
+            start->count = 0;            
+            start->left_count = 0;
+            ++start;
+        }
+
+        start->count = 1;
+        start->left_count = 0;
+
+        // update the left_counts 
+        unsigned short temp;
+        unsigned long symbol = alphabet_size-1;
+        while (symbol != 0)
+        {
+            // temp will be 1 if symbol is odd, 0 if it is even
+            temp = static_cast<unsigned short>(symbol&0x1);
+
+            // set symbol to its parent
+            symbol = (symbol-1)>>1;
+            
+            // note that all left subchidren are odd and also that 
+            // if symbol was a left subchild then we want to increment
+            // its parents left_count 
+            symbols[symbol].left_count += temp;
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_2<alphabet_size>::
+    get_memory_usage(
+    ) const
+    {
+        return global_state.memory_usage;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    typename conditioning_class_kernel_2<alphabet_size>::global_state_type& conditioning_class_kernel_2<alphabet_size>::
+    get_global_state(
+    )
+    {
+        return global_state;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    bool conditioning_class_kernel_2<alphabet_size>::
+    increment_count (
+        unsigned long symbol,
+        unsigned short amount
+    )
+    {        
+        // if we need to renormalize then do so
+        if (static_cast<unsigned long>(total)+static_cast<unsigned long>(amount) >= 65536)
+        {
+            unsigned long s;
+            unsigned short temp;
+            for (unsigned short i = 0; i < alphabet_size-1; ++i)
+            {
+                s = i;
+
+                // divide the count for this symbol by 2                
+                symbols[i].count >>= 1;
+                
+                symbols[i].left_count = 0;
+
+                // bubble this change up though the tree                
+                while (s != 0)
+                {
+                    // temp will be 1 if symbol is odd, 0 if it is even
+                    temp = static_cast<unsigned short>(s&0x1);
+
+                    // set s to its parent
+                    s = (s-1)>>1;
+                    
+                    // note that all left subchidren are odd and also that 
+                    // if s was a left subchild then we want to increment
+                    // its parents left_count 
+                    if (temp)
+                        symbols[s].left_count += symbols[i].count;
+                }   
+            }
+
+            // update symbols alphabet_size-1
+            {
+                s = alphabet_size-1;
+
+                // divide alphabet_size-1 symbol by 2 if it's > 1
+                if (symbols[alphabet_size-1].count > 1)
+                    symbols[alphabet_size-1].count >>= 1;
+                
+                // bubble this change up though the tree                
+                while (s != 0)
+                {
+                    // temp will be 1 if symbol is odd, 0 if it is even
+                    temp = static_cast<unsigned short>(s&0x1);
+
+                    // set s to its parent
+                    s = (s-1)>>1;
+                    
+                    // note that all left subchidren are odd and also that 
+                    // if s was a left subchild then we want to increment
+                    // its parents left_count 
+                    if (temp)
+                        symbols[s].left_count += symbols[alphabet_size-1].count;
+                }   
+            }        
+            
+
+
+
+
+
+            // calculate the new total
+            total = 0;
+            unsigned long m = 0;
+            while (m < alphabet_size)
+            {
+                total += symbols[m].count + symbols[m].left_count;
+                m = (m<<1) + 2;        
+            }
+            
+        }
+
+        
+
+
+        // increment the count for the specified symbol      
+        symbols[symbol].count += amount;;
+        total += amount;
+
+        
+        unsigned short temp;
+        while (symbol != 0)
+        {
+            // temp will be 1 if symbol is odd, 0 if it is even
+            temp = static_cast<unsigned short>(symbol&0x1);
+
+            // set symbol to its parent
+            symbol = (symbol-1)>>1;
+            
+            // note that all left subchidren are odd and also that 
+            // if symbol was a left subchild then we want to increment
+            // its parents left_count 
+            if (temp)
+                symbols[symbol].left_count += amount;
+        }
+
+        return true;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_2<alphabet_size>::
+    get_count (
+        unsigned long symbol
+    ) const
+    {
+        return symbols[symbol].count;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_2<alphabet_size>::
+    get_alphabet_size (        
+    ) 
+    {
+        return alphabet_size;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_2<alphabet_size>::
+    get_total (
+    ) const
+    {
+        return total;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_2<alphabet_size>::
+    get_range (
+        unsigned long symbol,
+        unsigned long& low_count,
+        unsigned long& high_count,
+        unsigned long& total_count
+    ) const
+    {
+        if (symbols[symbol].count == 0)
+            return 0;
+
+        unsigned long current = symbol;
+        total_count = total;
+        unsigned long high_count_temp = 0;
+        bool came_from_right = true;
+        while (true)
+        {                        
+            if (came_from_right)
+            {
+                high_count_temp += symbols[current].count + symbols[current].left_count;
+            }
+
+            // note that if current is even then it is a right child
+            came_from_right = !(current&0x1);
+
+            if (current == 0)
+                break;
+
+            // set current to its parent
+            current = (current-1)>>1 ;
+        }
+
+
+        low_count = high_count_temp - symbols[symbol].count;
+        high_count = high_count_temp;
+
+        return symbols[symbol].count;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    void conditioning_class_kernel_2<alphabet_size>::
+    get_symbol (
+        unsigned long target,
+        unsigned long& symbol,            
+        unsigned long& low_count,
+        unsigned long& high_count
+    ) const
+    {
+        unsigned long current = 0;
+        unsigned long low_count_temp = 0;
+        
+        while (true)
+        {
+            if (static_cast<unsigned short>(target) < symbols[current].left_count)
+            {
+                // we should go left
+                current = (current<<1) + 1;
+            }
+            else 
+            {
+                target -= symbols[current].left_count;
+                low_count_temp += symbols[current].left_count;
+                if (static_cast<unsigned short>(target) < symbols[current].count)
+                {
+                    // we have found our target
+                    symbol = current;
+                    high_count = low_count_temp + symbols[current].count;
+                    low_count = low_count_temp;
+                    break;
+                }
+                else
+                {   
+                    // go right
+                    target -= symbols[current].count;
+                    low_count_temp += symbols[current].count;
+                    current = (current<<1) + 2;
+                }
+            }
+
+        }
+
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_CONDITIONING_CLASS_KERNEl_1_
+

dlib/conditioning_class/conditioning_class_kernel_3.h

+// Copyright (C) 2004  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_CONDITIONING_CLASS_KERNEl_3_
+#define DLIB_CONDITIONING_CLASS_KERNEl_3_
+
+#include "conditioning_class_kernel_abstract.h"
+#include "../assert.h"
+#include "../algs.h"
+
+
+namespace dlib
+{
+
+    template <
+        unsigned long alphabet_size
+        >
+    class conditioning_class_kernel_3 
+    {
+        /*!
+            INITIAL VALUE
+                total == 1
+                counts == pointer to an array of alphabet_size data structs                
+                for all i except i == 0: counts[i].count == 0
+                counts[0].count == 1
+                counts[0].symbol == alphabet_size-1
+                for all i except i == alphabet_size-1:  counts[i].present == false
+                counts[alphabet_size-1].present == true
+
+            CONVENTION
+                counts == pointer to an array of alphabet_size data structs
+                get_total() == total
+                get_count(symbol) == counts[x].count where 
+                                     counts[x].symbol == symbol
+
+
+                LOW_COUNT(symbol) == sum of counts[0].count though counts[x-1].count
+                                     where counts[x].symbol == symbol
+                                     if (counts[0].symbol == symbol) LOW_COUNT(symbol)==0
+
+
+                if (counts[i].count == 0) then
+                    counts[i].symbol == undefined value
+
+                if (symbol has a nonzero count) then
+                    counts[symbol].present == true
+
+                get_memory_usage() == global_state.memory_usage
+        !*/
+
+    public:
+
+        class global_state_type
+        {
+        public:
+            global_state_type () : memory_usage(0) {}
+        private:
+            unsigned long memory_usage;
+
+            friend class conditioning_class_kernel_3<alphabet_size>;
+        };
+
+        conditioning_class_kernel_3 (
+            global_state_type& global_state_
+        );
+
+        ~conditioning_class_kernel_3 (
+        );
+
+        void clear(
+        );
+
+        bool increment_count (
+            unsigned long symbol,
+            unsigned short amount = 1
+        );
+
+        unsigned long get_count (
+            unsigned long symbol
+        ) const;
+
+        unsigned long get_total (
+        ) const;
+        
+        unsigned long get_range (
+            unsigned long symbol,
+            unsigned long& low_count,
+            unsigned long& high_count,
+            unsigned long& total_count
+        ) const;
+
+        void get_symbol (
+            unsigned long target,
+            unsigned long& symbol,            
+            unsigned long& low_count,
+            unsigned long& high_count
+        ) const;
+
+        unsigned long get_memory_usage (
+        ) const;
+
+        global_state_type& get_global_state (
+        );
+
+        static unsigned long get_alphabet_size (
+        );
+
+    private:
+
+        // restricted functions
+        conditioning_class_kernel_3(conditioning_class_kernel_3<alphabet_size>&);        // copy constructor
+        conditioning_class_kernel_3& operator=(conditioning_class_kernel_3<alphabet_size>&);    // assignment operator
+
+        struct data
+        {
+            unsigned short count;
+            unsigned short symbol;
+            bool present;
+        };
+
+        // data members
+        unsigned short total;
+        data* counts;
+        global_state_type& global_state;
+
+    };   
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    conditioning_class_kernel_3<alphabet_size>::
+    conditioning_class_kernel_3 (
+        global_state_type& global_state_
+    ) :
+        total(1),
+        counts(new data[alphabet_size]),
+        global_state(global_state_)
+    {
+        COMPILE_TIME_ASSERT( 1 < alphabet_size && alphabet_size < 65536 );
+
+        data* start = counts;
+        data* end = counts+alphabet_size;
+        start->count = 1;
+        start->symbol = alphabet_size-1;
+        start->present = false;
+        ++start;
+        while (start != end)
+        {
+            start->count = 0;
+            start->present = false;
+            ++start;
+        }        
+        counts[alphabet_size-1].present = true;
+
+        // update memory usage
+        global_state.memory_usage += sizeof(data)*alphabet_size + 
+                                     sizeof(conditioning_class_kernel_3);
+
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    conditioning_class_kernel_3<alphabet_size>::
+    ~conditioning_class_kernel_3 (
+    )
+    {
+        delete [] counts;
+        // update memory usage
+        global_state.memory_usage -= sizeof(data)*alphabet_size + 
+                                     sizeof(conditioning_class_kernel_3);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    void conditioning_class_kernel_3<alphabet_size>::
+    clear(
+    )
+    {
+        total = 1;
+        data* start = counts;
+        data* end = counts+alphabet_size;
+        start->count = 1;
+        start->symbol = alphabet_size-1;
+        start->present = false;
+        ++start;
+        while (start != end)
+        {
+            start->count = 0;
+            start->present = false;
+            ++start;
+        }        
+        counts[alphabet_size-1].present = true; 
+
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    typename conditioning_class_kernel_3<alphabet_size>::global_state_type& conditioning_class_kernel_3<alphabet_size>::
+    get_global_state(
+    )
+    {
+        return global_state;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_3<alphabet_size>::
+    get_memory_usage(
+    ) const
+    {
+        return global_state.memory_usage;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    bool conditioning_class_kernel_3<alphabet_size>::
+    increment_count (
+        unsigned long symbol,
+        unsigned short amount
+    )
+    {        
+        // if we are going over a total of 65535 then scale down all counts by 2
+        if (static_cast<unsigned long>(total)+static_cast<unsigned long>(amount) >= 65536)
+        {
+            total = 0;
+            data* start = counts;
+            data* end = counts+alphabet_size;
+            
+            while (start != end)
+            {
+                if (start->count == 1)
+                {
+                    if (start->symbol == alphabet_size-1)
+                    {
+                        // this symbol must never be zero so we will leave its count at 1
+                        ++total;
+                    }
+                    else
+                    {
+                        start->count = 0;
+                        counts[start->symbol].present = false;
+                    }
+                }
+                else
+                {
+                    start->count >>= 1;
+                    total += start->count;
+                }
+ 
+                ++start;
+            }  
+        }
+
+
+        data* start = counts;   
+        data* swap_spot = counts;
+
+        if (counts[symbol].present)
+        {
+            while (true)
+            {                
+                if (start->symbol == symbol && start->count!=0)
+                {                
+                    unsigned short temp = start->count + amount;
+
+                    start->symbol = swap_spot->symbol;
+                    start->count = swap_spot->count;
+
+                    swap_spot->symbol = static_cast<unsigned short>(symbol);                
+                    swap_spot->count  = temp;
+                    break;
+                }
+                
+                if ( (start->count) < (swap_spot->count))
+                {
+                    swap_spot = start;
+                }
+
+
+                ++start;
+            }
+        }
+        else
+        {
+            counts[symbol].present = true;
+            while (true)
+            {
+                if (start->count == 0)
+                {
+                    start->symbol = swap_spot->symbol;
+                    start->count = swap_spot->count;
+                    
+                    swap_spot->symbol = static_cast<unsigned short>(symbol);                
+                    swap_spot->count  = amount;
+                    break;
+                }
+                    
+                if ((start->count) < (swap_spot->count))
+                {
+                    swap_spot = start;
+                }
+
+                ++start;
+            }
+        }
+        
+        total += amount;
+
+        return true;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_3<alphabet_size>::
+    get_count (
+        unsigned long symbol
+    ) const
+    {
+        if (counts[symbol].present == false)
+            return 0;
+
+        data* start = counts;        
+        while (start->symbol != symbol)
+        {
+            ++start;
+        }
+        return start->count;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_3<alphabet_size>::
+    get_alphabet_size (        
+    ) 
+    {
+        return alphabet_size;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_3<alphabet_size>::
+    get_total (
+    ) const
+    {
+        return total;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    unsigned long conditioning_class_kernel_3<alphabet_size>::
+    get_range (
+        unsigned long symbol,
+        unsigned long& low_count,
+        unsigned long& high_count,
+        unsigned long& total_count
+    ) const
+    {
+        if (counts[symbol].present == false)
+            return 0;
+
+        total_count = total;
+        unsigned long low_count_temp = 0;
+        data* start = counts;        
+        while (start->symbol != symbol)
+        {
+            low_count_temp += start->count;
+            ++start;
+        } 
+
+        low_count = low_count_temp;
+        high_count = low_count_temp + start->count;
+        return start->count;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size
+        >
+    void conditioning_class_kernel_3<alphabet_size>::
+    get_symbol (
+        unsigned long target,
+        unsigned long& symbol,            
+        unsigned long& low_count,
+        unsigned long& high_count
+    ) const
+    {
+        unsigned long high_count_temp = counts->count;
+        const data* start = counts;        
+        while (target >= high_count_temp)
+        {
+            ++start;
+            high_count_temp += start->count;
+        } 
+
+        low_count = high_count_temp - start->count;
+        high_count = high_count_temp;
+        symbol = static_cast<unsigned long>(start->symbol);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_CONDITIONING_CLASS_KERNEl_3_
+

dlib/conditioning_class/conditioning_class_kernel_4.h

+// Copyright (C) 2004  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_CONDITIONING_CLASS_KERNEl_4_
+#define DLIB_CONDITIONING_CLASS_KERNEl_4_
+
+#include "conditioning_class_kernel_abstract.h"
+#include "../assert.h"
+#include "../algs.h"
+
+namespace dlib
+{
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    class conditioning_class_kernel_4 
+    {
+        /*!
+            REQUIREMENTS ON pool_size
+                pool_size > 0
+                this will be the number of nodes contained in our memory pool
+
+            REQUIREMENTS ON mem_manager
+                mem_manager is an implementation of memory_manager/memory_manager_kernel_abstract.h
+
+            INITIAL VALUE
+                total == 1
+                escapes == 1
+                next == 0
+                
+            CONVENTION                
+                get_total() == total
+                get_count(alphabet_size-1) == escapes
+
+                if (next != 0) then
+                    next == pointer to the start of a linked list and the linked list
+                            is terminated by a node with a next pointer of 0.
+
+                get_count(symbol) == node::count for the node where node::symbol==symbol 
+                                     or 0 if no such node currently exists.
+
+                if (there is a node for the symbol) then
+                    LOW_COUNT(symbol) == the sum of all node's counts in the linked list
+                    up to but not including the node for the symbol.
+
+                get_memory_usage() == global_state.memory_usage
+        !*/
+
+
+        struct node
+        {
+            unsigned short symbol;
+            unsigned short count;
+            node* next;
+        };
+
+    public:
+
+        class global_state_type
+        {
+        public:
+            global_state_type (
+            ) : 
+                memory_usage(pool_size*sizeof(node)+sizeof(global_state_type))
+                {}
+        private:
+            unsigned long memory_usage;
+
+            typename mem_manager::template rebind<node>::other pool;
+
+            friend class conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>;
+        };
+
+        conditioning_class_kernel_4 (
+            global_state_type& global_state_
+        );
+
+        ~conditioning_class_kernel_4 (
+        );
+
+        void clear(
+        );
+
+        bool increment_count (
+            unsigned long symbol,
+            unsigned short amount = 1
+        );
+
+        unsigned long get_count (
+            unsigned long symbol
+        ) const;
+
+        inline unsigned long get_total (
+        ) const;
+        
+        unsigned long get_range (
+            unsigned long symbol,
+            unsigned long& low_count,
+            unsigned long& high_count,
+            unsigned long& total_count
+        ) const;
+
+        void get_symbol (
+            unsigned long target,
+            unsigned long& symbol,            
+            unsigned long& low_count,
+            unsigned long& high_count
+        ) const;
+
+        unsigned long get_memory_usage (
+        ) const;
+
+        global_state_type& get_global_state (
+        );
+
+        static unsigned long get_alphabet_size (
+        );
+
+
+    private:
+
+        void half_counts (
+        );
+        /*!
+            ensures
+                - divides all counts by 2 but ensures that escapes is always at least 1
+        !*/
+
+        // restricted functions
+        conditioning_class_kernel_4(conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>&);        // copy constructor
+        conditioning_class_kernel_4& operator=(conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>&);    // assignment operator
+
+        // data members
+        unsigned short total;
+        unsigned short escapes;
+        node* next;
+        global_state_type& global_state;
+
+    };   
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::
+    conditioning_class_kernel_4 (
+        global_state_type& global_state_
+    ) :
+        total(1),
+        escapes(1),
+        next(0),
+        global_state(global_state_)
+    {
+        COMPILE_TIME_ASSERT( 1 < alphabet_size && alphabet_size < 65536 );
+
+        // update memory usage
+        global_state.memory_usage += sizeof(conditioning_class_kernel_4);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::
+    ~conditioning_class_kernel_4 (
+    )
+    {
+        clear();
+        // update memory usage
+        global_state.memory_usage -= sizeof(conditioning_class_kernel_4);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    void conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::
+    clear(
+    )
+    {
+        total = 1;
+        escapes = 1;
+        while (next)
+        {
+            node* temp = next;
+            next = next->next;
+            global_state.pool.deallocate(temp);
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    unsigned long conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::
+    get_memory_usage(
+    ) const
+    {
+        return global_state.memory_usage;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    typename conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::global_state_type& conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::
+    get_global_state(
+    )
+    {
+        return global_state;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    bool conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::
+    increment_count (
+        unsigned long symbol,
+        unsigned short amount
+    )
+    {        
+        if (symbol == alphabet_size-1)
+        {
+            // make sure we won't cause any overflow
+            if (total >= 65536 - amount )                        
+                half_counts();
+
+            escapes += amount;
+            total += amount;
+            return true;
+        }
+
+        
+        // find the symbol and increment it or add a new node to the list
+        if (next)
+        {
+            node* temp = next;
+            node* previous = 0;
+            while (true)
+            {
+                if (temp->symbol == static_cast<unsigned short>(symbol))
+                {
+                    // make sure we won't cause any overflow
+                    if (total >= 65536 - amount )                        
+                        half_counts();
+                    
+                    // we have found the symbol
+                    total += amount;
+                    temp->count += amount;
+
+                    // if this node now has a count greater than its parent node
+                    if (previous && temp->count > previous->count)
+                    {
+                        // swap the nodes so that the nodes will be in semi-sorted order
+                        swap(temp->count,previous->count);
+                        swap(temp->symbol,previous->symbol);
+                    }
+                    return true;
+                }
+                else if (temp->next == 0)
+                {
+                    // we did not find the symbol so try to add it to the list
+                    if (global_state.pool.get_number_of_allocations() < pool_size)
+                    {
+                        // make sure we won't cause any overflow
+                        if (total >= 65536 - amount )                        
+                            half_counts();
+
+                        node* t = global_state.pool.allocate();
+                        t->next = 0;
+                        t->symbol = static_cast<unsigned short>(symbol);
+                        t->count = amount;
+                        temp->next = t;
+                        total += amount;
+                        return true;
+                    }
+                    else
+                    {
+                        // no memory left
+                        return false;
+                    }
+                }
+                else if (temp->count == 0)
+                {
+                    // remove nodes that have a zero count
+                    if (previous)
+                    {
+                        previous->next = temp->next;
+                        node* t = temp;
+                        temp = temp->next;
+                        global_state.pool.deallocate(t);
+                    }
+                    else
+                    {
+                        next = temp->next;
+                        node* t = temp;
+                        temp = temp->next;
+                        global_state.pool.deallocate(t);
+                    }
+                }
+                else
+                {
+                    previous = temp;
+                    temp = temp->next;
+                }
+            } // while (true)
+        }
+        // if there aren't any nodes in the list yet then do this instead
+        else
+        {
+            if (global_state.pool.get_number_of_allocations() < pool_size)
+            {
+                // make sure we won't cause any overflow
+                if (total >= 65536 - amount )                        
+                    half_counts();
+
+                next = global_state.pool.allocate();
+                next->next = 0;
+                next->symbol = static_cast<unsigned short>(symbol);
+                next->count = amount;
+                total += amount;
+                return true;
+            }
+            else
+            {
+                // no memory left
+                return false;
+            }
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    unsigned long conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::
+    get_count (
+        unsigned long symbol
+    ) const
+    {
+        if (symbol == alphabet_size-1)
+        { 
+            return escapes;
+        }
+        else
+        {
+            node* temp = next;
+            while (temp)
+            {
+                if (temp->symbol == symbol)
+                    return temp->count;
+                temp = temp->next;
+            }
+            return 0;
+        }        
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    unsigned long conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::
+    get_alphabet_size (        
+    ) 
+    {
+        return alphabet_size;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    unsigned long conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::
+    get_total (
+    ) const
+    {
+        return total;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    unsigned long conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::
+    get_range (
+        unsigned long symbol,
+        unsigned long& low_count,
+        unsigned long& high_count,
+        unsigned long& total_count
+    ) const
+    {   
+        if (symbol != alphabet_size-1)
+        {
+            node* temp = next;
+            unsigned long low = 0;
+            while (temp)
+            {
+                if (temp->symbol == static_cast<unsigned short>(symbol))
+                {
+                    high_count = temp->count + low;
+                    low_count = low;                
+                    total_count = total;
+                    return temp->count;
+                }
+                low += temp->count;
+                temp = temp->next;
+            }
+            return 0;
+        }
+        else
+        {
+            total_count = total;
+            high_count = total;
+            low_count = total-escapes;
+            return escapes;
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    void conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::
+    get_symbol (
+        unsigned long target,
+        unsigned long& symbol,            
+        unsigned long& low_count,
+        unsigned long& high_count
+    ) const
+    {
+        node* temp = next;
+        unsigned long high = 0;
+        while (true)
+        {
+            if (temp != 0)
+            {
+                high += temp->count;
+                if (target < high)
+                {
+                    symbol = temp->symbol;
+                    high_count = high;
+                    low_count = high - temp->count;
+                    return;
+                }
+                temp = temp->next;
+            }
+            else
+            {
+                // this must be the escape symbol
+                symbol = alphabet_size-1;
+                low_count = total-escapes;
+                high_count = total;
+                return;
+            }            
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // private member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+
+    template <
+        unsigned long alphabet_size,
+        unsigned long pool_size,
+        typename mem_manager
+        >
+    void conditioning_class_kernel_4<alphabet_size,pool_size,mem_manager>::
+    half_counts (
+    ) 
+    {
+        total = 0;
+        if (escapes > 1)
+            escapes >>= 1;
+
+        //divide all counts by 2
+        node* temp = next;
+        while (temp)
+        {
+            temp->count >>= 1;
+            total += temp->count;
+            temp = temp->next;
+        }
+        total += escapes;
+    }
+
+// ----------------------------------------------------------------------------------------
+ 
+}
+
+#endif // DLIB_CONDITIONING_CLASS_KERNEl_4_
+

dlib/conditioning_class/conditioning_class_kernel_abstract.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_CONDITIONING_CLASS_KERNEl_ABSTRACT_
+#ifdef DLIB_CONDITIONING_CLASS_KERNEl_ABSTRACT_
+
+#include "../algs.h"
+
+namespace dlib
+{
+
+    template <
+        unsigned long alphabet_size
+        >
+    class conditioning_class 
+    {
+        /*!
+            REQUIREMENTS ON alphabet_size
+                1 < alphabet_size < 65536
+
+            INITIAL VALUE
+                get_total() == 1
+                get_count(X) == 0 : for all valid values of X except alphabet_size-1
+                get_count(alphabet_size-1) == 1
+
+            WHAT THIS OBJECT REPRESENTS
+                This object represents a conditioning class used for arithmetic style
+                compression.  It maintains the cumulative counts which are needed
+                by the entropy_coder and entropy_decoder objects.
+
+                At any moment a conditioning_class object represents a set of 
+                alphabet_size symbols.  Each symbol is associated with an integer 
+                called its count.  
+
+                All symbols start out with a count of zero except for alphabet_size-1.
+                This last symbol will always have a count of at least one.  It is
+                intended to be used as an escape into a lower context when coding
+                and so it must never have a zero probability or the decoder won't
+                be able to identify the escape symbol.
+
+            NOTATION:
+                Let MAP(i) be a function which maps integers to symbols.  MAP(i) is
+                one to one and onto.  Its domain is 1 to alphabet_size inclusive.
+               
+                Let RMAP(s) be the inverse of MAP(i).  
+                 ( i.e.  RMAP(MAP(i)) == i  and  MAP(RMAP(s)) == s  )
+
+                Let COUNT(i) give the count for the symbol MAP(i).  
+                 ( i.e.  COUNT(i) == get_count(MAP(i)) )
+              
+
+                Let LOW_COUNT(s) == the sum of COUNT(x) for x == 1 to x == RMAP(s)-1
+                  (note that the sum of COUNT(x) for x == 1 to x == 0 is 0)
+                Let HIGH_COUNT(s) == LOW_COUNT(s) + get_count(s)    
+
+
+
+                Basically what this is saying is just that you shoudln't assume you know
+                what order the symbols are placed in when calculating the cumulative
+                sums.  The specific mapping provided by the MAP() function is unspecified.  
+
+            THREAD SAFETY
+                This object can be used safely in a multithreaded program as long as the 
+                global state is not shared between conditioning classes which run on 
+                different threads.  
+
+            GLOBAL_STATE_TYPE
+                The global_state_type obejct allows instances of the conditioning_class
+                object to share any kind of global state the implementer desires. 
+                However, the global_state_type object exists primarily to facilitate the 
+                sharing of a memory pool between many instances of a conditioning_class 
+                object.  But note that it is not required that there be any kind of 
+                memory pool at all, it is just a possibility.
+        !*/
+
+    public:
+
+        class global_state_type 
+        {
+            global_state_type (
+            );
+            /*!
+                ensures
+                    - #*this is properly initialized
+                throws
+                    - std::bad_alloc
+            !*/
+
+            // my contents are implementation specific.               
+        };
+
+        conditioning_class (
+            global_state_type& global_state
+        );
+        /*!
+            ensures
+                - #*this is properly initialized
+                - &#get_global_state() == &global_state
+            throws
+                - std::bad_alloc
+        !*/
+
+        ~conditioning_class (
+        );
+        /*!
+            ensures
+                - all memory associated with *this has been released
+        !*/
+
+        void clear(
+        );
+        /*!
+            ensures
+                - #*this has its initial value
+            throws
+                - std::bad_alloc
+        !*/
+
+        bool increment_count (
+            unsigned long symbol,
+            unsigned short amount = 1
+        );
+        /*!
+            requires
+                - 0 <= symbol < alphabet_size
+                - 0 < amount < 32768
+            ensures
+                - if (sufficient memory is available to complete this operation) then
+                    - returns true
+                    - if (get_total()+amount < 65536) then 
+                        - #get_count(symbol) == get_count(symbol) + amount
+                    - else
+                        - #get_count(symbol) == get_count(symbol)/2 + amount
+                        - if (get_count(alphabet_size-1) == 1) then
+                            - #get_count(alphabet_size-1) == 1
+                        - else
+                            - #get_count(alphabet_size-1) == get_count(alphabet_size-1)/2
+                        - for all X where (X != symbol)&&(X != alpahbet_size-1): 
+                          #get_count(X) == get_count(X)/2  
+                - else
+                    - returns false
+        !*/
+
+        unsigned long get_count (
+            unsigned long symbol
+        ) const;
+        /*!
+            requires 
+                - 0 <= symbol < alphabet_size
+            ensures
+                - returns the count for the specified symbol
+        !*/
+
+        unsigned long get_total (
+        ) const;
+        /*!
+            ensures
+                - returns the sum of get_count(X) for all valid values of X
+                  (i.e. returns the sum of the counts for all the symbols)
+        !*/
+
+        unsigned long get_range (
+            unsigned long symbol,
+            unsigned long& low_count,
+            unsigned long& high_count,
+            unsigned long& total_count
+        ) const;
+        /*!
+            requires
+                - 0 <= symbol < alphabet_size
+            ensures                
+                - returns get_count(symbol)
+                - if (get_count(symbol) != 0) then
+                    - #total_count == get_total()
+                    - #low_count   == LOW_COUNT(symbol)
+                    - #high_count  == HIGH_COUNT(symbol)
+                    - #low_count < #high_count <= #total_count                
+        !*/
+
+        void get_symbol (
+            unsigned long target,
+            unsigned long& symbol,            
+            unsigned long& low_count,
+            unsigned long& high_count
+        ) const;
+        /*!
+            requires
+                - 0 <= target < get_total()
+            ensures
+                - LOW_COUNT(#symbol) <= target < HIGH_COUNT(#symbol)
+                - #low_count   == LOW_COUNT(#symbol)
+                - #high_count  == HIGH_COUNT(#symbol)
+                - #low_count < #high_count <= get_total()
+        !*/
+
+        global_state_type& get_global_state (
+        );
+        /*! 
+            ensures
+                - returns a reference to the global state used by *this
+        !*/
+
+        unsigned long get_memory_usage (
+        ) const;
+        /*!
+            ensures 
+                - returns the number of bytes of memory allocated by all conditioning_class
+                  objects that share the global state given by get_global_state()
+        !*/
+
+        static unsigned long get_alphabet_size (
+        );
+        /*!
+            ensures
+                - returns alphabet_size
+        !*/
+
+    private:
+
+        // restricted functions
+        conditioning_class(conditioning_class<alphabet_size>&);        // copy constructor
+        conditioning_class<alphabet_size>& operator=(conditioning_class<alphabet_size>&);    // assignment operator
+
+    };   
+
+}
+
+#endif // DLIB_CONDITIONING_CLASS_KERNEl_ABSTRACT_
+

dlib/conditioning_class/conditioning_class_kernel_c.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_CONDITIONING_CLASS_KERNEl_C_
+#define DLIB_CONDITIONING_CLASS_KERNEl_C_
+
+#include "conditioning_class_kernel_abstract.h"
+#include "../algs.h"
+#include "../assert.h"
+#include <iostream>
+
+namespace dlib
+{
+
+    template <
+        typename cc_base
+        >
+    class conditioning_class_kernel_c : public cc_base
+    {
+        const unsigned long alphabet_size;
+
+    public:
+
+        conditioning_class_kernel_c (
+            typename cc_base::global_state_type& global_state
+            ) : cc_base(global_state),alphabet_size(cc_base::get_alphabet_size()) {}
+
+        bool increment_count (
+            unsigned long symbol,
+            unsigned short amount = 1
+        );
+
+        unsigned long get_count (
+            unsigned long symbol
+        ) const;
+
+        unsigned long get_range (
+            unsigned long symbol,
+            unsigned long& low_count,
+            unsigned long& high_count,
+            unsigned long& total_count
+        ) const;
+
+        void get_symbol (
+            unsigned long target,
+            unsigned long& symbol,            
+            unsigned long& low_count,
+            unsigned long& high_count
+        ) const;
+    };
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+// member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename cc_base
+        >
+    bool conditioning_class_kernel_c<cc_base>::
+    increment_count (
+        unsigned long symbol,
+        unsigned short amount
+    )
+    {
+        // make sure requires clause is not broken
+        DLIB_CASSERT(symbol < alphabet_size &&
+                0 < amount && amount < 32768,
+                "\tvoid conditioning_class::increment_count()"
+                << "\n\tthe symbol must be in the range 0 to alphabet_size-1. and"
+                << "\n\tamount must be in the range 1 to 32767"
+                << "\n\talphabet_size: " << alphabet_size
+                << "\n\tsymbol:        " << symbol
+                << "\n\tamount:        " << amount
+                << "\n\tthis:          " << this
+        );
+
+        // call the real function
+        return cc_base::increment_count(symbol,amount);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename cc_base
+        >
+    unsigned long conditioning_class_kernel_c<cc_base>::
+    get_count (
+        unsigned long symbol
+    ) const
+    {
+        // make sure requires clause is not broken
+        DLIB_CASSERT(symbol < alphabet_size,
+                "\tvoid conditioning_class::get_count()"
+                << "\n\tthe symbol must be in the range 0 to alphabet_size-1"
+                << "\n\talphabet_size: " << alphabet_size
+                << "\n\tsymbol:        " << symbol
+                << "\n\tthis:          " << this
+        );
+
+        // call the real function
+        return cc_base::get_count(symbol);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename cc_base
+        >
+    unsigned long conditioning_class_kernel_c<cc_base>::
+    get_range (
+        unsigned long symbol,
+        unsigned long& low_count,
+        unsigned long& high_count,
+        unsigned long& total_count
+    ) const
+    {
+        // make sure requires clause is not broken
+        DLIB_CASSERT(symbol < alphabet_size,
+                "\tvoid conditioning_class::get_range()"
+                << "\n\tthe symbol must be in the range 0 to alphabet_size-1"
+                << "\n\talphabet_size: " << alphabet_size
+                << "\n\tsymbol:        " << symbol
+                << "\n\tthis:          " << this
+        );
+
+        // call the real function
+        return cc_base::get_range(symbol,low_count,high_count,total_count);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename cc_base
+        >
+    void conditioning_class_kernel_c<cc_base>::
+    get_symbol (
+        unsigned long target,
+        unsigned long& symbol,            
+        unsigned long& low_count,
+        unsigned long& high_count
+    ) const
+    {
+        // make sure requires clause is not broken
+        DLIB_CASSERT( target < this->get_total(),
+                 "\tvoid conditioning_class::get_symbol()"
+                 << "\n\tthe target must be in the range 0 to get_total()-1"
+                 << "\n\tget_total(): " << this->get_total()
+                 << "\n\ttarget:       " << target
+                 << "\n\tthis:         " << this
+        );
+
+        // call the real function
+        cc_base::get_symbol(target,symbol,low_count,high_count);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_CONDITIONING_CLASS_KERNEl_C_
+

dlib/config.h

+
+
+// If you are compiling dlib as a shared library and installing it somewhere on your system
+// then it is important that any programs that use dlib agree on the state of the
+// DLIB_ASSERT statements (i.e. they are either always on or always off).  Therefore,
+// uncomment one of the following lines to force all DLIB_ASSERTs to either always on or
+// always off.  If you don't define one of these two macros then DLIB_ASSERT will toggle
+// automatically depending on the state of certain other macros, which is not what you want
+// when creating a shared library.
+//#define ENABLE_ASSERTS       // asserts always enabled 
+//#define DLIB_DISABLE_ASSERTS // asserts always disabled 
+
+//#define DLIB_ISO_CPP_ONLY
+//#define DLIB_NO_GUI_SUPPORT
+//#define DLIB_ENABLE_STACK_TRACE
+
+// You should also consider telling dlib to link against libjpeg, libpng, libgif, fftw, CUDA, 
+// and a BLAS and LAPACK library.  To do this you need to uncomment the following #defines.
+// #define DLIB_JPEG_SUPPORT
+// #define DLIB_PNG_SUPPORT
+// #define DLIB_GIF_SUPPORT
+// #define DLIB_USE_FFTW
+// #define DLIB_USE_BLAS
+// #define DLIB_USE_LAPACK
+// #define DLIB_USE_CUDA
+
+
+// Define this so the code in dlib/test_for_odr_violations.h can detect ODR violations
+// related to users doing bad things with config.h
+#define DLIB_NOT_CONFIGURED
+

dlib/config.h.in

+
+
+// If you are compiling dlib as a shared library and installing it somewhere on your system
+// then it is important that any programs that use dlib agree on the state of the
+// DLIB_ASSERT statements (i.e. they are either always on or always off).  Therefore,
+// uncomment one of the following lines to force all DLIB_ASSERTs to either always on or
+// always off.  If you don't define one of these two macros then DLIB_ASSERT will toggle
+// automatically depending on the state of certain other macros, which is not what you want
+// when creating a shared library.
+#cmakedefine ENABLE_ASSERTS       // asserts always enabled 
+#cmakedefine DLIB_DISABLE_ASSERTS // asserts always disabled 
+
+#cmakedefine DLIB_ISO_CPP_ONLY
+#cmakedefine DLIB_NO_GUI_SUPPORT
+#cmakedefine DLIB_ENABLE_STACK_TRACE
+
+#cmakedefine LAPACK_FORCE_UNDERSCORE
+#cmakedefine LAPACK_FORCE_NOUNDERSCORE
+
+// You should also consider telling dlib to link against libjpeg, libpng, libgif, fftw, CUDA, 
+// and a BLAS and LAPACK library.  To do this you need to uncomment the following #defines.
+#cmakedefine DLIB_JPEG_SUPPORT
+#cmakedefine DLIB_WEBP_SUPPORT
+#cmakedefine DLIB_PNG_SUPPORT
+#cmakedefine DLIB_GIF_SUPPORT
+#cmakedefine DLIB_USE_FFTW
+#cmakedefine DLIB_USE_BLAS
+#cmakedefine DLIB_USE_LAPACK
+#cmakedefine DLIB_USE_CUDA
+#cmakedefine DLIB_USE_MKL_FFT
+#cmakedefine DLIB_USE_FFMPEG
+
+// This variable allows dlib/test_for_odr_violations.h to catch people who mistakenly use
+// headers from one version of dlib with a compiled dlib binary from a different dlib version.
+#cmakedefine DLIB_CHECK_FOR_VERSION_MISMATCH @DLIB_CHECK_FOR_VERSION_MISMATCH@
+

dlib/config_reader.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_CONFIG_READEr_
+#define DLIB_CONFIG_READEr_
+
+#include "config_reader/config_reader_kernel_1.h"
+#include "map.h"
+#include "tokenizer.h"
+#include "cmd_line_parser/get_option.h"
+
+#include "algs.h"
+#include "is_kind.h"
+
+
+namespace dlib
+{
+
+    typedef config_reader_kernel_1<
+        map<std::string,std::string>::kernel_1b,
+        map<std::string,void*>::kernel_1b,
+        tokenizer::kernel_1a
+        > config_reader;
+
+    template <> struct is_config_reader<config_reader> { const static bool value = true; };
+
+#ifndef DLIB_ISO_CPP_ONLY
+    typedef config_reader_thread_safe_1<
+        config_reader,
+        map<std::string,void*>::kernel_1b
+        > config_reader_thread_safe;
+
+    template <> struct is_config_reader<config_reader_thread_safe> { const static bool value = true; };
+#endif // DLIB_ISO_CPP_ONLY
+
+
+}
+
+#endif // DLIB_CONFIG_READEr_
+

dlib/config_reader/config_reader_kernel_1.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_CONFIG_READER_KERNEl_1_
+#define DLIB_CONFIG_READER_KERNEl_1_
+
+#include "config_reader_kernel_abstract.h"
+#include <string>
+#include <iostream>
+#include <sstream>
+#include <fstream>
+#include "../algs.h"
+#include "../stl_checked/std_vector_c.h"
+
+#ifndef DLIB_ISO_CPP_ONLY
+#include "config_reader_thread_safe_1.h"
+#endif
+
+namespace dlib
+{
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    class config_reader_kernel_1 
+    {
+
+        /*!                
+            REQUIREMENTS ON map_string_string
+                is an implementation of map/map_kernel_abstract.h that maps std::string to std::string
+
+            REQUIREMENTS ON map_string_void 
+                is an implementation of map/map_kernel_abstract.h that maps std::string to void*
+
+            REQUIREMENTS ON tokenizer
+                is an implementation of tokenizer/tokenizer_kernel_abstract.h 
+
+            CONVENTION
+                key_table.is_in_domain(x) == is_key_defined(x)
+                block_table.is_in_domain(x) == is_block_defined(x)
+
+                key_table[x] == operator[](x)
+                block_table[x] == (void*)&block(x)
+        !*/
+        
+    public:
+
+        // These two typedefs are defined for backwards compatibility with older versions of dlib.
+        typedef config_reader_kernel_1 kernel_1a;
+#ifndef DLIB_ISO_CPP_ONLY
+        typedef config_reader_thread_safe_1<
+            config_reader_kernel_1,
+            map_string_void 
+            > thread_safe_1a;
+#endif // DLIB_ISO_CPP_ONLY
+
+
+        config_reader_kernel_1();
+
+        class config_reader_error : public dlib::error 
+        {
+            friend class config_reader_kernel_1;
+            config_reader_error(
+                unsigned long ln, 
+                bool r = false
+            ) : 
+                dlib::error(ECONFIG_READER),
+                line_number(ln), 
+                redefinition(r)
+            {
+                std::ostringstream sout;
+                sout << "Error in config_reader while parsing at line number " << line_number << ".";
+                if (redefinition)
+                    sout << "\nThe identifier on this line has already been defined in this scope.";
+                const_cast<std::string&>(info) = sout.str();
+            }
+        public:
+            const unsigned long line_number;
+            const bool redefinition;
+        };
+
+        class file_not_found : public dlib::error 
+        {
+            friend class config_reader_kernel_1;
+            file_not_found(
+                const std::string& file_name_
+            ) : 
+                dlib::error(ECONFIG_READER, "Error in config_reader, unable to open file " + file_name_),
+                file_name(file_name_)
+            {}
+
+            ~file_not_found() throw() {}
+
+        public:
+            const std::string file_name;
+        };
+
+        class config_reader_access_error : public dlib::error
+        {
+        public:
+            config_reader_access_error(
+                const std::string& block_name_,
+                const std::string& key_name_
+            ) : 
+                dlib::error(ECONFIG_READER),
+                block_name(block_name_), 
+                key_name(key_name_)
+            {
+                std::ostringstream sout;
+                sout << "Error in config_reader.\n";
+                if (block_name.size() > 0)
+                    sout << "   A block with the name '" << block_name << "' was expected but not found.";
+                else if (key_name.size() > 0)
+                    sout << "   A key with the name '" << key_name << "' was expected but not found.";
+
+                const_cast<std::string&>(info) = sout.str();
+            }
+
+            ~config_reader_access_error() throw() {}
+            const std::string block_name;
+            const std::string key_name;
+        };
+
+        config_reader_kernel_1(
+            const std::string& config_file 
+        );
+
+        config_reader_kernel_1(
+            std::istream& in
+        );
+
+        virtual ~config_reader_kernel_1(
+        ); 
+
+        void clear (
+        );
+
+        void load_from (
+            std::istream& in
+        );
+
+        void load_from (
+            const std::string& config_file
+        );
+
+        bool is_key_defined (
+            const std::string& key
+        ) const;
+
+        bool is_block_defined (
+            const std::string& name
+        ) const;
+
+        typedef config_reader_kernel_1 this_type;
+        const this_type& block (
+            const std::string& name
+        ) const;
+
+        const std::string& operator[] (
+            const std::string& key
+        ) const;
+
+        template <
+            typename queue_of_strings
+            >
+        void get_keys (
+            queue_of_strings& keys
+        ) const;
+
+        template <
+            typename alloc 
+            >
+        void get_keys (
+            std::vector<std::string,alloc>& keys
+        ) const;
+
+        template <
+            typename alloc 
+            >
+        void get_keys (
+            std_vector_c<std::string,alloc>& keys
+        ) const;
+
+        template <
+            typename queue_of_strings
+            >
+        void get_blocks (
+            queue_of_strings& blocks
+        ) const;
+
+        template <
+            typename alloc 
+            >
+        void get_blocks (
+            std::vector<std::string,alloc>& blocks
+        ) const;
+
+        template <
+            typename alloc 
+            >
+        void get_blocks (
+            std_vector_c<std::string,alloc>& blocks
+        ) const;
+
+    private:
+
+        static void parse_config_file (
+            config_reader_kernel_1& cr,
+            tokenizer& tok,
+            unsigned long& line_number,
+            const bool top_of_recursion = true
+        );
+        /*!
+            requires
+                - line_number == 1
+                - cr == *this
+                - top_of_recursion == true
+            ensures
+                - parses the data coming from tok and puts it into cr.
+            throws
+                - config_reader_error
+        !*/
+
+        map_string_string key_table;
+        map_string_void block_table;
+
+        // restricted functions
+        config_reader_kernel_1(config_reader_kernel_1&);     
+        config_reader_kernel_1& operator=(config_reader_kernel_1&);
+
+    };
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+    // member function definitions
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    config_reader_kernel_1(
+    )
+    {
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    void config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    clear(
+    )
+    {
+        // free all our blocks
+        block_table.reset();
+        while (block_table.move_next())
+        {
+            delete static_cast<config_reader_kernel_1*>(block_table.element().value());
+        }
+        block_table.clear();
+        key_table.clear();
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    void config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    load_from(
+        std::istream& in
+    )
+    {
+        clear();
+
+        tokenizer tok;
+        tok.set_stream(in);
+        tok.set_identifier_token(
+            tok.lowercase_letters() + tok.uppercase_letters(),
+            tok.lowercase_letters() + tok.uppercase_letters() + tok.numbers() + "_-."
+        );
+
+        unsigned long line_number = 1;
+        try
+        {
+            parse_config_file(*this,tok,line_number);
+        }
+        catch (...)
+        {
+            clear();
+            throw;
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    void config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    load_from(
+        const std::string& config_file
+    )
+    {
+        clear();
+        std::ifstream fin(config_file.c_str());
+        if (!fin)
+            throw file_not_found(config_file);
+
+        load_from(fin);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    config_reader_kernel_1(
+        std::istream& in
+    )
+    {
+        load_from(in);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    config_reader_kernel_1(
+        const std::string& config_file
+    )
+    {
+        load_from(config_file);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    void config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    parse_config_file(
+        config_reader_kernel_1<map_string_string,map_string_void,tokenizer>& cr,
+        tokenizer& tok,
+        unsigned long& line_number,
+        const bool top_of_recursion
+    )
+    {
+        int type;
+        std::string token;
+        bool in_comment = false;
+        bool seen_identifier = false;
+        std::string identifier;
+        while (true)
+        {
+            tok.get_token(type,token);
+            // ignore white space
+            if (type == tokenizer::WHITE_SPACE)
+                continue;
+
+            // basically ignore end of lines
+            if (type == tokenizer::END_OF_LINE)
+            {
+                ++line_number;
+                in_comment = false;
+                continue;
+            }
+
+            // we are in a comment still so ignore this
+            if (in_comment)
+                continue;
+
+            // if this is the start of a comment
+            if (type == tokenizer::CHAR && token[0] == '#')
+            {
+                in_comment = true;
+                continue;
+            }
+
+            // if this is the case then we have just finished parsing a block so we should
+            // quit this function
+            if ( (type == tokenizer::CHAR && token[0] == '}' && !top_of_recursion) ||
+                 (type == tokenizer::END_OF_FILE && top_of_recursion) )
+            {
+                break;
+            }
+
+            if (seen_identifier)
+            {
+                seen_identifier = false;
+                // the next character should be either a '=' or a '{'
+                if (type != tokenizer::CHAR || (token[0] != '=' && token[0] != '{'))
+                    throw config_reader_error(line_number);
+                
+                if (token[0] == '=')
+                {
+                    // we should parse the value out now
+                    // first discard any white space
+                    if (tok.peek_type() == tokenizer::WHITE_SPACE)
+                        tok.get_token(type,token);
+
+                    std::string value;
+                    type = tok.peek_type();
+                    token = tok.peek_token();
+                    while (true)
+                    {
+                        if (type == tokenizer::END_OF_FILE || type == tokenizer::END_OF_LINE)
+                            break;
+
+                        if (type == tokenizer::CHAR && token[0] == '\\')
+                        {
+                            tok.get_token(type,token);
+                            if (tok.peek_type() == tokenizer::CHAR && 
+                                tok.peek_token()[0] == '#')
+                            {
+                                tok.get_token(type,token);
+                                value += '#';
+                            }
+                            else if (tok.peek_type() == tokenizer::CHAR && 
+                                tok.peek_token()[0] == '}')
+                            {
+                                tok.get_token(type,token);
+                                value += '}';
+                            }
+                            else
+                            {
+                                value += '\\';
+                            }
+                        }
+                        else if (type == tokenizer::CHAR && 
+                                 (token[0] == '#' || token[0] == '}'))
+                        {
+                            break;
+                        }
+                        else
+                        {
+                            value += token;
+                            tok.get_token(type,token);
+                        }
+                        type = tok.peek_type();
+                        token = tok.peek_token();
+                    } // while(true)
+
+                    // strip of any tailing white space from value
+                    std::string::size_type pos = value.find_last_not_of(" \t\r\n");
+                    if (pos == std::string::npos)
+                        value.clear();
+                    else
+                        value.erase(pos+1);
+
+                    // make sure this key isn't already in the key_table
+                    if (cr.key_table.is_in_domain(identifier))
+                        throw config_reader_error(line_number,true);
+
+                    // add this key/value pair to the key_table
+                    cr.key_table.add(identifier,value);
+
+                }
+                else  // when token[0] == '{'
+                {
+                    // make sure this identifier isn't already in the block_table
+                    if (cr.block_table.is_in_domain(identifier))
+                        throw config_reader_error(line_number,true);
+
+                    config_reader_kernel_1* new_cr = new config_reader_kernel_1;
+                    void* vtemp = new_cr;
+                    try { cr.block_table.add(identifier,vtemp); }
+                    catch (...) { delete new_cr; throw; }
+
+                    // now parse this block 
+                    parse_config_file(*new_cr,tok,line_number,false);
+                }
+            }
+            else
+            {
+                // the next thing should be an identifier but if it isn't this is an error
+                if (type != tokenizer::IDENTIFIER)
+                    throw config_reader_error(line_number);
+
+                seen_identifier = true;
+                identifier = token;
+            }
+        } // while (true) 
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    ~config_reader_kernel_1(
+    ) 
+    {
+        clear();
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    bool config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    is_key_defined (
+        const std::string& key
+    ) const
+    {
+        return key_table.is_in_domain(key);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    bool config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    is_block_defined (
+        const std::string& name
+    ) const
+    {
+        return block_table.is_in_domain(name);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename mss,
+        typename msv,
+        typename tokenizer
+        >
+    const config_reader_kernel_1<mss,msv,tokenizer>& config_reader_kernel_1<mss,msv,tokenizer>::
+    block (
+        const std::string& name
+    ) const
+    {
+        if (is_block_defined(name) == false)
+        {
+            throw config_reader_access_error(name,"");
+        }
+
+        return *static_cast<config_reader_kernel_1*>(block_table[name]);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    const std::string& config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    operator[] (
+        const std::string& key
+    ) const
+    {
+        if (is_key_defined(key) == false)
+        {
+            throw config_reader_access_error("",key);
+        }
+
+        return key_table[key];
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    template <
+        typename queue_of_strings
+        >
+    void config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    get_keys (
+        queue_of_strings& keys
+    ) const
+    {
+        keys.clear();
+        key_table.reset();
+        std::string temp;
+        while (key_table.move_next())
+        {
+            temp = key_table.element().key();
+            keys.enqueue(temp);
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    template <
+        typename alloc 
+        >
+    void config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    get_keys (
+        std::vector<std::string,alloc>& keys
+    ) const
+    {
+        keys.clear();
+        key_table.reset();
+        while (key_table.move_next())
+        {
+            keys.push_back(key_table.element().key());
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    template <
+        typename alloc 
+        >
+    void config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    get_keys (
+        std_vector_c<std::string,alloc>& keys
+    ) const
+    {
+        keys.clear();
+        key_table.reset();
+        while (key_table.move_next())
+        {
+            keys.push_back(key_table.element().key());
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    template <
+        typename queue_of_strings
+        >
+    void config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    get_blocks (
+        queue_of_strings& blocks
+    ) const
+    {
+        blocks.clear();
+        block_table.reset();
+        std::string temp;
+        while (block_table.move_next())
+        {
+            temp = block_table.element().key();
+            blocks.enqueue(temp);
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    template <
+        typename alloc 
+        >
+    void config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    get_blocks (
+        std::vector<std::string,alloc>& blocks
+    ) const
+    {
+        blocks.clear();
+        block_table.reset();
+        while (block_table.move_next())
+        {
+            blocks.push_back(block_table.element().key());
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename map_string_string,
+        typename map_string_void,
+        typename tokenizer
+        >
+    template <
+        typename alloc 
+        >
+    void config_reader_kernel_1<map_string_string,map_string_void,tokenizer>::
+    get_blocks (
+        std_vector_c<std::string,alloc>& blocks
+    ) const
+    {
+        blocks.clear();
+        block_table.reset();
+        while (block_table.move_next())
+        {
+            blocks.push_back(block_table.element().key());
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_CONFIG_READER_KERNEl_1_
+

dlib/config_reader/config_reader_kernel_abstract.h

+// Copyright (C) 2003  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_CONFIG_READER_KERNEl_ABSTRACT_
+#ifdef DLIB_CONFIG_READER_KERNEl_ABSTRACT_
+
+#include <string>
+#include <iosfwd>
+
+namespace dlib
+{
+
+    class config_reader 
+    {
+
+        /*!                
+            INITIAL VALUE
+                - there aren't any keys defined for this object
+                - there aren't any blocks defined for this object
+
+            POINTERS AND REFERENCES TO INTERNAL DATA
+                The destructor, clear(), and load_from() invalidate pointers
+                and references to internal data.  All other functions are guaranteed
+                to NOT invalidate pointers or references to internal data.
+
+            WHAT THIS OBJECT REPRESENTS
+                This object represents something which is intended to be used to read
+                text configuration files that are defined by the following EBNF (with
+                config_file as the starting symbol):
+            
+                config_file    = block;
+                block          = { key_value_pair | sub_block };
+                key_value_pair = key_name, "=", value;
+                sub_block      = block_name, "{", block, "}";
+
+                key_name       = identifier;
+                block_name     = identifier;
+                value          = matches any string of text that ends with a newline character, # or }.  
+                                 note that the trailing newline, # or } is not part of the value though.
+                identifier     = Any string that matches the following regular expression:
+                                 [a-zA-Z][a-zA-Z0-9_-\.]*
+                                 i.e. Any string that starts with a letter and then is continued
+                                 with any number of letters, numbers, _ . or - characters.
+
+                Whitespace and comments are ignored.  A comment is text that starts with # (but not \#
+                since the \ escapes the # so that you can have a # symbol in a value if you want) and 
+                ends in a new line.  You can also escape a } (e.g. "\}") if you want to have one in a 
+                value.
+
+                Note that in a value the leading and trailing white spaces are stripped off but any 
+                white space inside the value is preserved.
+
+                Also note that all key_names and block_names within a block syntax group must be unique 
+                but don't have to be globally unique.  I.e. different blocks can reuse names. 
+
+                EXAMPLE CONFIG FILES:
+
+                    Example 1:
+                        #comment.  This line is ignored because it starts with #
+
+                        #here we have key1 which will have the value of "my value"
+                        key1 = my value 
+
+                        another_key=  another value  # this is another key called "another_key" with
+                                                     # a value of "another value"
+
+                        # this key's value is the empty string.  I.e. ""
+                        key2=
+
+                    Example 2:
+                        #this example illustrates the use of blocks
+                        some_key = blah blah
+
+                        # now here is a block
+                        our_block
+                        {
+                            # here we can define some keys and values that are local to this block.
+                            a_key = something
+                            foo = bar
+                            some_key = more stuff  # note that it is ok to name our key this even though
+                                                   # there is a key called some_key above.  This is because
+                                                   # we are doing so inside a different block
+                        }
+
+                        another_block { foo = bar2 }  # this block has only one key and is all on a single line
+        !*/
+    
+    public:
+
+        // exception classes
+        class config_reader_error : public dlib::error 
+        {
+            /*!
+                GENERAL
+                    This exception is thrown if there is an error while parsing the
+                    config file.  The type member of this exception will be set
+                    to ECONFIG_READER.
+
+                INTERPRETING THIS EXCEPTION
+                    - line_number == the line number the parser was at when the 
+                      error occurred.
+                    - if (redefinition) then
+                        - The key or block name on line line_number has already
+                          been defined in this scope which is an error.
+                    - else
+                        - Some other general syntax error was detected
+            !*/
+        public:
+            const unsigned long line_number;
+            const bool redefinition;
+        };
+
+        class file_not_found : public dlib::error 
+        {
+            /*!
+                GENERAL
+                    This exception is thrown if the config file can't be opened for
+                    some reason.  The type member of this exception will be set
+                    to ECONFIG_READER.
+
+                INTERPRETING THIS EXCEPTION
+                    - file_name == the name of the config file which we failed to open
+            !*/
+        public:
+            const std::string file_name;
+        };
+
+
+        class config_reader_access_error : public dlib::error
+        {
+            /*!
+                GENERAL
+                    This exception is thrown if you try to access a key or
+                    block that doesn't exist inside a config reader.  The type 
+                    member of this exception will be set to ECONFIG_READER.
+            !*/
+        public:
+            config_reader_access_error(
+                const std::string& block_name_,
+                const std::string& key_name_
+            );
+            /*!
+                ensures
+                    - #block_name == block_name_
+                    - #key_name == key_name_
+            !*/
+
+            const std::string block_name;
+            const std::string key_name;
+        };
+
+    // --------------------------
+
+        config_reader(
+        );
+        /*!
+            ensures 
+                - #*this is properly initialized
+                - This object will not have any keys or blocks defined in it.  
+            throws
+                - std::bad_alloc
+                - config_reader_error
+        !*/
+
+        config_reader(
+            std::istream& in
+        );
+        /*!
+            ensures 
+                - #*this is properly initialized
+                - reads the config file to parse from the given input stream,
+                  parses it and loads this object up with all the sub blocks and
+                  key/value pairs it finds.
+                - before the load is performed, the previous state of the config file
+                  reader is erased.  So after the load the config file reader will contain
+                  only information from the given config file.
+                - This object will represent the top most block of the config file.
+            throws
+                - std::bad_alloc
+                - config_reader_error
+        !*/
+
+        config_reader(
+            const std::string& config_file 
+        );
+        /*!
+            ensures 
+                - #*this is properly initialized
+                - parses the config file named by the config_file string.  Specifically, 
+                  parses it and loads this object up with all the sub blocks and
+                  key/value pairs it finds in the file.
+                - before the load is performed, the previous state of the config file
+                  reader is erased.  So after the load the config file reader will contain
+                  only information from the given config file.
+                - This object will represent the top most block of the config file.
+            throws
+                - std::bad_alloc
+                - config_reader_error
+                - file_not_found
+        !*/
+
+        virtual ~config_reader(
+        ); 
+        /*!
+            ensures
+                - all memory associated with *this has been released
+        !*/
+
+        void clear(
+        );
+        /*!
+            ensures
+                - #*this has its initial value
+            throws
+                - std::bad_alloc 
+                    If this exception is thrown then *this is unusable 
+                    until clear() is called and succeeds
+        !*/
+
+        void load_from (
+            std::istream& in
+        );
+        /*!
+            ensures 
+                - reads the config file to parse from the given input stream,
+                  parses it and loads this object up with all the sub blocks and
+                  key/value pairs it finds.
+                - before the load is performed, the previous state of the config file
+                  reader is erased.  So after the load the config file reader will contain
+                  only information from the given config file.
+                - *this will represent the top most block of the config file contained
+                  in the input stream in.
+            throws
+                - std::bad_alloc 
+                    If this exception is thrown then *this is unusable 
+                    until clear() is called and succeeds
+                - config_reader_error
+                    If this exception is thrown then this object will
+                    revert to its initial value.
+        !*/
+
+        void load_from (
+            const std::string& config_file
+        );
+        /*!
+            ensures 
+                - parses the config file named by the config_file string.  Specifically, 
+                  parses it and loads this object up with all the sub blocks and
+                  key/value pairs it finds in the file.  
+                - before the load is performed, the previous state of the config file
+                  reader is erased.  So after the load the config file reader will contain
+                  only information from the given config file.
+                - This object will represent the top most block of the config file.
+            throws
+                - std::bad_alloc 
+                    If this exception is thrown then *this is unusable 
+                    until clear() is called and succeeds
+                - config_reader_error
+                    If this exception is thrown then this object will
+                    revert to its initial value.
+                - file_not_found
+                    If this exception is thrown then this object will
+                    revert to its initial value.
+        !*/
+
+        bool is_key_defined (
+            const std::string& key_name
+        ) const;
+        /*!
+            ensures
+                - if (there is a key with the given name defined within this config_reader's block) then
+                    - returns true
+                - else
+                    - returns false
+        !*/
+
+        bool is_block_defined (
+            const std::string& block_name
+        ) const;
+        /*!
+            ensures
+                - if (there is a sub block with the given name defined within this config_reader's block) then
+                    - returns true
+                - else
+                    - returns false
+        !*/
+
+        typedef config_reader this_type;
+        const this_type& block (
+            const std::string& block_name
+        ) const;
+        /*!
+            ensures
+                - if (is_block_defined(block_name) == true) then
+                    - returns a const reference to the config_reader that represents the given named sub block
+                - else
+                    - throws config_reader_access_error
+            throws
+                - config_reader_access_error
+                    if this exception is thrown then its block_name field will be set to the
+                    given block_name string.
+        !*/
+
+        const std::string& operator[] (
+            const std::string& key_name
+        ) const;
+        /*!
+            ensures
+                - if (is_key_defined(key_name) == true) then
+                    - returns a const reference to the value string associated with the given key in 
+                      this config_reader's block.
+                - else
+                    - throws config_reader_access_error
+            throws
+                - config_reader_access_error
+                    if this exception is thrown then its key_name field will be set to the
+                    given key_name string.
+        !*/
+
+        template <
+            typename queue_of_strings
+            >
+        void get_keys (
+            queue_of_strings& keys
+        ) const;
+        /*!
+            requires
+                - queue_of_strings is an implementation of queue/queue_kernel_abstract.h 
+                  with T set to std::string, or std::vector<std::string>, or 
+                  dlib::std_vector_c<std::string>
+            ensures 
+                - #keys == a collection containing all the keys defined in this config_reader's block.
+                  (i.e. for all strings str in keys it is the case that is_key_defined(str) == true)
+        !*/
+
+        template <
+            typename queue_of_strings
+            >
+        void get_blocks (
+            queue_of_strings& blocks
+        ) const;
+        /*!
+            requires
+                - queue_of_strings is an implementation of queue/queue_kernel_abstract.h 
+                  with T set to std::string, or std::vector<std::string>, or 
+                  dlib::std_vector_c<std::string>
+            ensures 
+                - #blocks == a collection containing the names of all the blocks defined in this 
+                  config_reader's block.
+                  (i.e. for all strings str in blocks it is the case that is_block_defined(str) == true)
+        !*/
+
+    private:
+
+        // restricted functions
+        config_reader(config_reader&);        // copy constructor
+        config_reader& operator=(config_reader&);    // assignment operator
+
+    };
+
+}
+
+#endif // DLIB_CONFIG_READER_KERNEl_ABSTRACT_
+