Added a mostly complete implementation of the --to-xml option.

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%403718

docs/htmlify/CMakeLists.txt

@@ -18,6 +18,7 @@ PROJECT(${target_name})
 # cmake that they are part of our target (which is the exectuable named htmlify)
 ADD_EXECUTABLE(${target_name} 
 htmlify.cpp
+to_xml.cpp
 )
 
 # add the folder containing the dlib folder to the include path

docs/htmlify/htmlify.cpp

@@ -8,9 +8,10 @@
 #include "dlib/queue.h"
 #include "dlib/misc_api.h"
 #include "dlib/dir_nav.h"
+#include "to_xml.h"
 
 
-const char* VERSION = "2.7";
+const char* VERSION = "3.0";
 
 using namespace std;
 using namespace dlib;
@@ -98,13 +99,15 @@ int main(int argc, char** argv)
         parser.add_option("index","Create an index.");
         parser.add_option("v","Display version.");
         parser.add_option("man","Display the manual.");
-        parser.add_option("f","Specifies a list of file extensions separated by spaces.  The default is \"cpp h c\".",1);
+        parser.add_option("f","Specifies a list of file extensions to process when using the -i option.  The list elements should be separated by spaces.  The default is \"cpp h c\".",1);
         parser.add_option("i","Specifies an input directory.",1);
         parser.add_option("cat","Puts all the output into a single html file with the given name.",1);
         parser.add_option("depth","Specifies how many directories deep to search when using the i option.  The default value is 30.",1);
         parser.add_option("o","This option causes all the output files to be created inside the given directory.  If this option is not given then all output goes to the current working directory.",1);
         parser.add_option("flatten","When this option is given it prevents the input directory structure from being replicated.");
         parser.add_option("title","This option specifies a string which is prepended onto the title of the generated HTML",1);
+        parser.add_option("to-xml","Instead of generating HTML output, create a single output file called output.xml that contains "
+                          "a simple XML database which lists all documented classes and functions.");
 
         
         parser.parse(argc,argv);
@@ -115,8 +118,15 @@ int main(int argc, char** argv)
         parser.check_incompatible_options("cat","index");
         parser.check_option_arg_type<unsigned long>("depth");
 
+        parser.check_incompatible_options("to-xml", "b");
+        parser.check_incompatible_options("to-xml", "n");
+        parser.check_incompatible_options("to-xml", "index");
+        parser.check_incompatible_options("to-xml", "cat");
+        parser.check_incompatible_options("to-xml", "o");
+        parser.check_incompatible_options("to-xml", "flatten");
+        parser.check_incompatible_options("to-xml", "title");
 
-        const char* singles[] = {"b","n","h","index","v","man","f","cat","depth","o","flatten","title"};
+        const char* singles[] = {"b","n","h","index","v","man","f","cat","depth","o","flatten","title","to-xml"};
         parser.check_one_time_options(singles);
 
         const char* i_sub_ops[] = {"f","depth","flatten"};
@@ -135,6 +145,7 @@ int main(int argc, char** argv)
         const clp::option_type& flatten_opt = parser.option("flatten");
         const clp::option_type& depth_opt   = parser.option("depth");
         const clp::option_type& title_opt   = parser.option("title");
+        const clp::option_type& to_xml_opt  = parser.option("to-xml");
 
 
         string filter = "cpp h c";
@@ -160,7 +171,7 @@ int main(int argc, char** argv)
 
         if (h_opt)
         {
-            cout << "This program pretty prints C/C++ source code to HTML.\n";
+            cout << "This program pretty prints C or C++ source code to HTML.\n";
             cout << "Usage: htmlify [options] [file]...\n";
             parser.print_options(cout);
             cout << "\n\n";
@@ -191,6 +202,12 @@ int main(int argc, char** argv)
             search_depth = string_cast<unsigned long>(depth_opt.argument());
         }
 
+        if (to_xml_opt)
+        {
+            generate_xml_markup(parser, filter, search_depth);
+            return 0;
+        }
+
         if (o_opt)
         {
             // make sure this directory exists
@@ -577,7 +594,7 @@ void print_manual (
 
 
     sout << "The colored style puts HTML anchors on class and function names.  This means "
-         << "you can link directly the part of the code that contains these names.  For example, "
+         << "you can link directly to the part of the code that contains these names.  For example, "
          << "if you had a source file bar.cpp with a function called foo in it you could link "
          << "directly to the function with a link address of \"bar.cpp.html#foo\".  It is also "
          << "possible to instruct Htmlify to place HTML anchors at arbitrary spots by using a "

docs/htmlify/to_xml.cpp

+
+#include "to_xml.h"
+#include "dlib/dir_nav.h"
+#include <vector>
+#include <sstream>
+#include <iostream>
+#include <fstream>
+#include <stack>
+#include "dlib/cpp_tokenizer.h"
+
+using namespace dlib;
+using namespace std;
+
+// ----------------------------------------------------------------------------------------
+
+typedef cpp_tokenizer::kernel_1a_c tok_type;
+
+// ----------------------------------------------------------------------------------------
+
+class file_filter
+{
+public:
+
+    file_filter( 
+        const string& filter
+    )
+    {
+        // pick out the filter strings
+        istringstream sin(filter);
+        string temp;
+        sin >> temp;
+        while (sin)
+        {
+            endings.push_back("." + temp);
+            sin >> temp;
+        }
+    }
+
+    bool operator() ( const file& f) const
+    {
+        // check if any of the endings match
+        for (unsigned long i = 0; i < endings.size(); ++i)
+        {
+            // if the ending is bigger than f's name then it obviously doesn't match
+            if (endings[i].size() > f.name().size())
+                continue;
+
+            // now check if the actual characters that make up the end of the file name 
+            // matches what is in endings[i].
+            if ( std::equal(endings[i].begin(), endings[i].end(), f.name().end()-endings[i].size()))
+                return true;
+        }
+
+        return false;
+    }
+
+    std::vector<string> endings;
+};
+
+// ----------------------------------------------------------------------------------------
+
+void obtain_list_of_files (
+    const cmd_line_parser<char>::check_1a_c& parser, 
+    const std::string& filter, 
+    const unsigned long search_depth,
+    std::vector<file>& files
+)
+{
+    for (unsigned long i = 0; i < parser.option("i").count(); ++i)
+    {
+        const std::vector<file>& temp = get_files_in_directory_tree(parser.option("i").argument(0,i), file_filter(filter), search_depth);
+        files.insert(files.begin(), temp.begin(), temp.end());
+    }
+
+    for (unsigned long i = 0; i < parser.number_of_arguments(); ++i)
+    {
+        files.push_back(file(parser[i]));
+    }
+
+    std::sort(files.begin(), files.end());
+}
+
+// ----------------------------------------------------------------------------------------
+
+struct tok_function_record
+{
+    std::vector<std::pair<int,string> > declaration;
+    string scope;
+    string file;
+    string comment;
+};
+
+struct tok_method_record
+{
+    std::vector<std::pair<int,string> > declaration;
+    string comment;
+};
+
+struct tok_variable_record
+{
+    std::vector<std::pair<int,string> > declaration;
+};
+
+struct tok_typedef_record
+{
+    std::vector<std::pair<int,string> > declaration;
+};
+
+struct tok_class_record 
+{
+    std::vector<std::pair<int,string> > declaration;
+    string name;
+    string scope;
+    string file;
+    string comment;
+
+    std::vector<tok_method_record> public_methods;
+    std::vector<tok_variable_record> public_variables;
+    std::vector<tok_typedef_record> public_typedefs;
+    std::vector<tok_class_record> public_subclasses;
+};
+
+// ----------------------------------------------------------------------------------------
+
+struct function_record
+{
+    string name;
+    string scope;
+    string declaration;
+    string file;
+    string comment;
+};
+
+struct method_record
+{
+    string name;
+    string declaration;
+    string comment;
+};
+
+struct variable_record
+{
+    string declaration;
+};
+
+struct typedef_record
+{
+    string declaration;
+};
+
+struct class_record 
+{
+    string name;
+    string scope;
+    string declaration;
+    string file;
+    string comment;
+
+    std::vector<method_record> public_methods;
+    std::vector<variable_record> public_variables;
+    std::vector<typedef_record> public_typedefs;
+    std::vector<class_record> public_subclasses;
+};
+
+// ----------------------------------------------------------------------------------------
+
+unsigned long count_newlines (
+    const string& str
+)
+/*!
+    ensures
+        - returns the number of '\n' characters inside str
+!*/
+{
+    unsigned long count = 0;
+    for (unsigned long i = 0; i < str.size(); ++i)
+    {
+        if (str[i] == '\n')
+            ++count;
+    }
+    return count;
+}
+
+// ----------------------------------------------------------------------------------------
+
+bool contains_unescaped_newline (
+    const string& str
+)
+/*!
+    ensures
+        - returns true if str contains a '\n' character that isn't preceded by a '\' 
+          character.
+!*/
+{
+    if (str.size() == 0)
+        return false;
+
+    if (str[0] == '\n')
+        return true;
+
+    for (unsigned long i = 1; i < str.size(); ++i)
+    {
+        if (str[i] == '\n' && str[i-1] != '\\')
+            return true;
+    }
+
+    return false;
+}
+
+// ----------------------------------------------------------------------------------------
+
+bool is_formal_comment (
+    const string& str
+)
+{
+    if (str.size() < 6)
+        return false;
+
+    if (str[0] == '/' &&
+        str[1] == '*' &&
+        str[2] == '!' &&
+        str[3] != 'P' &&
+        str[3] != 'p' &&
+        str[str.size()-3] == '!' &&
+        str[str.size()-2] == '*' &&
+        str[str.size()-1] == '/' )
+        return true;
+
+    return false;
+}
+
+// ----------------------------------------------------------------------------------------
+
+string make_scope_string (
+    const std::vector<string>& namespaces,
+    unsigned long exclude_last_num_scopes = 0 
+)
+{
+    string temp;
+    for (unsigned long i = 0; i + exclude_last_num_scopes < namespaces.size(); ++i)
+    {
+        if (namespaces[i].size() == 0)
+            continue;
+
+        if (temp.size() == 0)
+            temp = namespaces[i];
+        else
+            temp += "::" + namespaces[i];
+    }
+    return temp;
+}
+
+// ----------------------------------------------------------------------------------------
+
+bool looks_like_function_declaration (
+    const std::vector<std::pair<int,string> >& declaration
+)
+{
+
+    // Check if declaration contains IDENTIFIER ( ) somewhere in it.
+    bool seen_first_part = false;
+    bool seen_operator = false;
+    int local_paren_count = 0;
+    for (unsigned long i = 1; i < declaration.size(); ++i)
+    {
+        if (declaration[i].first == tok_type::KEYWORD &&
+            declaration[i].second == "operator")
+        {
+            seen_operator = true;
+        }
+
+        if (declaration[i].first == tok_type::OTHER &&
+            declaration[i].second == "(" &&
+            (declaration[i-1].first == tok_type::IDENTIFIER || seen_operator))
+        {
+            seen_first_part = true;
+        }
+
+        if (declaration[i].first == tok_type::OTHER) 
+        {
+            if ( declaration[i].second == "(")
+                ++local_paren_count;
+            else if ( declaration[i].second == ")")
+                --local_paren_count;
+        }
+    }
+
+    if (seen_first_part && local_paren_count == 0)
+        return true;
+    else
+        return false;
+}
+
+// ----------------------------------------------------------------------------------------
+
+void process_file (
+    const string& file,
+    std::vector<tok_function_record>& functions,
+    std::vector<tok_class_record>& classes
+)
+/*!
+    ensures
+        - scans the given file for global functions and appends any found into functions.
+        - scans the given file for global classes and appends any found into classes.
+!*/
+{
+    tok_type tok;
+    ifstream fin(file.c_str());
+    tok.set_stream(fin);
+
+    bool recently_seen_struct_keyword = false;
+        // true if we have seen the struct keyword and
+        // we have not seen any identifiers or { characters
+
+    string last_struct_name;
+        // the name of the last struct we have seen
+
+    bool recently_seen_class_keyword = false;
+        // true if we have seen the class keyword and
+        // we have not seen any identifiers or { characters
+
+    string last_class_name;
+        // the name of the last class we have seen
+
+    bool recently_seen_namespace_keyword = false;
+        // true if we have seen the namespace keyword and
+        // we have not seen any identifiers or { characters
+
+    string last_namespace_name;
+        // the name of the last namespace we have seen
+
+    bool recently_seen_pound_define = false;
+        // true if we have seen a #define and haven't seen an unescaped newline
+
+    bool recently_seen_typedef = false;
+        // true if we have seen a typedef keyword and haven't seen a ;
+
+    bool recently_seen_paren_0 = false;
+        // true if we have seen paren_count transition to zero but haven't yet seen a ; or { or 
+        // a new line if recently_seen_pound_define is true.
+
+    bool recently_seen_closing_bracket = false;
+        // true if we have seen a } and haven't yet seen an IDENTIFIER or ;
+
+    bool recently_seen_new_scope = false;  
+        // true if we have seen the keywords class, namespace, struct, or extern and
+        // we have not seen the characters {, ), or ; since then
+
+    bool at_top_of_new_scope = false;
+        // true if we have seen the { that started a new scope but haven't seen anything yet but WHITE_SPACE
+
+    std::vector<string> namespaces; 
+        // a stack to hold the names of the scopes we have entered.  This is the classes, structs, and namespaces we enter.
+    namespaces.push_back(""); // this is the global namespace
+
+
+    std::stack<bool> inside_public_scope;
+        // If the stack isn't empty then we are inside a class or struct and the top value
+        // in the stack tells if we are in a public region.
+
+    std::stack<unsigned long> scopes; // a stack to hold current and old scope counts 
+                             // the top of the stack counts the number of new scopes (i.e. unmatched { ) we have entered 
+                             // since we were at a scope where functions can be defined.
+                             // We also maintain the invariant that scopes.size() == namespaces.size()
+    scopes.push(0);
+
+    std::stack<tok_class_record> class_stack;
+        // This is a stack where class_stack.top() == the incomplete class record for the class declaration we are
+        // currently in.
+
+    unsigned long paren_count = 0; 
+        // this is the number of ( we have seen minus the number of ) we have
+        // seen.
+
+    std::vector<std::pair<int,string> > token_accum;
+        // Used to accumulate tokens for function and class declarations
+
+    std::vector<std::pair<int,string> > last_full_declaration;
+        // Once we determine that token_accum has a full declaration in it we copy it into last_full_declaration. 
+
+    int type;
+    string token;
+
+    tok.get_token(type, token);
+
+    while (type != tok_type::END_OF_FILE)
+    {
+        switch(type)
+        {
+            case tok_type::KEYWORD: // ------------------------------------------
+                {
+                    token_accum.push_back(make_pair(type,token));
+
+                    if (token == "class")
+                    {
+                        recently_seen_class_keyword = true;
+                        recently_seen_new_scope = true;
+                    }
+                    else if (token == "struct")
+                    {
+                        recently_seen_struct_keyword = true;
+                        recently_seen_new_scope = true;
+                    }
+                    else if (token == "namespace")
+                    {
+                        cout << "hit namespace" << endl;
+                        recently_seen_namespace_keyword = true;
+                        recently_seen_new_scope = true;
+                    }
+                    else if (token == "extern")
+                    {
+                        recently_seen_new_scope = true;
+                    }
+                    else if (token == "#define")
+                    {
+                        recently_seen_pound_define = true;
+                    }
+                    else if (token == "typedef")
+                    {
+                        recently_seen_typedef = true;
+                    }
+                    else if (recently_seen_pound_define == false)
+                    {
+                        // eat white space
+                        int temp_type;
+                        string temp_token;
+                        if (tok.peek_type() == tok_type::WHITE_SPACE)
+                            tok.get_token(temp_type, temp_token);
+
+                        const bool next_is_colon = (tok.peek_type() == tok_type::OTHER && tok.peek_token() == ":");
+                        if (next_is_colon)
+                        {
+                            // eat the colon
+                            tok.get_token(temp_type, temp_token);
+
+                            if (inside_public_scope.size() > 0 && token == "public")
+                            {
+                                inside_public_scope.top() = true;
+                                token_accum.clear();
+                                last_full_declaration.clear();
+                            }
+                            else if (inside_public_scope.size() > 0 && token == "protected")
+                            {
+                                inside_public_scope.top() = true;
+                                token_accum.clear();
+                                last_full_declaration.clear();
+                            }
+                            else if (inside_public_scope.size() > 0 && token == "private")
+                            {
+                                inside_public_scope.top() = false;
+                                token_accum.clear();
+                                last_full_declaration.clear();
+                            }
+                        }
+                    }
+
+                    at_top_of_new_scope = false;
+
+                }break;
+
+            case tok_type::COMMENT: // ------------------------------------------
+                {
+                    if (scopes.top() == 0 && last_full_declaration.size() > 0 && is_formal_comment(token) &&
+                        paren_count == 0)
+                    {
+
+                        // if we are inside a class or struct
+                        if (inside_public_scope.size() > 0)
+                        {
+                            // if we are looking at a comment at the top of a class
+                            if (at_top_of_new_scope)
+                            {
+                                // push an entry for this class into the class_stack
+                                tok_class_record temp;
+                                temp.declaration = last_full_declaration;
+                                temp.file = file;
+                                temp.name = namespaces.back();
+                                temp.scope = make_scope_string(namespaces,1);
+                                temp.comment = token;
+                                class_stack.push(temp);
+                            }
+                            else if (inside_public_scope.top())
+                            {
+                                // This should be a member function.  
+                                // Only do anything if the class that contains this member function is
+                                // in the class_stack.
+                                if (class_stack.size() > 0 && class_stack.top().name == namespaces.back() &&
+                                    looks_like_function_declaration(last_full_declaration))
+                                {
+                                    tok_method_record temp;
+                                    temp.declaration = last_full_declaration;
+                                    temp.comment = token;
+                                    class_stack.top().public_methods.push_back(temp);
+                                }
+                            }
+                        }
+                        else
+                        {
+                            // we should be looking at a global declaration of some kind.   
+                            if (looks_like_function_declaration(last_full_declaration))
+                            {
+                                tok_function_record temp;
+                                temp.declaration = last_full_declaration;
+                                temp.file = file;
+                                temp.scope = make_scope_string(namespaces);
+                                temp.comment = token;
+                                functions.push_back(temp);
+                            }
+                        }
+
+                        token_accum.clear();
+                        last_full_declaration.clear();
+                    }
+
+                    at_top_of_new_scope = false;
+                }break;
+
+            case tok_type::IDENTIFIER: // ------------------------------------------
+                {
+                    if (recently_seen_class_keyword)
+                    {
+                        last_class_name = token;
+                        last_struct_name.clear();
+                        last_namespace_name.clear();
+                    }
+                    else if (recently_seen_struct_keyword)
+                    {
+                        last_struct_name = token;
+                        last_class_name.clear();
+                        last_namespace_name.clear();
+                    }
+                    else if (recently_seen_namespace_keyword)
+                    {
+                        last_namespace_name = token;
+                        last_class_name.clear();
+                        last_struct_name.clear();
+                    }
+
+                    recently_seen_class_keyword = false;
+                    recently_seen_struct_keyword = false;
+                    recently_seen_namespace_keyword = false;
+                    recently_seen_closing_bracket = false;
+                    at_top_of_new_scope = false;
+
+                    token_accum.push_back(make_pair(type,token));
+                }break;
+
+            case tok_type::OTHER: // ------------------------------------------
+                {
+                    switch(token[0])
+                    {
+                        case '{':
+                            // if we are entering a new scope
+                            if (recently_seen_new_scope)
+                            {
+                                cout << "new scope" << endl;
+                                scopes.push(0);
+                                at_top_of_new_scope = true;
+
+                                // if we are entering a class 
+                                if (last_class_name.size() > 0)
+                                {
+                                    inside_public_scope.push(false);
+                                    namespaces.push_back(last_class_name);
+                                }
+                                else if (last_struct_name.size() > 0)
+                                {
+                                    inside_public_scope.push(true);
+                                    namespaces.push_back(last_struct_name);
+                                }
+                                else if (last_namespace_name.size() > 0)
+                                {
+                                    namespaces.push_back(last_namespace_name);
+                                }
+                                else
+                                {
+                                    namespaces.push_back("");
+                                }
+                            }
+                            else
+                            {
+                                scopes.top() += 1;
+                            }
+                            recently_seen_new_scope = false;
+                            recently_seen_class_keyword = false;
+                            recently_seen_struct_keyword = false;
+                            recently_seen_namespace_keyword = false;
+                            recently_seen_paren_0 = false;
+
+                            // a { at function scope is an end of a potential declaration
+                            if (scopes.top() == 0)
+                            {
+                                // put token_accum into last_full_declaration
+                                token_accum.swap(last_full_declaration);
+                            }
+                            token_accum.clear();
+                            break;
+
+                        case '}':
+                            if (scopes.top() > 0)
+                            {
+                                scopes.top() -= 1;
+                            }
+                            else if (scopes.size() > 1)
+                            {
+                                scopes.pop();
+                                namespaces.pop_back();
+
+                                if (inside_public_scope.size() > 0)
+                                    inside_public_scope.pop();
+
+                                // if this class is a subclass of another then push it into the
+                                // public_subclasses field of it's containing class
+                                if (class_stack.size() > 1)
+                                {
+                                    tok_class_record temp = class_stack.top();
+                                    class_stack.pop();
+                                    class_stack.top().public_subclasses.push_back(temp);
+                                }
+                                else if (class_stack.size() > 0)
+                                {
+                                    classes.push_back(class_stack.top());
+                                    class_stack.pop();
+                                }
+                            }
+
+                            token_accum.clear();
+                            recently_seen_closing_bracket = true;
+                            at_top_of_new_scope = false;
+                            break;
+
+                        case ';':
+                            // a ; at function scope is an end of a potential declaration
+                            if (scopes.top() == 0)
+                            {
+                                // put token_accum into last_full_declaration
+                                token_accum.swap(last_full_declaration);
+                            }
+                            token_accum.clear();
+
+                            // if we are inside the public area of a class and this ; might be the end
+                            // of a typedef or variable declaration
+                            if (scopes.top() == 0 && inside_public_scope.size() > 0 && 
+                                inside_public_scope.top() == true &&
+                                recently_seen_closing_bracket == false)
+                            {
+                                if (recently_seen_typedef)
+                                {
+                                    // This should be a typedef inside the public area of a class or struct:
+                                    // Only do anything if the class that contains this typedef is in the class_stack.
+                                    if (class_stack.size() > 0 && class_stack.top().name == namespaces.back())
+                                    {
+                                        tok_typedef_record temp;
+                                        temp.declaration = last_full_declaration;
+                                        class_stack.top().public_typedefs.push_back(temp);
+                                    }
+
+                                }
+                                else if (recently_seen_paren_0 == false && recently_seen_new_scope == false)
+                                {
+                                    // This should be some kind of public variable declaration inside a class or struct:
+                                    // Only do anything if the class that contains this member variable is in the class_stack.
+                                    if (class_stack.size() > 0 && class_stack.top().name == namespaces.back())
+                                    {
+                                        tok_variable_record temp;
+                                        temp.declaration = last_full_declaration;
+                                        class_stack.top().public_variables.push_back(temp);
+                                    }
+
+                                }
+                            }
+
+                            recently_seen_new_scope = false;
+                            recently_seen_typedef = false;
+                            recently_seen_paren_0 = false;
+                            recently_seen_closing_bracket = false;
+                            at_top_of_new_scope = false;
+                            break;
+
+                        case '(':
+                            ++paren_count;
+                            token_accum.push_back(make_pair(type,token));
+                            at_top_of_new_scope = false;
+                            break;
+
+                        case ')':
+                            token_accum.push_back(make_pair(type,token));
+
+                            --paren_count;
+                            if (paren_count == 0)
+                            {
+                                recently_seen_paren_0 = true;
+                                last_full_declaration = token_accum;
+                            }
+
+                            recently_seen_new_scope = false;
+                            at_top_of_new_scope = false;
+                            break;
+
+                        default:
+                            token_accum.push_back(make_pair(type,token));
+                            at_top_of_new_scope = false;
+                            break;
+                    }
+                }break;
+
+
+            case tok_type::WHITE_SPACE: // ------------------------------------------
+                {
+                    if (recently_seen_pound_define)
+                    {
+                        if (contains_unescaped_newline(token))
+                        {
+                            recently_seen_pound_define = false;
+                            recently_seen_paren_0 = false;
+
+                            // this is an end of a potential declaration
+                            token_accum.swap(last_full_declaration);
+                            token_accum.clear();
+                        }
+                    }
+                }break;
+
+            default: // ------------------------------------------
+                {
+                    token_accum.push_back(make_pair(type,token));
+                    at_top_of_new_scope = false;
+                }break;
+        }
+
+
+        tok.get_token(type, token);
+    }
+}
+
+// ----------------------------------------------------------------------------------------
+
+string get_function_name (
+    const std::vector<std::pair<int,string> >& declaration
+)
+{
+    string name;
+
+    bool last_was_operator = false;
+    bool seen_operator = false;
+    for (unsigned long i = 0; i < declaration.size(); ++i)
+    {
+        if (declaration[i].first == tok_type::OTHER &&
+            declaration[i].second == "(" && !last_was_operator )
+        {
+            if (i != 0 && !seen_operator)
+            {
+                name = declaration[i-1].second;
+            }
+
+            break;
+        }
+
+        if (declaration[i].first == tok_type::KEYWORD &&
+            declaration[i].second == "operator")
+        {
+            last_was_operator = true;
+            seen_operator = true;
+        }
+        else
+        {
+            last_was_operator = false;
+        }
+
+        if (seen_operator)
+        {
+            if (name.size() != 0 && 
+                (declaration[i].first == tok_type::IDENTIFIER || declaration[i].first == tok_type::KEYWORD) )
+            {
+                name += " ";
+            }
+
+            name += declaration[i].second;
+        }
+
+    }
+
+    return name;
+}
+
+// ----------------------------------------------------------------------------------------
+
+typedef_record convert_tok_typedef_record (
+    const tok_typedef_record& rec
+)
+{
+    typedef_record temp;
+    for (unsigned long i = 0; i < rec.declaration.size(); ++i)
+    {
+        temp.declaration += rec.declaration[i].second + " ";
+    }
+    return temp;
+}
+
+// ----------------------------------------------------------------------------------------
+
+variable_record convert_tok_variable_record (
+    const tok_variable_record& rec
+)
+{
+    variable_record temp;
+    for (unsigned long i = 0; i < rec.declaration.size(); ++i)
+    {
+        temp.declaration += rec.declaration[i].second + " ";
+    }
+    return temp;
+}
+
+// ----------------------------------------------------------------------------------------
+
+method_record convert_tok_method_record (
+    const tok_method_record& rec
+)
+{
+    method_record temp;
+
+    temp.comment = rec.comment;
+    temp.name = get_function_name(rec.declaration);
+
+    for (unsigned long i = 0; i < rec.declaration.size(); ++i)
+    {
+        temp.declaration += rec.declaration[i].second + " ";
+    }
+    return temp;
+}
+
+// ----------------------------------------------------------------------------------------
+
+class_record convert_tok_class_record (
+    const tok_class_record& rec
+)
+{
+    class_record crec;
+
+
+    crec.scope = rec.scope;
+    crec.file = rec.file;
+    crec.comment = rec.comment;
+
+    crec.name.clear();
+
+    // find the first class token
+    for (unsigned long i = 0; i+1 < rec.declaration.size(); ++i)
+    {
+        if (rec.declaration[i].first == tok_type::KEYWORD &&
+            (rec.declaration[i].second == "class" ||
+            rec.declaration[i].second == "struct" )
+            )
+        {
+            crec.name = rec.declaration[i+1].second;
+            break;
+        }
+    }
+
+    crec.declaration.clear();
+    for (unsigned long i = 0; i < rec.declaration.size(); ++i)
+    {
+        crec.declaration += rec.declaration[i].second + " ";
+    }
+
+
+    for (unsigned long i = 0; i < rec.public_typedefs.size(); ++i)
+        crec.public_typedefs.push_back(convert_tok_typedef_record(rec.public_typedefs[i]));
+
+    for (unsigned long i = 0; i < rec.public_variables.size(); ++i)
+        crec.public_variables.push_back(convert_tok_variable_record(rec.public_variables[i]));
+
+    for (unsigned long i = 0; i < rec.public_methods.size(); ++i)
+        crec.public_methods.push_back(convert_tok_method_record(rec.public_methods[i]));
+
+    for (unsigned long i = 0; i < rec.public_subclasses.size(); ++i)
+        crec.public_subclasses.push_back(convert_tok_class_record(rec.public_subclasses[i]));
+
+
+    return crec;
+}
+
+// ----------------------------------------------------------------------------------------
+
+function_record convert_tok_function_record (
+    const tok_function_record& rec
+)
+{
+    function_record temp;
+
+    temp.scope = rec.scope;
+    temp.file = rec.file;
+    temp.comment = rec.comment;
+    temp.name = get_function_name(rec.declaration);
+
+    for (unsigned long i = 0; i < rec.declaration.size(); ++i)
+    {
+        temp.declaration += rec.declaration[i].second + " ";
+    }
+
+    return temp;
+}
+
+// ----------------------------------------------------------------------------------------
+
+void convert_to_normal_records (
+    const std::vector<tok_function_record>& tok_functions,
+    const std::vector<tok_class_record>& tok_classes,
+    std::vector<function_record>& functions,
+    std::vector<class_record>& classes
+)
+{
+    functions.clear();
+    classes.clear();
+
+
+    for (unsigned long i = 0; i < tok_functions.size(); ++i)
+    {
+        functions.push_back(convert_tok_function_record(tok_functions[i]));
+    }
+
+
+    for (unsigned long i = 0; i < tok_classes.size(); ++i)
+    {
+        classes.push_back(convert_tok_class_record(tok_classes[i]));
+    }
+
+
+}
+
+// ----------------------------------------------------------------------------------------
+
+void write_as_xml (
+    const function_record& rec,
+    ostream& fout
+)
+{
+    fout << "    <function>\n";
+    fout << "      <name>"        << rec.name        << "</name>\n";
+    fout << "      <scope>"       << rec.scope       << "</scope>\n";
+    fout << "      <declaration>" << rec.declaration << "</declaration>\n";
+    fout << "      <file>"        << rec.file        << "</file>\n";
+    fout << "      <comment>"     << rec.comment     << "</comment>\n";
+    fout << "    </function>\n";
+}
+
+// ----------------------------------------------------------------------------------------
+
+void write_as_xml (
+    const class_record& rec,
+    ostream& fout
+)
+{
+    fout << "    <class>\n";
+    fout << "      <name>"        << rec.name        << "</name>\n";
+    fout << "      <scope>"       << rec.scope       << "</scope>\n";
+    fout << "      <declaration>" << rec.declaration << "</declaration>\n";
+    fout << "      <file>"        << rec.file        << "</file>\n";
+    fout << "      <comment>"     << rec.comment     << "</comment>\n";
+
+
+    fout << "      <public_typedefs>\n";
+    for (unsigned long i = 0; i < rec.public_typedefs.size(); ++i)
+    {
+        fout << "        <typedef>"        << rec.public_typedefs[i].declaration        << "</typedef>\n";
+    }
+    fout << "      </public_typedefs>\n";
+
+
+    fout << "      <public_variables>\n";
+    for (unsigned long i = 0; i < rec.public_variables.size(); ++i)
+    {
+        fout << "        <variable>"        << rec.public_variables[i].declaration        << "</variable>\n";
+    }
+    fout << "      </public_variables>\n";
+
+
+    fout << "      <public_methods>\n";
+    for (unsigned long i = 0; i < rec.public_methods.size(); ++i)
+    {
+        fout << "        <method>\n";
+        fout << "          <name>"        << rec.public_methods[i].name        << "</name>\n";
+        fout << "          <declaration>" << rec.public_methods[i].declaration << "</declaration>\n";
+        fout << "          <comment>"     << rec.public_methods[i].comment     << "</comment>\n";
+        fout << "        </method>\n";
+    }
+    fout << "      </public_methods>\n";
+
+
+    fout << "      <public_subclasses>\n";
+    for (unsigned long i = 0; i < rec.public_subclasses.size(); ++i)
+    {
+        write_as_xml(rec.public_subclasses[i], fout);
+    }
+    fout << "      </public_subclasses>\n";
+
+
+    fout << "    </class>\n";
+}
+
+// ----------------------------------------------------------------------------------------
+
+void save_to_xml_file (
+    const std::vector<function_record>& functions,
+    const std::vector<class_record>& classes
+)
+{
+    ofstream fout("output.xml");
+
+    fout << "<code>" << endl;
+
+    fout << "  <classes>" << endl;
+    for (unsigned long i = 0; i < classes.size(); ++i)
+    {
+        write_as_xml(classes[i], fout);
+        fout << "\n";
+    }
+    fout << "  </classes>\n\n" << endl;
+
+
+    fout << "  <global_functions>" << endl;
+    for (unsigned long i = 0; i < functions.size(); ++i)
+    {
+        write_as_xml(functions[i], fout);
+        fout << "\n";
+    }
+    fout << "  </global_functions>" << endl;
+
+    fout << "</code>" << endl;
+}
+
+// ----------------------------------------------------------------------------------------
+
+void generate_xml_markup(
+    const cmd_line_parser<char>::check_1a_c& parser, 
+    const std::string& filter, 
+    const unsigned long search_depth
+)
+{
+
+    // first figure out which files should be processed
+    std::vector<file> files; 
+    obtain_list_of_files(parser, filter, search_depth, files);
+
+    cout << "files.size(): " << files.size() << endl;
+
+    std::vector<tok_function_record> tok_functions;
+    std::vector<tok_class_record> tok_classes;
+
+    for (unsigned long i = 0; i < files.size(); ++i)
+    {
+        process_file(files[i].full_name(), tok_functions, tok_classes); 
+    }
+
+    cout << "\ntok_functions.size(): " << tok_functions.size() << endl;
+    cout << "tok_classes.size(): " << tok_classes.size() << endl;
+    cout << "tok_classes[0].public_methods.size():    " << tok_classes[0].public_methods.size() << endl;
+    cout << "tok_classes[0].public_typedefs.size():   " << tok_classes[0].public_typedefs.size() << endl;
+    cout << "tok_classes[0].public_variables.size():  " << tok_classes[0].public_variables.size() << endl;
+    cout << "tok_classes[0].public_subclasses.size(): " << tok_classes[0].public_subclasses.size() << endl;
+    cout << endl;
+
+    //cout << tok_functions[0].comment << endl;
+
+    std::vector<function_record> functions;
+    std::vector<class_record> classes;
+
+    convert_to_normal_records(tok_functions, tok_classes, functions, classes);
+
+    save_to_xml_file(functions, classes);
+}
+
+// ----------------------------------------------------------------------------------------
+

docs/htmlify/to_xml.h

+#ifndef DLIB_HTMLIFY_TO_XmL_H__
+#define DLIB_HTMLIFY_TO_XmL_H__
+
+#include "dlib/cmd_line_parser.h"
+#include <string>
+
+void generate_xml_markup(
+    const dlib::cmd_line_parser<char>::check_1a_c& parser, 
+    const std::string& filter, 
+    const unsigned long search_depth
+);
+/*!
+    ensures
+        - reads all the 
+!*/
+
+#endif // DLIB_HTMLIFY_TO_XmL_H__
+