Update test driver to continue executing after exceptions and other failures (#868)

* Improve handling of exceptions in test driver

* Formatting

* Auto print exception

* Formatting

* Fork each test case

* Formatting

* Exclude gcc 5 debug build

* Fix tidy issues

* Add color

* Formatting

* Create driver class

* Formatting

* Customize test_case names

* Formatting

* Report status from forked processes

* Formatting

* Update the verify driver

* Formatting

* Print out failed tests

* Formatting

* Fix tidy issues

* Formatting

* Expect passing

* Improve failure reporting on non-linux systems

* Fix ifdef

* Flush code code cov

* Formatting

* Fix tidy

* Check if weak symbols is linked

* Formatting

* Add continue flag

* Formatting

* Set exe name

* Use stringstream and use quotes
Co-authored-by: mvermeulen <5479696+mvermeulen@users.noreply.github.com>

.github/workflows/ci.yaml

@@ -151,6 +151,9 @@ jobs:
           - debug
           - release
           - codecov
+        exclude:
+          - os: ubuntu-16.04
+            configuration: debug
 
     steps:
     - uses: actions/checkout@v2

src/compile_src.cpp

@@ -32,7 +32,7 @@ std::vector<char> src_compiler::compile(const std::vector<src_file>& srcs) const
         }
     }
 
-    params += " -o" + out;
+    params += " -o " + out;
 
     td.execute(compiler, params);
 

test/include/test.hpp

@@ -9,6 +9,10 @@
 #include <unordered_map>
 #include <vector>
 
+#ifdef __linux__
+#include <unistd.h>
+#endif
+
 #ifndef MIGRAPHX_GUARD_TEST_TEST_HPP
 #define MIGRAPHX_GUARD_TEST_TEST_HPP
 
@@ -264,6 +268,32 @@ struct capture
     }
 };
 
+enum class color
+{
+    reset      = 0,
+    bold       = 1,
+    underlined = 4,
+    fg_red     = 31,
+    fg_green   = 32,
+    fg_yellow  = 33,
+    fg_blue    = 34,
+    fg_default = 39,
+    bg_red     = 41,
+    bg_green   = 42,
+    bg_yellow  = 43,
+    bg_blue    = 44,
+    bg_default = 49
+};
+inline std::ostream& operator<<(std::ostream& os, const color& c)
+{
+#ifndef _WIN32
+    static const bool use_color = isatty(STDOUT_FILENO) != 0;
+    if(use_color)
+        return os << "\033[" << static_cast<std::size_t>(c) << "m";
+#endif
+    return os;
+}
+
 template <class T, class F>
 void failed(T x, const char* msg, const char* func, const char* file, int line, F f)
 {
@@ -271,7 +301,7 @@ void failed(T x, const char* msg, const char* func, const char* file, int line,
     {
         std::cout << func << std::endl;
         std::cout << file << ":" << line << ":" << std::endl;
-        std::cout << "    FAILED: " << msg << " "
+        std::cout << color::bold << color::fg_red << "    FAILED: " << color::reset << msg << " "
                   << "[ " << x << " ]" << std::endl;
         f();
     }
@@ -315,7 +345,7 @@ auto near(T px, U py, double ptol = 1e-6f)
 using string_map = std::unordered_map<std::string, std::vector<std::string>>;
 
 template <class Keyword>
-string_map parse(std::vector<std::string> as, Keyword keyword)
+string_map generic_parse(std::vector<std::string> as, Keyword keyword)
 {
     string_map result;
 
@@ -331,19 +361,22 @@ string_map parse(std::vector<std::string> as, Keyword keyword)
         {
             flag = f.front();
             result[flag]; // Ensure the flag exists
+            flag = f.back();
         }
     }
     return result;
 }
 
+using test_case = std::function<void()>;
+
 inline auto& get_test_cases()
 {
     // NOLINTNEXTLINE
-    static std::vector<std::pair<std::string, std::function<void()>>> cases;
+    static std::vector<std::pair<std::string, test_case>> cases;
     return cases;
 }
 
-inline void add_test_case(std::string name, std::function<void()> f)
+inline void add_test_case(std::string name, test_case f)
 {
     get_test_cases().emplace_back(std::move(name), std::move(f));
 }
@@ -357,37 +390,243 @@ struct auto_register_test_case
     }
 };
 
-inline void run_test_case(const std::string& name, const std::function<void()>& f)
+struct failure_error
 {
-    std::cout << "[   RUN    ] " << name << std::endl;
-    f();
-    std::cout << "[ COMPLETE ] " << name << std::endl;
-}
+};
 
-inline void run(int argc, const char* argv[])
+[[noreturn]] inline void fail() { throw failure_error{}; }
+
+struct driver
 {
-    std::vector<std::string> as(argv + 1, argv + argc);
+    driver()
+    {
+        add_flag({"--help", "-h"}, "Show help");
+        add_flag({"--list", "-l"}, "List all test cases");
+        add_flag({"--continue", "-c"}, "Continue after failure");
+        add_flag({"--quiet", "-q"}, "Don't print out extra output");
+    }
+    struct argument
+    {
+        std::vector<std::string> flags = {};
+        std::string help               = "";
+        int nargs                      = 1;
+    };
+
+    void add_arg(const std::vector<std::string>& flags, const std::string& help = "")
+    {
+        arguments.push_back(argument{flags, help, 1});
+    }
+
+    void add_flag(const std::vector<std::string>& flags, const std::string& help = "")
+    {
+        arguments.push_back(argument{flags, help, 0});
+    }
+
+    void show_help(const std::string& exe) const
+    {
+        std::cout << std::endl;
+        std::cout << color::fg_yellow << "USAGE:" << color::reset << std::endl;
+        std::cout << "    ";
+        std::cout << exe << " <test-case>... <options>" << std::endl;
+        std::cout << std::endl;
+
+        std::cout << color::fg_yellow << "ARGS:" << color::reset << std::endl;
+        std::cout << "    ";
+        std::cout << color::fg_green << "<test-case>..." << color::reset;
+        std::cout << std::endl;
+        std::cout << "        "
+                  << "Test case name to run" << std::endl;
+        std::cout << std::endl;
+        std::cout << color::fg_yellow << "OPTIONS:" << color::reset << std::endl;
+        for(auto&& arg : arguments)
+        {
+            std::string prefix = "    ";
+            std::cout << color::fg_green;
+            for(const std::string& a : arg.flags)
+            {
+                std::cout << prefix;
+                std::cout << a;
+                prefix = ", ";
+            }
+            std::cout << color::reset << std::endl;
+            std::cout << "        " << arg.help << std::endl;
+        }
+    }
+
+    std::ostream& out() const
+    {
+        struct null_buffer : std::streambuf
+        {
+            virtual int overflow(int c) override { return c; }
+        };
+        static null_buffer buffer;
+        static std::ostream null_stream(&buffer);
+        if(quiet)
+            return null_stream;
+        return std::cout;
+    }
+
+    string_map parse(int argc, const char* argv[]) const
+    {
+        std::vector<std::string> args(argv + 1, argv + argc);
+        string_map keys;
+        for(auto&& arg : arguments)
+        {
+            for(auto&& flag : arg.flags)
+            {
+                keys[flag] = {arg.flags.front()};
+                if(arg.nargs == 0)
+                    keys[flag].push_back("");
+            }
+        }
+        auto result = generic_parse(args, [&](auto&& s) -> std::vector<std::string> {
+            if(keys.count(s) > 0)
+                return keys[s];
+            else
+                return {};
+        });
+        result["__exe__"].push_back(argv[0]);
+        return result;
+    }
+
+    static std::string create_command(const string_map& args)
+    {
+        std::stringstream ss;
+        ss << args.at("__exe__").front();
+        if(args.count("") > 0)
+        {
+            for(auto&& arg : args.at(""))
+                ss << " \"" << arg << "\"";
+        }
+        for(auto&& p : args)
+        {
+            if(p.first == "__exe__")
+                continue;
+            if(p.first.empty())
+                continue;
+            ss << " " << p.first;
+            for(auto&& arg : p.second)
+                ss << " \"" << arg << "\"";
+        }
+        return ss.str();
+    }
+
+    static std::string fork(const std::string& name, string_map args)
+    {
+        std::string msg;
+        args[""] = {name};
+        args.erase("--continue");
+        args["--quiet"];
+        auto cmd = create_command(args);
+        auto r   = std::system(cmd.c_str()); // NOLINT
+        if(r != 0)
+            msg = "Exited with " + std::to_string(r);
+        return msg;
+    }
+
+    void run_test_case(const std::string& name, const test_case& f, const string_map& args)
+    {
+        ran++;
+        out() << color::fg_green << "[   RUN    ] " << color::reset << color::bold << name
+              << color::reset << std::endl;
+        std::string msg;
+        if(args.count("--continue") > 0)
+        {
+            msg = fork(name, args);
+        }
+        else
+        {
+            try
+            {
+                f();
+            }
+            catch(const failure_error&)
+            {
+                msg = "Test failure";
+            }
+        }
+        if(msg.empty())
+        {
+            out() << color::fg_green << "[ COMPLETE ] " << color::reset << color::bold << name
+                  << color::reset << std::endl;
+        }
+        else
+        {
+            failed.push_back(name);
+            out() << color::fg_red << "[  FAILED  ] " << color::reset << color::bold << name
+                  << color::reset << ": " << color::fg_yellow << msg << color::reset << std::endl;
+        }
+    }
+
+    void run(int argc, const char* argv[])
+    {
+        auto args = parse(argc, argv);
+        if(args.count("--help") > 0)
+        {
+            show_help(args.at("__exe__").front());
+            return;
+        }
+        if(args.count("--list") > 0)
+        {
+            for(auto&& tc : get_test_cases())
+                out() << tc.first << std::endl;
+            return;
+        }
+
+        if(args.count("--quiet") > 0)
+            quiet = true;
 
-    auto args  = parse(as, [](auto &&) -> std::vector<std::string> { return {}; });
         auto cases = args[""];
         if(cases.empty())
         {
             for(auto&& tc : get_test_cases())
-            run_test_case(tc.first, tc.second);
+                run_test_case(tc.first, tc.second, args);
         }
         else
         {
-        std::unordered_map<std::string, std::function<void()>> m(get_test_cases().begin(),
+            std::unordered_map<std::string, test_case> m(get_test_cases().begin(),
                                                          get_test_cases().end());
-        for(auto&& name : cases)
+            for(auto&& iname : cases)
+            {
+                for(auto&& name : get_case_names(iname))
                 {
                     auto f = m.find(name);
                     if(f == m.end())
-                std::cout << "[  ERROR   ] Test case '" << name << "' not found." << std::endl;
+                    {
+                        out() << color::fg_red << "[  ERROR   ] Test case '" << name
+                              << "' not found." << color::reset << std::endl;
+                        failed.push_back(name);
+                    }
                     else
-                run_test_case(name, f->second);
+                        run_test_case(name, f->second, args);
+                }
+            }
+        }
+        out() << color::fg_green << "[==========] " << color::fg_yellow << ran << " tests ran"
+              << color::reset << std::endl;
+        if(not failed.empty())
+        {
+            out() << color::fg_red << "[  FAILED  ] " << color::fg_yellow << failed.size()
+                  << " tests failed" << color::reset << std::endl;
+            for(auto&& name : failed)
+                out() << color::fg_red << "[  FAILED  ] " << color::fg_yellow << name
+                      << color::reset << std::endl;
+            std::exit(1);
         }
     }
+
+    std::function<std::vector<std::string>(const std::string&)> get_case_names =
+        [](const std::string& name) -> std::vector<std::string> { return {name}; };
+    std::vector<argument> arguments = {};
+    std::vector<std::string> failed = {};
+    std::size_t ran                 = 0;
+    bool quiet                      = false;
+};
+
+inline void run(int argc, const char* argv[])
+{
+    driver d{};
+    d.run(argc, argv);
 }
 
 } // namespace test
@@ -404,7 +643,7 @@ inline void run(int argc, const char* argv[])
                  __PRETTY_FUNCTION__,           \
                  __FILE__,                      \
                  __LINE__,                      \
-                 &std::abort)
+                 &test::fail)
 // NOLINTNEXTLINE
 #define STATUS(...) EXPECT((__VA_ARGS__) == 0)
 

test/main.cpp

+#include "test.hpp"
 
-int main() {}
+int main(int argc, const char* argv[]) { test::run(argc, argv); }

test/verify/auto_print.cpp

@@ -44,3 +44,23 @@ void auto_print::set_terminate_handler(const std::string& name)
             get_handler(tname)();
     });
 }
+
+static bool in_exception()
+{
+#if __cplusplus >= 201703L
+    return std::uncaught_exceptions() > 0;
+#else
+    return std::uncaught_exception();
+#endif
+}
+
+auto_print::~auto_print()
+{
+    if(in_exception())
+    {
+        std::cout << std::endl;
+        for(const auto& tname : migraphx::get_targets())
+            get_handler(tname)();
+    }
+    get_handler(name) = [] {};
+}

test/verify/auto_print.hpp

@@ -15,10 +15,7 @@ struct auto_print
         get_handler(name) = [&x] { std::cout << x << std::endl; };
     }
 
-    ~auto_print()
-    {
-        get_handler(name) = [] {};
-    }
+    ~auto_print();
 };
 
 #endif

test/verify/run_verify.cpp

 #include "run_verify.hpp"
 #include "auto_print.hpp"
 #include "verify_program.hpp"
+#include "test.hpp"
 #include <migraphx/env.hpp>
 #include <migraphx/ref/target.hpp>
 #include <migraphx/ranges.hpp>
@@ -121,7 +122,6 @@ void run_verify::verify(const std::string& name, const migraphx::program& p) con
 {
     using result_future =
         std::future<std::pair<migraphx::program, std::vector<migraphx::argument>>>;
-    std::cout << "[   RUN    ] " << name << std::endl;
     auto_print::set_terminate_handler(name);
     std::vector<std::pair<std::string, result_future>> results;
     std::vector<std::string> target_names;
@@ -180,25 +180,27 @@ void run_verify::verify(const std::string& name, const migraphx::program& p) con
                 std::cout << tname << ":\n" << cp << std::endl;
                 std::cout << std::endl;
             }
+            EXPECT(passed);
         }
     }
     std::set_terminate(nullptr);
-    std::cout << "[ COMPLETE ] " << name << std::endl;
 }
 
 void run_verify::run(int argc, const char* argv[]) const
 {
-    std::set<std::string> args(argv + 1, argv + argc);
-    const auto& ps = get_programs();
-    for(auto&& p : ps)
+    std::unordered_map<std::string, std::vector<std::string>> labels;
+    for(auto&& p : get_programs())
     {
-        if(not args.empty())
-        {
-            if(args.count(p.name) == 0 and args.count(p.section) == 0)
-                continue;
-        }
-        verify(p.name, p.get_program());
+        labels[p.section].push_back(p.name);
+        test::add_test_case(p.name, [=] { verify(p.name, p.get_program()); });
     }
+    test::driver d{};
+    d.get_case_names = [&](const std::string& name) -> std::vector<std::string> {
+        if(labels.count(name) > 0)
+            return labels.at(name);
+        return {name};
+    };
+    d.run(argc, argv);
 }
 
 void run_verify::disable_parallel_for(const std::string& name) { info[name].parallel = false; }