Implements --gtest_also_run_disabled_tests. By Eric Roman.

src/gtest-internal-inl.h

@@ -63,6 +63,7 @@ namespace testing {
 // We don't want the users to modify these flags in the code, but want
 // Google Test's own unit tests to be able to access them.  Therefore we
 // declare them here as opposed to in gtest.h.
+GTEST_DECLARE_bool_(also_run_disabled_tests);
 GTEST_DECLARE_bool_(break_on_failure);
 GTEST_DECLARE_bool_(catch_exceptions);
 GTEST_DECLARE_string_(color);
@@ -72,8 +73,8 @@ GTEST_DECLARE_bool_(list_tests);
 GTEST_DECLARE_string_(output);
 GTEST_DECLARE_bool_(print_time);
 GTEST_DECLARE_int32_(repeat);
-GTEST_DECLARE_int32_(stack_trace_depth);
 GTEST_DECLARE_bool_(show_internal_stack_frames);
+GTEST_DECLARE_int32_(stack_trace_depth);
 
 namespace internal {
 
@@ -82,6 +83,7 @@ namespace internal {
 extern const TypeId kTestTypeIdInGoogleTest;
 
 // Names of the flags (needed for parsing Google Test flags).
+const char kAlsoRunDisabledTestsFlag[] = "also_run_disabled_tests";
 const char kBreakOnFailureFlag[] = "break_on_failure";
 const char kCatchExceptionsFlag[] = "catch_exceptions";
 const char kColorFlag[] = "color";
@@ -97,6 +99,7 @@ class GTestFlagSaver {
  public:
   // The c'tor.
   GTestFlagSaver() {
+    also_run_disabled_tests_ = GTEST_FLAG(also_run_disabled_tests);
     break_on_failure_ = GTEST_FLAG(break_on_failure);
     catch_exceptions_ = GTEST_FLAG(catch_exceptions);
     color_ = GTEST_FLAG(color);
@@ -112,6 +115,7 @@ class GTestFlagSaver {
 
   // The d'tor is not virtual.  DO NOT INHERIT FROM THIS CLASS.
   ~GTestFlagSaver() {
+    GTEST_FLAG(also_run_disabled_tests) = also_run_disabled_tests_;
     GTEST_FLAG(break_on_failure) = break_on_failure_;
     GTEST_FLAG(catch_exceptions) = catch_exceptions_;
     GTEST_FLAG(color) = color_;
@@ -126,6 +130,7 @@ class GTestFlagSaver {
   }
  private:
   // Fields for saving the original values of flags.
+  bool also_run_disabled_tests_;
   bool break_on_failure_;
   bool catch_exceptions_;
   String color_;

src/gtest.cc

@@ -153,6 +153,11 @@ const char kStackTraceMarker[] = "\nStack trace:\n";
 
 }  // namespace internal
 
+GTEST_DEFINE_bool_(
+    also_run_disabled_tests,
+    internal::BoolFromGTestEnv("also_run_disabled_tests", false),
+    "Run disabled tests too, in addition to the tests normally being run.");
+
 GTEST_DEFINE_bool_(
     break_on_failure,
     internal::BoolFromGTestEnv("break_on_failure", false),
@@ -2736,7 +2741,7 @@ void PrettyUnitTestResultPrinter::OnUnitTestEnd(
   }
 
   int num_disabled = impl->disabled_test_count();
-  if (num_disabled) {
+  if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) {
     if (!num_failures) {
       printf("\n");  // Add a spacer if no FAILURE banner is displayed.
     }
@@ -3602,7 +3607,8 @@ int UnitTestImpl::FilterTests() {
                                                  kDisableTestFilter);
       test_info->impl()->set_is_disabled(is_disabled);
 
-      const bool should_run = !is_disabled &&
+      const bool should_run =
+          (GTEST_FLAG(also_run_disabled_tests) || !is_disabled) &&
           internal::UnitTestOptions::FilterMatchesTest(test_case_name,
                                                        test_name);
       test_info->impl()->set_should_run(should_run);
@@ -3860,7 +3866,9 @@ void ParseGoogleTestFlagsOnlyImpl(int* argc, CharType** argv) {
     using internal::ParseStringFlag;
 
     // Do we see a Google Test flag?
-    if (ParseBoolFlag(arg, kBreakOnFailureFlag,
+    if (ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag,
+                      &GTEST_FLAG(also_run_disabled_tests)) ||
+        ParseBoolFlag(arg, kBreakOnFailureFlag,
                       &GTEST_FLAG(break_on_failure)) ||
         ParseBoolFlag(arg, kCatchExceptionsFlag,
                       &GTEST_FLAG(catch_exceptions)) ||

test/gtest_filter_unittest.py

@@ -40,12 +40,11 @@ environments and command line flags.
 
 __author__ = 'wan@google.com (Zhanyong Wan)'
 
-import gtest_test_utils
 import os
 import re
 import sets
-import sys
 import unittest
+import gtest_test_utils
 
 # Constants.
 
@@ -55,6 +54,9 @@ FILTER_ENV_VAR = 'GTEST_FILTER'
 # The command line flag for specifying the test filters.
 FILTER_FLAG = 'gtest_filter'
 
+# The command line flag for including disabled tests.
+ALSO_RUN_DISABED_TESTS_FLAG = 'gtest_also_run_disabled_tests'
+
 # Command to run the gtest_filter_unittest_ program.
 COMMAND = os.path.join(gtest_test_utils.GetBuildDir(),
                        'gtest_filter_unittest_')
@@ -80,7 +82,17 @@ PARAM_TESTS = [
     'SeqQ/ParamTest.TestY/1',
     ]
 
-ALL_TESTS = [
+DISABLED_TESTS = [
+    'BarTest.DISABLED_TestFour',
+    'BarTest.DISABLED_TestFive',
+    'BazTest.DISABLED_TestC',
+    'DISABLED_FoobarTest.Test1',
+    'DISABLED_FoobarTest.DISABLED_Test2',
+    'DISABLED_FoobarbazTest.TestA',
+    ]
+
+# All the non-disabled tests.
+ACTIVE_TESTS = [
     'FooTest.Abc',
     'FooTest.Xyz',
 
@@ -176,6 +188,18 @@ class GTestFilterUnitTest(unittest.TestCase):
     tests_run = Run(command)
     self.AssertSetEqual(tests_run, tests_to_run)
 
+  def RunAndVerifyAllowingDisabled(self, gtest_filter, tests_to_run):
+    """Runs gtest_flag_unittest_ with the given filter, and enables
+    disabled tests. Verifies that the right set of tests were run.
+    """
+    # Construct the command line.
+    command = '%s --%s' % (COMMAND, ALSO_RUN_DISABED_TESTS_FLAG)
+    if gtest_filter is not None:
+      command = '%s --%s=%s' % (command, FILTER_FLAG, gtest_filter)
+
+    tests_run = Run(command)
+    self.AssertSetEqual(tests_run, tests_to_run)
+
   def setUp(self):
     """Sets up test case. Determines whether value-parameterized tests are
     enabled in the binary and sets flags accordingly.
@@ -188,7 +212,7 @@ class GTestFilterUnitTest(unittest.TestCase):
   def testDefaultBehavior(self):
     """Tests the behavior of not specifying the filter."""
 
-    self.RunAndVerify(None, ALL_TESTS)
+    self.RunAndVerify(None, ACTIVE_TESTS)
 
   def testEmptyFilter(self):
     """Tests an empty filter."""
@@ -199,28 +223,62 @@ class GTestFilterUnitTest(unittest.TestCase):
     """Tests a filter that matches nothing."""
 
     self.RunAndVerify('BadFilter', [])
+    self.RunAndVerifyAllowingDisabled('BadFilter', [])
 
   def testFullName(self):
     """Tests filtering by full name."""
 
     self.RunAndVerify('FooTest.Xyz', ['FooTest.Xyz'])
+    self.RunAndVerifyAllowingDisabled('FooTest.Xyz', ['FooTest.Xyz'])
 
   def testUniversalFilters(self):
     """Tests filters that match everything."""
 
-    self.RunAndVerify('*', ALL_TESTS)
-    self.RunAndVerify('*.*', ALL_TESTS)
+    self.RunAndVerify('*', ACTIVE_TESTS)
+    self.RunAndVerify('*.*', ACTIVE_TESTS)
+    self.RunAndVerifyAllowingDisabled('*', ACTIVE_TESTS + DISABLED_TESTS)
+    self.RunAndVerifyAllowingDisabled('*.*', ACTIVE_TESTS + DISABLED_TESTS)
 
   def testFilterByTestCase(self):
     """Tests filtering by test case name."""
 
     self.RunAndVerify('FooTest.*', ['FooTest.Abc', 'FooTest.Xyz'])
 
+    BAZ_TESTS = ['BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB']
+    self.RunAndVerify('BazTest.*', BAZ_TESTS)
+    self.RunAndVerifyAllowingDisabled('BazTest.*',
+                                      BAZ_TESTS + ['BazTest.DISABLED_TestC'])
+
   def testFilterByTest(self):
     """Tests filtering by test name."""
 
     self.RunAndVerify('*.TestOne', ['BarTest.TestOne', 'BazTest.TestOne'])
 
+  def testFilterDisabledTests(self):
+    """Select only the disabled tests to run."""
+
+    self.RunAndVerify('DISABLED_FoobarTest.Test1', [])
+    self.RunAndVerifyAllowingDisabled('DISABLED_FoobarTest.Test1',
+                                      ['DISABLED_FoobarTest.Test1'])
+
+    self.RunAndVerify('*DISABLED_*', [])
+    self.RunAndVerifyAllowingDisabled('*DISABLED_*', DISABLED_TESTS)
+
+    self.RunAndVerify('*.DISABLED_*', [])
+    self.RunAndVerifyAllowingDisabled('*.DISABLED_*', [
+        'BarTest.DISABLED_TestFour',
+        'BarTest.DISABLED_TestFive',
+        'BazTest.DISABLED_TestC',
+        'DISABLED_FoobarTest.DISABLED_Test2',
+        ])
+
+    self.RunAndVerify('DISABLED_*', [])
+    self.RunAndVerifyAllowingDisabled('DISABLED_*', [
+        'DISABLED_FoobarTest.Test1',
+        'DISABLED_FoobarTest.DISABLED_Test2',
+        'DISABLED_FoobarbazTest.TestA',
+        ])
+
   def testWildcardInTestCaseName(self):
     """Tests using wildcard in the test case name."""
 
@@ -231,7 +289,7 @@ class GTestFilterUnitTest(unittest.TestCase):
 
         'BazTest.TestOne',
         'BazTest.TestA',
-        'BazTest.TestB',] + PARAM_TESTS)
+        'BazTest.TestB' ] + PARAM_TESTS)
 
   def testWildcardInTestName(self):
     """Tests using wildcard in the test name."""

test/gtest_filter_unittest_.cc

@@ -67,6 +67,13 @@ TEST(BarTest, TestTwo) {
 TEST(BarTest, TestThree) {
 }
 
+TEST(BarTest, DISABLED_TestFour) {
+  FAIL() << "Expected failure.";
+}
+
+TEST(BarTest, DISABLED_TestFive) {
+  FAIL() << "Expected failure.";
+}
 
 // Test case BazTest.
 
@@ -80,6 +87,26 @@ TEST(BazTest, TestA) {
 TEST(BazTest, TestB) {
 }
 
+TEST(BazTest, DISABLED_TestC) {
+  FAIL() << "Expected failure.";
+}
+
+// Test case FoobarTest
+
+TEST(DISABLED_FoobarTest, Test1) {
+  FAIL() << "Expected failure.";
+}
+
+TEST(DISABLED_FoobarTest, DISABLED_Test2) {
+  FAIL() << "Expected failure.";
+}
+
+// Test case FoobarbazTest
+
+TEST(DISABLED_FoobarbazTest, TestA) {
+  FAIL() << "Expected failure.";
+}
+
 #ifdef GTEST_HAS_PARAM_TEST
 class ParamTest : public testing::TestWithParam<int> {
 };

test/gtest_output_test.py

@@ -40,12 +40,12 @@ SYNOPSIS
 
 __author__ = 'wan@google.com (Zhanyong Wan)'
 
-import gtest_test_utils
 import os
 import re
 import string
 import sys
 import unittest
+import gtest_test_utils
 
 
 # The flag for generating the golden file
@@ -64,10 +64,13 @@ PROGRAM_PATH = os.path.join(gtest_test_utils.GetBuildDir(), PROGRAM)
 COMMAND_WITH_COLOR = PROGRAM_PATH + ' --gtest_color=yes'
 COMMAND_WITH_TIME = (PROGRAM_PATH + ' --gtest_print_time '
                      + '--gtest_filter="FatalFailureTest.*:LoggingTest.*"')
+COMMAND_WITH_DISABLED = (PROGRAM_PATH + ' --gtest_also_run_disabled_tests '
+                         + '--gtest_filter="*DISABLED_*"')
 
 GOLDEN_PATH = os.path.join(gtest_test_utils.GetSourceDir(),
                            GOLDEN_NAME)
 
+
 def ToUnixLineEnding(s):
   """Changes all Windows/Mac line endings in s to UNIX line endings."""
 
@@ -191,7 +194,8 @@ def GetCommandOutput(cmd):
 class GTestOutputTest(unittest.TestCase):
   def testOutput(self):
     output = (GetCommandOutput(COMMAND_WITH_COLOR) +
-              GetCommandOutput(COMMAND_WITH_TIME))
+              GetCommandOutput(COMMAND_WITH_TIME) +
+              GetCommandOutput(COMMAND_WITH_DISABLED))
     golden_file = open(GOLDEN_PATH, 'rb')
     golden = golden_file.read()
     golden_file.close()
@@ -206,7 +210,8 @@ class GTestOutputTest(unittest.TestCase):
 if __name__ == '__main__':
   if sys.argv[1:] == [GENGOLDEN_FLAG]:
     output = (GetCommandOutput(COMMAND_WITH_COLOR) +
-              GetCommandOutput(COMMAND_WITH_TIME))
+              GetCommandOutput(COMMAND_WITH_TIME) +
+              GetCommandOutput(COMMAND_WITH_DISABLED))
     golden_file = open(GOLDEN_PATH, 'wb')
     golden_file.write(output)
     golden_file.close()

test/gtest_output_test_.cc

@@ -212,6 +212,14 @@ TEST(SCOPED_TRACETest, CanBeRepeated) {
                 << "trace point A, B, and D.";
 }
 
+TEST(DisabledTestsWarningTest,
+     DISABLED_AlsoRunDisabledTestsFlagSuppressesWarning) {
+  // This test body is intentionally empty.  Its sole purpose is for
+  // verifying that the --gtest_also_run_disabled_tests flag
+  // suppresses the "YOU HAVE 12 DISABLED TESTS" warning at the end of
+  // the test output.
+}
+
 // Tests using assertions outside of TEST and TEST_F.
 //
 // This function creates two failures intentionally.

test/gtest_output_test_golden_lin.txt

@@ -534,7 +534,9 @@ Expected fatal failure.
 [  FAILED  ] ExpectFailureTest.ExpectNonFatalFailureOnAllThreads
 
 33 FAILED TESTS
-The non-test part of the code is expected to have 2 failures.
+  YOU HAVE 1 DISABLED TEST
+
+The non-test part of the code is expected to have 2 failures.
 
 gtest_output_test_.cc:#: Failure
 Value of: false
@@ -604,3 +606,36 @@ Expected fatal failure.
 [  FAILED  ] LoggingTest.InterleavingLoggingAndAssertions
 
  4 FAILED TESTS
+  YOU HAVE 1 DISABLED TEST
+
+The non-test part of the code is expected to have 2 failures.
+
+gtest_output_test_.cc:#: Failure
+Value of: false
+  Actual: false
+Expected: true
+gtest_output_test_.cc:#: Failure
+Value of: 3
+Expected: 2
+Note: Google Test filter = *DISABLED_*
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+FooEnvironment::SetUp() called.
+BarEnvironment::SetUp() called.
+[----------] 1 test from DisabledTestsWarningTest
+[ RUN      ] DisabledTestsWarningTest.DISABLED_AlsoRunDisabledTestsFlagSuppressesWarning
+[       OK ] DisabledTestsWarningTest.DISABLED_AlsoRunDisabledTestsFlagSuppressesWarning
+[----------] Global test environment tear-down
+BarEnvironment::TearDown() called.
+gtest_output_test_.cc:#: Failure
+Failed
+Expected non-fatal failure.
+FooEnvironment::TearDown() called.
+gtest_output_test_.cc:#: Failure
+Failed
+Expected fatal failure.
+[==========] 1 test from 1 test case ran.
+[  PASSED  ] 1 test.
+[  FAILED  ] 0 tests, listed below:
+
+ 0 FAILED TESTS

test/gtest_output_test_golden_win.txt

@@ -481,6 +481,8 @@ Expected fatal failure.
 [  FAILED  ] ExpectFailureTest.ExpectNonFatalFailureOnAllThreads
 
 36 FAILED TESTS
+  YOU HAVE 1 DISABLED TEST
+
 The non-test part of the code is expected to have 2 failures.
 
 gtest_output_test_.cc:#: error: Value of: false
@@ -542,3 +544,32 @@ Expected fatal failure.
 [  FAILED  ] LoggingTest.InterleavingLoggingAndAssertions
 
  4 FAILED TESTS
+  YOU HAVE 1 DISABLED TEST
+
+The non-test part of the code is expected to have 2 failures.
+
+gtest_output_test_.cc:#: error: Value of: false
+  Actual: false
+Expected: true
+gtest_output_test_.cc:#: error: Value of: 3
+Expected: 2
+Note: Google Test filter = *DISABLED_*
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+FooEnvironment::SetUp() called.
+BarEnvironment::SetUp() called.
+[----------] 1 test from DisabledTestsWarningTest
+[ RUN      ] DisabledTestsWarningTest.DISABLED_AlsoRunDisabledTestsFlagSuppressesWarning
+[       OK ] DisabledTestsWarningTest.DISABLED_AlsoRunDisabledTestsFlagSuppressesWarning
+[----------] Global test environment tear-down
+BarEnvironment::TearDown() called.
+gtest_output_test_.cc:#: error: Failed
+Expected non-fatal failure.
+FooEnvironment::TearDown() called.
+gtest_output_test_.cc:#: error: Failed
+Expected fatal failure.
+[==========] 1 test from 1 test case ran.
+[  PASSED  ] 1 test.
+[  FAILED  ] 0 tests, listed below:
+
+ 0 FAILED TESTS

test/gtest_unittest.cc

@@ -84,6 +84,7 @@ using testing::AssertionResult;
 using testing::AssertionSuccess;
 using testing::DoubleLE;
 using testing::FloatLE;
+using testing::GTEST_FLAG(also_run_disabled_tests);
 using testing::GTEST_FLAG(break_on_failure);
 using testing::GTEST_FLAG(catch_exceptions);
 using testing::GTEST_FLAG(death_test_use_fork);
@@ -1128,6 +1129,7 @@ class GTestFlagSaverTest : public Test {
   static void SetUpTestCase() {
     saver_ = new GTestFlagSaver;
 
+    GTEST_FLAG(also_run_disabled_tests) = false;
     GTEST_FLAG(break_on_failure) = false;
     GTEST_FLAG(catch_exceptions) = false;
     GTEST_FLAG(death_test_use_fork) = false;
@@ -1149,6 +1151,7 @@ class GTestFlagSaverTest : public Test {
   // Verifies that the Google Test flags have their default values, and then
   // modifies each of them.
   void VerifyAndModifyFlags() {
+    EXPECT_FALSE(GTEST_FLAG(also_run_disabled_tests));
     EXPECT_FALSE(GTEST_FLAG(break_on_failure));
     EXPECT_FALSE(GTEST_FLAG(catch_exceptions));
     EXPECT_STREQ("auto", GTEST_FLAG(color).c_str());
@@ -1159,6 +1162,7 @@ class GTestFlagSaverTest : public Test {
     EXPECT_FALSE(GTEST_FLAG(print_time));
     EXPECT_EQ(1, GTEST_FLAG(repeat));
 
+    GTEST_FLAG(also_run_disabled_tests) = true;
     GTEST_FLAG(break_on_failure) = true;
     GTEST_FLAG(catch_exceptions) = true;
     GTEST_FLAG(color) = "no";
@@ -4067,7 +4071,8 @@ TEST_F(SetUpTestCaseTest, Test2) {
 // The Flags struct stores a copy of all Google Test flags.
 struct Flags {
   // Constructs a Flags struct where each flag has its default value.
-  Flags() : break_on_failure(false),
+  Flags() : also_run_disabled_tests(false),
+            break_on_failure(false),
             catch_exceptions(false),
             death_test_use_fork(false),
             filter(""),
@@ -4078,6 +4083,14 @@ struct Flags {
 
   // Factory methods.
 
+  // Creates a Flags struct where the gtest_also_run_disabled_tests flag has
+  // the given value.
+  static Flags AlsoRunDisabledTests(bool also_run_disabled_tests) {
+    Flags flags;
+    flags.also_run_disabled_tests = also_run_disabled_tests;
+    return flags;
+  }
+
   // Creates a Flags struct where the gtest_break_on_failure flag has
   // the given value.
   static Flags BreakOnFailure(bool break_on_failure) {
@@ -4143,6 +4156,7 @@ struct Flags {
   }
 
   // These fields store the flag values.
+  bool also_run_disabled_tests;
   bool break_on_failure;
   bool catch_exceptions;
   bool death_test_use_fork;
@@ -4158,6 +4172,7 @@ class InitGoogleTestTest : public Test {
  protected:
   // Clears the flags before each test.
   virtual void SetUp() {
+    GTEST_FLAG(also_run_disabled_tests) = false;
     GTEST_FLAG(break_on_failure) = false;
     GTEST_FLAG(catch_exceptions) = false;
     GTEST_FLAG(death_test_use_fork) = false;
@@ -4181,6 +4196,8 @@ class InitGoogleTestTest : public Test {
 
   // Verifies that the flag values match the expected values.
   static void CheckFlags(const Flags& expected) {
+    EXPECT_EQ(expected.also_run_disabled_tests,
+              GTEST_FLAG(also_run_disabled_tests));
     EXPECT_EQ(expected.break_on_failure, GTEST_FLAG(break_on_failure));
     EXPECT_EQ(expected.catch_exceptions, GTEST_FLAG(catch_exceptions));
     EXPECT_EQ(expected.death_test_use_fork, GTEST_FLAG(death_test_use_fork));
@@ -4687,6 +4704,54 @@ TEST_F(InitGoogleTestTest, Repeat) {
   TEST_PARSING_FLAGS(argv, argv2, Flags::Repeat(1000));
 }
 
+// Tests having a --gtest_also_run_disabled_tests flag
+TEST_F(InitGoogleTestTest, AlsoRunDisabledTestsFlag) {
+    const char* argv[] = {
+      "foo.exe",
+      "--gtest_also_run_disabled_tests",
+      NULL
+    };
+
+    const char* argv2[] = {
+      "foo.exe",
+      NULL
+    };
+
+    TEST_PARSING_FLAGS(argv, argv2, Flags::AlsoRunDisabledTests(true));
+}
+
+// Tests having a --gtest_also_run_disabled_tests flag with a "true" value
+TEST_F(InitGoogleTestTest, AlsoRunDisabledTestsTrue) {
+    const char* argv[] = {
+      "foo.exe",
+      "--gtest_also_run_disabled_tests=1",
+      NULL
+    };
+
+    const char* argv2[] = {
+      "foo.exe",
+      NULL
+    };
+
+    TEST_PARSING_FLAGS(argv, argv2, Flags::AlsoRunDisabledTests(true));
+}
+
+// Tests having a --gtest_also_run_disabled_tests flag with a "false" value
+TEST_F(InitGoogleTestTest, AlsoRunDisabledTestsFalse) {
+    const char* argv[] = {
+      "foo.exe",
+      "--gtest_also_run_disabled_tests=0",
+      NULL
+    };
+
+    const char* argv2[] = {
+      "foo.exe",
+      NULL
+    };
+
+    TEST_PARSING_FLAGS(argv, argv2, Flags::AlsoRunDisabledTests(false));
+}
+
 #ifdef GTEST_OS_WINDOWS
 // Tests parsing wide strings.
 TEST_F(InitGoogleTestTest, WideStrings) {