googletest-output-test.py 12.3 KB
Newer Older
shiqian's avatar
shiqian committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
#!/usr/bin/env python
#
# Copyright 2008, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Gennadiy Civil's avatar
Gennadiy Civil committed
32
"""Tests the text output of Google C++ Testing and Mocking Framework.
shiqian's avatar
shiqian committed
33

Gennadiy Civil's avatar
 
Gennadiy Civil committed
34
35
36
37
38
To update the golden file:
googletest_output_test.py --build_dir=BUILD/DIR --gengolden
where BUILD/DIR contains the built googletest-output-test_ file.
googletest_output_test.py --gengolden
googletest_output_test.py
shiqian's avatar
shiqian committed
39
40
"""

kosak's avatar
kosak committed
41
import difflib
shiqian's avatar
shiqian committed
42
43
44
import os
import re
import sys
45
import gtest_test_utils
shiqian's avatar
shiqian committed
46
47
48
49


# The flag for generating the golden file
GENGOLDEN_FLAG = '--gengolden'
50
CATCH_EXCEPTIONS_ENV_VAR_NAME = 'GTEST_CATCH_EXCEPTIONS'
shiqian's avatar
shiqian committed
51

52
53
54
# The flag indicating stacktraces are not supported
NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support'

Gennadiy Civil's avatar
 
Gennadiy Civil committed
55
IS_LINUX = os.name == 'posix' and os.uname()[0] == 'Linux'
shiqian's avatar
shiqian committed
56
57
IS_WINDOWS = os.name == 'nt'

Gennadiy Civil's avatar
 
Gennadiy Civil committed
58
# FIXME: remove the _lin suffix.
59
GOLDEN_NAME = 'googletest-output-test-golden-lin.txt'
shiqian's avatar
shiqian committed
60

61
PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath('googletest-output-test_')
62
63

# At least one command we exercise must not have the
kosak's avatar
kosak committed
64
# 'internal_skip_environment_and_ad_hoc_tests' argument.
65
66
67
68
COMMAND_LIST_TESTS = ({}, [PROGRAM_PATH, '--gtest_list_tests'])
COMMAND_WITH_COLOR = ({}, [PROGRAM_PATH, '--gtest_color=yes'])
COMMAND_WITH_TIME = ({}, [PROGRAM_PATH,
                          '--gtest_print_time',
kosak's avatar
kosak committed
69
                          'internal_skip_environment_and_ad_hoc_tests',
70
71
72
73
                          '--gtest_filter=FatalFailureTest.*:LoggingTest.*'])
COMMAND_WITH_DISABLED = (
    {}, [PROGRAM_PATH,
         '--gtest_also_run_disabled_tests',
kosak's avatar
kosak committed
74
         'internal_skip_environment_and_ad_hoc_tests',
75
76
77
78
         '--gtest_filter=*DISABLED_*'])
COMMAND_WITH_SHARDING = (
    {'GTEST_SHARD_INDEX': '1', 'GTEST_TOTAL_SHARDS': '2'},
    [PROGRAM_PATH,
kosak's avatar
kosak committed
79
     'internal_skip_environment_and_ad_hoc_tests',
80
     '--gtest_filter=PassingTest.*'])
shiqian's avatar
shiqian committed
81

82
GOLDEN_PATH = os.path.join(gtest_test_utils.GetSourceDir(), GOLDEN_NAME)
shiqian's avatar
shiqian committed
83

84

shiqian's avatar
shiqian committed
85
86
87
88
89
90
def ToUnixLineEnding(s):
  """Changes all Windows/Mac line endings in s to UNIX line endings."""

  return s.replace('\r\n', '\n').replace('\r', '\n')


91
def RemoveLocations(test_output):
shiqian's avatar
shiqian committed
92
93
94
  """Removes all file location info from a Google Test program's output.

  Args:
95
       test_output:  the output of a Google Test program.
shiqian's avatar
shiqian committed
96
97
98

  Returns:
       output with all file location info (in the form of
99
100
       'DIRECTORY/FILE_NAME:LINE_NUMBER: 'or
       'DIRECTORY\\FILE_NAME(LINE_NUMBER): ') replaced by
shiqian's avatar
shiqian committed
101
102
103
       'FILE_NAME:#: '.
  """

104
  return re.sub(r'.*[/\\]((googletest-output-test_|gtest).cc)(\:\d+|\(\d+\))\: ',
105
                r'\1:#: ', test_output)
shiqian's avatar
shiqian committed
106
107


108
def RemoveStackTraceDetails(output):
shiqian's avatar
shiqian committed
109
110
111
112
113
114
115
  """Removes all stack traces from a Google Test program's output."""

  # *? means "find the shortest string that matches".
  return re.sub(r'Stack trace:(.|\n)*?\n\n',
                'Stack trace: (omitted)\n\n', output)


116
117
118
119
120
121
122
def RemoveStackTraces(output):
  """Removes all traces of stack traces from a Google Test program's output."""

  # *? means "find the shortest string that matches".
  return re.sub(r'Stack trace:(.|\n)*?\n\n', '', output)


123
124
125
126
127
128
def RemoveTime(output):
  """Removes all time information from a Google Test program's output."""

  return re.sub(r'\(\d+ ms', '(? ms', output)


129
130
131
132
133
134
135
136
137
138
139
140
141
142
def RemoveTypeInfoDetails(test_output):
  """Removes compiler-specific type info from Google Test program's output.

  Args:
       test_output:  the output of a Google Test program.

  Returns:
       output with type information normalized to canonical form.
  """

  # some compilers output the name of type 'unsigned int' as 'unsigned'
  return re.sub(r'unsigned int', 'unsigned', test_output)


143
144
145
146
147
148
149
150
151
152
153
154
155
156
def NormalizeToCurrentPlatform(test_output):
  """Normalizes platform specific output details for easier comparison."""

  if IS_WINDOWS:
    # Removes the color information that is not present on Windows.
    test_output = re.sub('\x1b\\[(0;3\d)?m', '', test_output)
    # Changes failure message headers into the Windows format.
    test_output = re.sub(r': Failure\n', r': error: ', test_output)
    # Changes file(line_number) to file:line_number.
    test_output = re.sub(r'((\w|\.)+)\((\d+)\):', r'\1:\3:', test_output)

  return test_output


157
158
159
def RemoveTestCounts(output):
  """Removes test counts from a Google Test program's output."""

160
  output = re.sub(r'\d+ tests?, listed below',
161
162
163
                  '? tests, listed below', output)
  output = re.sub(r'\d+ FAILED TESTS',
                  '? FAILED TESTS', output)
164
  output = re.sub(r'\d+ tests? from \d+ test cases?',
165
                  '? tests from ? test cases', output)
166
  output = re.sub(r'\d+ tests? from ([a-zA-Z_])',
167
                  r'? tests from \1', output)
168
  return re.sub(r'\d+ tests?\.', '? tests.', output)
169
170


171
def RemoveMatchingTests(test_output, pattern):
172
  """Removes output of specified tests from a Google Test program's output.
173

174
175
  This function strips not only the beginning and the end of a test but also
  all output in between.
176
177
178

  Args:
    test_output:       A string containing the test output.
179
180
    pattern:           A regex string that matches names of test cases or
                       tests to remove.
181
182

  Returns:
183
    Contents of test_output with tests whose names match pattern removed.
184
185
186
  """

  test_output = re.sub(
187
      r'.*\[ RUN      \] .*%s(.|\n)*?\[(  FAILED  |       OK )\] .*%s.*\n' % (
188
189
190
191
          pattern, pattern),
      '',
      test_output)
  return re.sub(r'.*%s.*\n' % pattern, '', test_output)
192
193


shiqian's avatar
shiqian committed
194
def NormalizeOutput(output):
195
  """Normalizes output (the output of googletest-output-test_.exe)."""
shiqian's avatar
shiqian committed
196
197
198

  output = ToUnixLineEnding(output)
  output = RemoveLocations(output)
199
  output = RemoveStackTraceDetails(output)
200
  output = RemoveTime(output)
shiqian's avatar
shiqian committed
201
202
203
  return output


204
205
def GetShellCommandOutput(env_cmd):
  """Runs a command in a sub-process, and returns its output in a string.
shiqian's avatar
shiqian committed
206
207

  Args:
208
209
210
    env_cmd: The shell command. A 2-tuple where element 0 is a dict of extra
             environment variables to set, and element 1 is a string with
             the command and any flags.
shiqian's avatar
shiqian committed
211

212
213
  Returns:
    A string with the command's combined standard and diagnostic output.
shiqian's avatar
shiqian committed
214
215
216
  """

  # Spawns cmd in a sub-process, and gets its standard I/O file objects.
217
  # Set and save the environment properly.
218
219
  environ = os.environ.copy()
  environ.update(env_cmd[0])
vladlosev's avatar
vladlosev committed
220
  p = gtest_test_utils.Subprocess(env_cmd[1], env=environ)
shiqian's avatar
shiqian committed
221

222
  return p.output
shiqian's avatar
shiqian committed
223
224


225
def GetCommandOutput(env_cmd):
shiqian's avatar
shiqian committed
226
227
228
229
  """Runs a command and returns its output with all file location
  info stripped off.

  Args:
230
231
232
    env_cmd:  The shell command. A 2-tuple where element 0 is a dict of extra
              environment variables to set, and element 1 is a string with
              the command and any flags.
shiqian's avatar
shiqian committed
233
234
235
  """

  # Disables exception pop-ups on Windows.
236
237
238
239
  environ, cmdline = env_cmd
  environ = dict(environ)  # Ensures we are modifying a copy.
  environ[CATCH_EXCEPTIONS_ENV_VAR_NAME] = '1'
  return NormalizeOutput(GetShellCommandOutput((environ, cmdline)))
240
241
242
243
244
245
246
247
248


def GetOutputOfAllCommands():
  """Returns concatenated output from several representative commands."""

  return (GetCommandOutput(COMMAND_WITH_COLOR) +
          GetCommandOutput(COMMAND_WITH_TIME) +
          GetCommandOutput(COMMAND_WITH_DISABLED) +
          GetCommandOutput(COMMAND_WITH_SHARDING))
shiqian's avatar
shiqian committed
249
250


251
test_list = GetShellCommandOutput(COMMAND_LIST_TESTS)
252
253
SUPPORTS_DEATH_TESTS = 'DeathTest' in test_list
SUPPORTS_TYPED_TESTS = 'TypedTest' in test_list
254
SUPPORTS_THREADS = 'ExpectFailureWithThreadsTest' in test_list
255
SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv
256

257
258
CAN_GENERATE_GOLDEN_FILE = (SUPPORTS_DEATH_TESTS and
                            SUPPORTS_TYPED_TESTS and
259
                            SUPPORTS_THREADS and
260
                            SUPPORTS_STACK_TRACES)
261

262
class GTestOutputTest(gtest_test_utils.TestCase):
263
264
265
266
267
  def RemoveUnsupportedTests(self, test_output):
    if not SUPPORTS_DEATH_TESTS:
      test_output = RemoveMatchingTests(test_output, 'DeathTest')
    if not SUPPORTS_TYPED_TESTS:
      test_output = RemoveMatchingTests(test_output, 'TypedTest')
268
269
      test_output = RemoveMatchingTests(test_output, 'TypedDeathTest')
      test_output = RemoveMatchingTests(test_output, 'TypeParamDeathTest')
270
271
272
273
274
275
276
277
278
279
    if not SUPPORTS_THREADS:
      test_output = RemoveMatchingTests(test_output,
                                        'ExpectFailureWithThreadsTest')
      test_output = RemoveMatchingTests(test_output,
                                        'ScopedFakeTestPartResultReporterTest')
      test_output = RemoveMatchingTests(test_output,
                                        'WorksConcurrently')
    if not SUPPORTS_STACK_TRACES:
      test_output = RemoveStackTraces(test_output)

280
281
    return test_output

shiqian's avatar
shiqian committed
282
  def testOutput(self):
283
    output = GetOutputOfAllCommands()
284

Gennadiy Civil's avatar
 
Gennadiy Civil committed
285
    golden_file = open(GOLDEN_PATH, 'rb')
286
287
288
289
290
    # A mis-configured source control system can cause \r appear in EOL
    # sequences when we read the golden file irrespective of an operating
    # system used. Therefore, we need to strip those \r's from newlines
    # unconditionally.
    golden = ToUnixLineEnding(golden_file.read())
shiqian's avatar
shiqian committed
291
292
    golden_file.close()

293
    # We want the test to pass regardless of certain features being
294
    # supported or not.
295
296
297
298
299

    # We still have to remove type name specifics in all cases.
    normalized_actual = RemoveTypeInfoDetails(output)
    normalized_golden = RemoveTypeInfoDetails(golden)

300
    if CAN_GENERATE_GOLDEN_FILE:
kosak's avatar
kosak committed
301
302
303
304
305
      self.assertEqual(normalized_golden, normalized_actual,
                       '\n'.join(difflib.unified_diff(
                           normalized_golden.split('\n'),
                           normalized_actual.split('\n'),
                           'golden', 'actual')))
306
    else:
307
308
309
310
      normalized_actual = NormalizeToCurrentPlatform(
          RemoveTestCounts(normalized_actual))
      normalized_golden = NormalizeToCurrentPlatform(
          RemoveTestCounts(self.RemoveUnsupportedTests(normalized_golden)))
311

312
313
314
315
      # This code is very handy when debugging golden file differences:
      if os.getenv('DEBUG_GTEST_OUTPUT_TEST'):
        open(os.path.join(
            gtest_test_utils.GetSourceDir(),
316
            '_googletest-output-test_normalized_actual.txt'), 'wb').write(
317
318
319
                normalized_actual)
        open(os.path.join(
            gtest_test_utils.GetSourceDir(),
320
            '_googletest-output-test_normalized_golden.txt'), 'wb').write(
321
                normalized_golden)
322

323
      self.assertEqual(normalized_golden, normalized_actual)
shiqian's avatar
shiqian committed
324
325
326


if __name__ == '__main__':
327
328
329
330
331
  if NO_STACKTRACE_SUPPORT_FLAG in sys.argv:
    # unittest.main() can't handle unknown flags
    sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)

  if GENGOLDEN_FLAG in sys.argv:
332
    if CAN_GENERATE_GOLDEN_FILE:
333
334
335
336
337
      output = GetOutputOfAllCommands()
      golden_file = open(GOLDEN_PATH, 'wb')
      golden_file.write(output)
      golden_file.close()
    else:
338
339
      message = (
          """Unable to write a golden file when compiled in an environment
Gennadiy Civil's avatar
 
Gennadiy Civil committed
340
341
342
that does not support all the required features (death tests,
typed tests, stack traces, and multiple threads).
Please build this test and generate the golden file using Blaze on Linux.""")
343
344

      sys.stderr.write(message)
345
      sys.exit(1)
shiqian's avatar
shiqian committed
346
347
  else:
    gtest_test_utils.Main()