gtest_output_test.py 12.4 KB
Newer Older
shiqian's avatar
shiqian committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
#!/usr/bin/env python
#
# Copyright 2008, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Gennadiy Civil's avatar
Gennadiy Civil committed
32
"""Tests the text output of Google C++ Testing and Mocking Framework.
shiqian's avatar
shiqian committed
33

Gennadiy Civil's avatar
 
Gennadiy Civil committed
34

shiqian's avatar
shiqian committed
35
SYNOPSIS
vladlosev's avatar
vladlosev committed
36
       gtest_output_test.py --build_dir=BUILD/DIR --gengolden
37
         # where BUILD/DIR contains the built gtest_output_test_ file.
shiqian's avatar
shiqian committed
38
39
40
41
42
43
       gtest_output_test.py --gengolden
       gtest_output_test.py
"""

__author__ = 'wan@google.com (Zhanyong Wan)'

kosak's avatar
kosak committed
44
import difflib
shiqian's avatar
shiqian committed
45
46
47
import os
import re
import sys
48
import gtest_test_utils
shiqian's avatar
shiqian committed
49
50
51
52


# The flag for generating the golden file
GENGOLDEN_FLAG = '--gengolden'
53
CATCH_EXCEPTIONS_ENV_VAR_NAME = 'GTEST_CATCH_EXCEPTIONS'
shiqian's avatar
shiqian committed
54

55
56
57
# The flag indicating stacktraces are not supported
NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support'

Gennadiy Civil's avatar
 
Gennadiy Civil committed
58
IS_LINUX = os.name == 'posix' and os.uname()[0] == 'Linux'
shiqian's avatar
shiqian committed
59
60
IS_WINDOWS = os.name == 'nt'

61
62
# TODO(vladl@google.com): remove the _lin suffix.
GOLDEN_NAME = 'gtest_output_test_golden_lin.txt'
shiqian's avatar
shiqian committed
63

64
PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath('gtest_output_test_')
65
66

# At least one command we exercise must not have the
kosak's avatar
kosak committed
67
# 'internal_skip_environment_and_ad_hoc_tests' argument.
68
69
70
71
COMMAND_LIST_TESTS = ({}, [PROGRAM_PATH, '--gtest_list_tests'])
COMMAND_WITH_COLOR = ({}, [PROGRAM_PATH, '--gtest_color=yes'])
COMMAND_WITH_TIME = ({}, [PROGRAM_PATH,
                          '--gtest_print_time',
kosak's avatar
kosak committed
72
                          'internal_skip_environment_and_ad_hoc_tests',
73
74
75
76
                          '--gtest_filter=FatalFailureTest.*:LoggingTest.*'])
COMMAND_WITH_DISABLED = (
    {}, [PROGRAM_PATH,
         '--gtest_also_run_disabled_tests',
kosak's avatar
kosak committed
77
         'internal_skip_environment_and_ad_hoc_tests',
78
79
80
81
         '--gtest_filter=*DISABLED_*'])
COMMAND_WITH_SHARDING = (
    {'GTEST_SHARD_INDEX': '1', 'GTEST_TOTAL_SHARDS': '2'},
    [PROGRAM_PATH,
kosak's avatar
kosak committed
82
     'internal_skip_environment_and_ad_hoc_tests',
83
     '--gtest_filter=PassingTest.*'])
shiqian's avatar
shiqian committed
84

85
GOLDEN_PATH = os.path.join(gtest_test_utils.GetSourceDir(), GOLDEN_NAME)
shiqian's avatar
shiqian committed
86

87

shiqian's avatar
shiqian committed
88
89
90
91
92
93
def ToUnixLineEnding(s):
  """Changes all Windows/Mac line endings in s to UNIX line endings."""

  return s.replace('\r\n', '\n').replace('\r', '\n')


94
def RemoveLocations(test_output):
shiqian's avatar
shiqian committed
95
96
97
  """Removes all file location info from a Google Test program's output.

  Args:
98
       test_output:  the output of a Google Test program.
shiqian's avatar
shiqian committed
99
100
101

  Returns:
       output with all file location info (in the form of
102
103
       'DIRECTORY/FILE_NAME:LINE_NUMBER: 'or
       'DIRECTORY\\FILE_NAME(LINE_NUMBER): ') replaced by
shiqian's avatar
shiqian committed
104
105
106
       'FILE_NAME:#: '.
  """

107
108
  return re.sub(r'.*[/\\]((gtest_output_test_|gtest).cc)(\:\d+|\(\d+\))\: ',
                r'\1:#: ', test_output)
shiqian's avatar
shiqian committed
109
110


111
def RemoveStackTraceDetails(output):
shiqian's avatar
shiqian committed
112
113
114
115
116
117
118
  """Removes all stack traces from a Google Test program's output."""

  # *? means "find the shortest string that matches".
  return re.sub(r'Stack trace:(.|\n)*?\n\n',
                'Stack trace: (omitted)\n\n', output)


119
120
121
122
123
124
125
def RemoveStackTraces(output):
  """Removes all traces of stack traces from a Google Test program's output."""

  # *? means "find the shortest string that matches".
  return re.sub(r'Stack trace:(.|\n)*?\n\n', '', output)


126
127
128
129
130
131
def RemoveTime(output):
  """Removes all time information from a Google Test program's output."""

  return re.sub(r'\(\d+ ms', '(? ms', output)


132
133
134
135
136
137
138
139
140
141
142
143
144
145
def RemoveTypeInfoDetails(test_output):
  """Removes compiler-specific type info from Google Test program's output.

  Args:
       test_output:  the output of a Google Test program.

  Returns:
       output with type information normalized to canonical form.
  """

  # some compilers output the name of type 'unsigned int' as 'unsigned'
  return re.sub(r'unsigned int', 'unsigned', test_output)


146
147
148
149
150
151
152
153
154
155
156
157
158
159
def NormalizeToCurrentPlatform(test_output):
  """Normalizes platform specific output details for easier comparison."""

  if IS_WINDOWS:
    # Removes the color information that is not present on Windows.
    test_output = re.sub('\x1b\\[(0;3\d)?m', '', test_output)
    # Changes failure message headers into the Windows format.
    test_output = re.sub(r': Failure\n', r': error: ', test_output)
    # Changes file(line_number) to file:line_number.
    test_output = re.sub(r'((\w|\.)+)\((\d+)\):', r'\1:\3:', test_output)

  return test_output


160
161
162
def RemoveTestCounts(output):
  """Removes test counts from a Google Test program's output."""

163
  output = re.sub(r'\d+ tests?, listed below',
164
165
166
                  '? tests, listed below', output)
  output = re.sub(r'\d+ FAILED TESTS',
                  '? FAILED TESTS', output)
167
  output = re.sub(r'\d+ tests? from \d+ test cases?',
168
                  '? tests from ? test cases', output)
169
  output = re.sub(r'\d+ tests? from ([a-zA-Z_])',
170
                  r'? tests from \1', output)
171
  return re.sub(r'\d+ tests?\.', '? tests.', output)
172
173


174
def RemoveMatchingTests(test_output, pattern):
175
  """Removes output of specified tests from a Google Test program's output.
176

177
178
  This function strips not only the beginning and the end of a test but also
  all output in between.
179
180
181

  Args:
    test_output:       A string containing the test output.
182
183
    pattern:           A regex string that matches names of test cases or
                       tests to remove.
184
185

  Returns:
186
    Contents of test_output with tests whose names match pattern removed.
187
188
189
  """

  test_output = re.sub(
190
      r'.*\[ RUN      \] .*%s(.|\n)*?\[(  FAILED  |       OK )\] .*%s.*\n' % (
191
192
193
194
          pattern, pattern),
      '',
      test_output)
  return re.sub(r'.*%s.*\n' % pattern, '', test_output)
195
196


shiqian's avatar
shiqian committed
197
198
199
200
201
def NormalizeOutput(output):
  """Normalizes output (the output of gtest_output_test_.exe)."""

  output = ToUnixLineEnding(output)
  output = RemoveLocations(output)
202
  output = RemoveStackTraceDetails(output)
203
  output = RemoveTime(output)
shiqian's avatar
shiqian committed
204
205
206
  return output


207
208
def GetShellCommandOutput(env_cmd):
  """Runs a command in a sub-process, and returns its output in a string.
shiqian's avatar
shiqian committed
209
210

  Args:
211
212
213
    env_cmd: The shell command. A 2-tuple where element 0 is a dict of extra
             environment variables to set, and element 1 is a string with
             the command and any flags.
shiqian's avatar
shiqian committed
214

215
216
  Returns:
    A string with the command's combined standard and diagnostic output.
shiqian's avatar
shiqian committed
217
218
219
  """

  # Spawns cmd in a sub-process, and gets its standard I/O file objects.
220
  # Set and save the environment properly.
221
222
  environ = os.environ.copy()
  environ.update(env_cmd[0])
vladlosev's avatar
vladlosev committed
223
  p = gtest_test_utils.Subprocess(env_cmd[1], env=environ)
shiqian's avatar
shiqian committed
224

225
  return p.output
shiqian's avatar
shiqian committed
226
227


228
def GetCommandOutput(env_cmd):
shiqian's avatar
shiqian committed
229
230
231
232
  """Runs a command and returns its output with all file location
  info stripped off.

  Args:
233
234
235
    env_cmd:  The shell command. A 2-tuple where element 0 is a dict of extra
              environment variables to set, and element 1 is a string with
              the command and any flags.
shiqian's avatar
shiqian committed
236
237
238
  """

  # Disables exception pop-ups on Windows.
239
240
241
242
  environ, cmdline = env_cmd
  environ = dict(environ)  # Ensures we are modifying a copy.
  environ[CATCH_EXCEPTIONS_ENV_VAR_NAME] = '1'
  return NormalizeOutput(GetShellCommandOutput((environ, cmdline)))
243
244
245
246
247
248
249
250
251


def GetOutputOfAllCommands():
  """Returns concatenated output from several representative commands."""

  return (GetCommandOutput(COMMAND_WITH_COLOR) +
          GetCommandOutput(COMMAND_WITH_TIME) +
          GetCommandOutput(COMMAND_WITH_DISABLED) +
          GetCommandOutput(COMMAND_WITH_SHARDING))
shiqian's avatar
shiqian committed
252
253


254
test_list = GetShellCommandOutput(COMMAND_LIST_TESTS)
255
256
SUPPORTS_DEATH_TESTS = 'DeathTest' in test_list
SUPPORTS_TYPED_TESTS = 'TypedTest' in test_list
257
SUPPORTS_THREADS = 'ExpectFailureWithThreadsTest' in test_list
258
SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv
259

260
261
CAN_GENERATE_GOLDEN_FILE = (SUPPORTS_DEATH_TESTS and
                            SUPPORTS_TYPED_TESTS and
262
                            SUPPORTS_THREADS and
263
                            SUPPORTS_STACK_TRACES)
264

265
class GTestOutputTest(gtest_test_utils.TestCase):
266
267
268
269
270
  def RemoveUnsupportedTests(self, test_output):
    if not SUPPORTS_DEATH_TESTS:
      test_output = RemoveMatchingTests(test_output, 'DeathTest')
    if not SUPPORTS_TYPED_TESTS:
      test_output = RemoveMatchingTests(test_output, 'TypedTest')
271
272
      test_output = RemoveMatchingTests(test_output, 'TypedDeathTest')
      test_output = RemoveMatchingTests(test_output, 'TypeParamDeathTest')
273
274
275
276
277
278
279
280
281
282
    if not SUPPORTS_THREADS:
      test_output = RemoveMatchingTests(test_output,
                                        'ExpectFailureWithThreadsTest')
      test_output = RemoveMatchingTests(test_output,
                                        'ScopedFakeTestPartResultReporterTest')
      test_output = RemoveMatchingTests(test_output,
                                        'WorksConcurrently')
    if not SUPPORTS_STACK_TRACES:
      test_output = RemoveStackTraces(test_output)

283
284
    return test_output

shiqian's avatar
shiqian committed
285
  def testOutput(self):
286
    output = GetOutputOfAllCommands()
287

Gennadiy Civil's avatar
 
Gennadiy Civil committed
288
    golden_file = open(GOLDEN_PATH, 'rb')
289
290
291
292
293
    # A mis-configured source control system can cause \r appear in EOL
    # sequences when we read the golden file irrespective of an operating
    # system used. Therefore, we need to strip those \r's from newlines
    # unconditionally.
    golden = ToUnixLineEnding(golden_file.read())
shiqian's avatar
shiqian committed
294
295
    golden_file.close()

296
    # We want the test to pass regardless of certain features being
297
    # supported or not.
298
299
300
301
302

    # We still have to remove type name specifics in all cases.
    normalized_actual = RemoveTypeInfoDetails(output)
    normalized_golden = RemoveTypeInfoDetails(golden)

303
    if CAN_GENERATE_GOLDEN_FILE:
kosak's avatar
kosak committed
304
305
306
307
308
      self.assertEqual(normalized_golden, normalized_actual,
                       '\n'.join(difflib.unified_diff(
                           normalized_golden.split('\n'),
                           normalized_actual.split('\n'),
                           'golden', 'actual')))
309
    else:
310
311
312
313
      normalized_actual = NormalizeToCurrentPlatform(
          RemoveTestCounts(normalized_actual))
      normalized_golden = NormalizeToCurrentPlatform(
          RemoveTestCounts(self.RemoveUnsupportedTests(normalized_golden)))
314

315
316
317
318
319
320
321
322
323
324
      # This code is very handy when debugging golden file differences:
      if os.getenv('DEBUG_GTEST_OUTPUT_TEST'):
        open(os.path.join(
            gtest_test_utils.GetSourceDir(),
            '_gtest_output_test_normalized_actual.txt'), 'wb').write(
                normalized_actual)
        open(os.path.join(
            gtest_test_utils.GetSourceDir(),
            '_gtest_output_test_normalized_golden.txt'), 'wb').write(
                normalized_golden)
325

326
      self.assertEqual(normalized_golden, normalized_actual)
shiqian's avatar
shiqian committed
327
328
329


if __name__ == '__main__':
330
331
332
333
334
  if NO_STACKTRACE_SUPPORT_FLAG in sys.argv:
    # unittest.main() can't handle unknown flags
    sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)

  if GENGOLDEN_FLAG in sys.argv:
335
    if CAN_GENERATE_GOLDEN_FILE:
336
337
338
339
340
      output = GetOutputOfAllCommands()
      golden_file = open(GOLDEN_PATH, 'wb')
      golden_file.write(output)
      golden_file.close()
    else:
341
342
      message = (
          """Unable to write a golden file when compiled in an environment
Gennadiy Civil's avatar
 
Gennadiy Civil committed
343
344
345
that does not support all the required features (death tests,
typed tests, stack traces, and multiple threads).
Please build this test and generate the golden file using Blaze on Linux.""")
346
347

      sys.stderr.write(message)
348
      sys.exit(1)
shiqian's avatar
shiqian committed
349
350
  else:
    gtest_test_utils.Main()