3 # Copyright 2008, Google Inc.
6 # Redistribution and use in source and binary forms, with or without
7 # modification, are permitted provided that the following conditions are
10 # * Redistributions of source code must retain the above copyright
11 # notice, this list of conditions and the following disclaimer.
12 # * Redistributions in binary form must reproduce the above
13 # copyright notice, this list of conditions and the following disclaimer
14 # in the documentation and/or other materials provided with the
16 # * Neither the name of Google Inc. nor the names of its
17 # contributors may be used to endorse or promote products derived from
18 # this software without specific prior written permission.
20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 """Tests the text output of Google C++ Testing Framework.
35 gtest_output_test.py --build_dir=BUILD/DIR --gengolden
36 # where BUILD/DIR contains the built gtest_output_test_ file.
37 gtest_output_test.py --gengolden
41 __author__
= 'wan@google.com (Zhanyong Wan)'
46 import gtest_test_utils
49 # The flag for generating the golden file
50 GENGOLDEN_FLAG
= '--gengolden'
51 CATCH_EXCEPTIONS_ENV_VAR_NAME
= 'GTEST_CATCH_EXCEPTIONS'
53 IS_WINDOWS
= os
.name
== 'nt'
55 # TODO(vladl@google.com): remove the _lin suffix.
56 GOLDEN_NAME
= 'gtest_output_test_golden_lin.txt'
58 PROGRAM_PATH
= gtest_test_utils
.GetTestExecutablePath('gtest_output_test_')
60 # At least one command we exercise must not have the
61 # --gtest_internal_skip_environment_and_ad_hoc_tests flag.
62 COMMAND_LIST_TESTS
= ({}, [PROGRAM_PATH
, '--gtest_list_tests'])
63 COMMAND_WITH_COLOR
= ({}, [PROGRAM_PATH
, '--gtest_color=yes'])
64 COMMAND_WITH_TIME
= ({}, [PROGRAM_PATH
,
66 '--gtest_internal_skip_environment_and_ad_hoc_tests',
67 '--gtest_filter=FatalFailureTest.*:LoggingTest.*'])
68 COMMAND_WITH_DISABLED
= (
70 '--gtest_also_run_disabled_tests',
71 '--gtest_internal_skip_environment_and_ad_hoc_tests',
72 '--gtest_filter=*DISABLED_*'])
73 COMMAND_WITH_SHARDING
= (
74 {'GTEST_SHARD_INDEX': '1', 'GTEST_TOTAL_SHARDS': '2'},
76 '--gtest_internal_skip_environment_and_ad_hoc_tests',
77 '--gtest_filter=PassingTest.*'])
79 GOLDEN_PATH
= os
.path
.join(gtest_test_utils
.GetSourceDir(), GOLDEN_NAME
)
82 def ToUnixLineEnding(s
):
83 """Changes all Windows/Mac line endings in s to UNIX line endings."""
85 return s
.replace('\r\n', '\n').replace('\r', '\n')
88 def RemoveLocations(test_output
):
89 """Removes all file location info from a Google Test program's output.
92 test_output: the output of a Google Test program.
95 output with all file location info (in the form of
96 'DIRECTORY/FILE_NAME:LINE_NUMBER: 'or
97 'DIRECTORY\\FILE_NAME(LINE_NUMBER): ') replaced by
101 return re
.sub(r
'.*[/\\](.+)(\:\d+|\(\d+\))\: ', r
'\1:#: ', test_output
)
104 def RemoveStackTraceDetails(output
):
105 """Removes all stack traces from a Google Test program's output."""
107 # *? means "find the shortest string that matches".
108 return re
.sub(r
'Stack trace:(.|\n)*?\n\n',
109 'Stack trace: (omitted)\n\n', output
)
112 def RemoveStackTraces(output
):
113 """Removes all traces of stack traces from a Google Test program's output."""
115 # *? means "find the shortest string that matches".
116 return re
.sub(r
'Stack trace:(.|\n)*?\n\n', '', output
)
119 def RemoveTime(output
):
120 """Removes all time information from a Google Test program's output."""
122 return re
.sub(r
'\(\d+ ms', '(? ms', output
)
125 def RemoveTypeInfoDetails(test_output
):
126 """Removes compiler-specific type info from Google Test program's output.
129 test_output: the output of a Google Test program.
132 output with type information normalized to canonical form.
135 # some compilers output the name of type 'unsigned int' as 'unsigned'
136 return re
.sub(r
'unsigned int', 'unsigned', test_output
)
139 def NormalizeToCurrentPlatform(test_output
):
140 """Normalizes platform specific output details for easier comparison."""
143 # Removes the color information that is not present on Windows.
144 test_output
= re
.sub('\x1b\\[(0;3\d)?m', '', test_output
)
145 # Changes failure message headers into the Windows format.
146 test_output
= re
.sub(r
': Failure\n', r
': error: ', test_output
)
147 # Changes file(line_number) to file:line_number.
148 test_output
= re
.sub(r
'((\w|\.)+)\((\d+)\):', r
'\1:\3:', test_output
)
153 def RemoveTestCounts(output
):
154 """Removes test counts from a Google Test program's output."""
156 output
= re
.sub(r
'\d+ tests?, listed below',
157 '? tests, listed below', output
)
158 output
= re
.sub(r
'\d+ FAILED TESTS',
159 '? FAILED TESTS', output
)
160 output
= re
.sub(r
'\d+ tests? from \d+ test cases?',
161 '? tests from ? test cases', output
)
162 output
= re
.sub(r
'\d+ tests? from ([a-zA-Z_])',
163 r
'? tests from \1', output
)
164 return re
.sub(r
'\d+ tests?\.', '? tests.', output
)
167 def RemoveMatchingTests(test_output
, pattern
):
168 """Removes output of specified tests from a Google Test program's output.
170 This function strips not only the beginning and the end of a test but also
171 all output in between.
174 test_output: A string containing the test output.
175 pattern: A regex string that matches names of test cases or
179 Contents of test_output with tests whose names match pattern removed.
182 test_output
= re
.sub(
183 r
'.*\[ RUN \] .*%s(.|\n)*?\[( FAILED | OK )\] .*%s.*\n' % (
187 return re
.sub(r
'.*%s.*\n' % pattern
, '', test_output
)
190 def NormalizeOutput(output
):
191 """Normalizes output (the output of gtest_output_test_.exe)."""
193 output
= ToUnixLineEnding(output
)
194 output
= RemoveLocations(output
)
195 output
= RemoveStackTraceDetails(output
)
196 output
= RemoveTime(output
)
200 def GetShellCommandOutput(env_cmd
):
201 """Runs a command in a sub-process, and returns its output in a string.
204 env_cmd: The shell command. A 2-tuple where element 0 is a dict of extra
205 environment variables to set, and element 1 is a string with
206 the command and any flags.
209 A string with the command's combined standard and diagnostic output.
212 # Spawns cmd in a sub-process, and gets its standard I/O file objects.
213 # Set and save the environment properly.
214 environ
= os
.environ
.copy()
215 environ
.update(env_cmd
[0])
216 p
= gtest_test_utils
.Subprocess(env_cmd
[1], env
=environ
)
221 def GetCommandOutput(env_cmd
):
222 """Runs a command and returns its output with all file location
226 env_cmd: The shell command. A 2-tuple where element 0 is a dict of extra
227 environment variables to set, and element 1 is a string with
228 the command and any flags.
231 # Disables exception pop-ups on Windows.
232 environ
, cmdline
= env_cmd
233 environ
= dict(environ
) # Ensures we are modifying a copy.
234 environ
[CATCH_EXCEPTIONS_ENV_VAR_NAME
] = '1'
235 return NormalizeOutput(GetShellCommandOutput((environ
, cmdline
)))
238 def GetOutputOfAllCommands():
239 """Returns concatenated output from several representative commands."""
241 return (GetCommandOutput(COMMAND_WITH_COLOR
) +
242 GetCommandOutput(COMMAND_WITH_TIME
) +
243 GetCommandOutput(COMMAND_WITH_DISABLED
) +
244 GetCommandOutput(COMMAND_WITH_SHARDING
))
247 test_list
= GetShellCommandOutput(COMMAND_LIST_TESTS
)
248 SUPPORTS_DEATH_TESTS
= 'DeathTest' in test_list
249 SUPPORTS_TYPED_TESTS
= 'TypedTest' in test_list
250 SUPPORTS_THREADS
= 'ExpectFailureWithThreadsTest' in test_list
251 SUPPORTS_STACK_TRACES
= False
253 CAN_GENERATE_GOLDEN_FILE
= (SUPPORTS_DEATH_TESTS
and
254 SUPPORTS_TYPED_TESTS
and
258 class GTestOutputTest(gtest_test_utils
.TestCase
):
259 def RemoveUnsupportedTests(self
, test_output
):
260 if not SUPPORTS_DEATH_TESTS
:
261 test_output
= RemoveMatchingTests(test_output
, 'DeathTest')
262 if not SUPPORTS_TYPED_TESTS
:
263 test_output
= RemoveMatchingTests(test_output
, 'TypedTest')
264 test_output
= RemoveMatchingTests(test_output
, 'TypedDeathTest')
265 test_output
= RemoveMatchingTests(test_output
, 'TypeParamDeathTest')
266 if not SUPPORTS_THREADS
:
267 test_output
= RemoveMatchingTests(test_output
,
268 'ExpectFailureWithThreadsTest')
269 test_output
= RemoveMatchingTests(test_output
,
270 'ScopedFakeTestPartResultReporterTest')
271 test_output
= RemoveMatchingTests(test_output
,
273 if not SUPPORTS_STACK_TRACES
:
274 test_output
= RemoveStackTraces(test_output
)
278 def testOutput(self
):
279 output
= GetOutputOfAllCommands()
281 golden_file
= open(GOLDEN_PATH
, 'rb')
282 # A mis-configured source control system can cause \r appear in EOL
283 # sequences when we read the golden file irrespective of an operating
284 # system used. Therefore, we need to strip those \r's from newlines
286 golden
= ToUnixLineEnding(golden_file
.read())
289 # We want the test to pass regardless of certain features being
292 # We still have to remove type name specifics in all cases.
293 normalized_actual
= RemoveTypeInfoDetails(output
)
294 normalized_golden
= RemoveTypeInfoDetails(golden
)
296 if CAN_GENERATE_GOLDEN_FILE
:
297 self
.assertEqual(normalized_golden
, normalized_actual
)
299 normalized_actual
= NormalizeToCurrentPlatform(
300 RemoveTestCounts(normalized_actual
))
301 normalized_golden
= NormalizeToCurrentPlatform(
302 RemoveTestCounts(self
.RemoveUnsupportedTests(normalized_golden
)))
304 # This code is very handy when debugging golden file differences:
305 if os
.getenv('DEBUG_GTEST_OUTPUT_TEST'):
307 gtest_test_utils
.GetSourceDir(),
308 '_gtest_output_test_normalized_actual.txt'), 'wb').write(
311 gtest_test_utils
.GetSourceDir(),
312 '_gtest_output_test_normalized_golden.txt'), 'wb').write(
315 self
.assertEqual(normalized_golden
, normalized_actual
)
318 if __name__
== '__main__':
319 if sys
.argv
[1:] == [GENGOLDEN_FLAG
]:
320 if CAN_GENERATE_GOLDEN_FILE
:
321 output
= GetOutputOfAllCommands()
322 golden_file
= open(GOLDEN_PATH
, 'wb')
323 golden_file
.write(output
)
327 """Unable to write a golden file when compiled in an environment
328 that does not support all the required features (death tests, typed tests,
329 and multiple threads). Please generate the golden file using a binary built
330 with those features enabled.""")
332 sys
.stderr
.write(message
)
335 gtest_test_utils
.Main()