]>
Commit | Line | Data |
---|---|---|
31f18b77 FG |
1 | #!/usr/bin/env python |
2 | # | |
3 | # Copyright 2008, Google Inc. | |
4 | # All rights reserved. | |
5 | # | |
6 | # Redistribution and use in source and binary forms, with or without | |
7 | # modification, are permitted provided that the following conditions are | |
8 | # met: | |
9 | # | |
10 | # * Redistributions of source code must retain the above copyright | |
11 | # notice, this list of conditions and the following disclaimer. | |
12 | # * Redistributions in binary form must reproduce the above | |
13 | # copyright notice, this list of conditions and the following disclaimer | |
14 | # in the documentation and/or other materials provided with the | |
15 | # distribution. | |
16 | # * Neither the name of Google Inc. nor the names of its | |
17 | # contributors may be used to endorse or promote products derived from | |
18 | # this software without specific prior written permission. | |
19 | # | |
20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
31 | ||
1e59de90 TL |
32 | """Tests the text output of Google C++ Testing and Mocking Framework. |
33 | ||
31f18b77 FG |
34 | |
35 | SYNOPSIS | |
36 | gtest_output_test.py --build_dir=BUILD/DIR --gengolden | |
37 | # where BUILD/DIR contains the built gtest_output_test_ file. | |
38 | gtest_output_test.py --gengolden | |
39 | gtest_output_test.py | |
40 | """ | |
41 | ||
42 | __author__ = 'wan@google.com (Zhanyong Wan)' | |
43 | ||
44 | import difflib | |
45 | import os | |
46 | import re | |
47 | import sys | |
48 | import gtest_test_utils | |
49 | ||
50 | ||
51 | # The flag for generating the golden file | |
52 | GENGOLDEN_FLAG = '--gengolden' | |
53 | CATCH_EXCEPTIONS_ENV_VAR_NAME = 'GTEST_CATCH_EXCEPTIONS' | |
54 | ||
1e59de90 | 55 | IS_LINUX = os.name == 'posix' and os.uname()[0] == 'Linux' |
31f18b77 FG |
56 | IS_WINDOWS = os.name == 'nt' |
57 | ||
58 | # TODO(vladl@google.com): remove the _lin suffix. | |
59 | GOLDEN_NAME = 'gtest_output_test_golden_lin.txt' | |
60 | ||
61 | PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath('gtest_output_test_') | |
62 | ||
63 | # At least one command we exercise must not have the | |
64 | # 'internal_skip_environment_and_ad_hoc_tests' argument. | |
65 | COMMAND_LIST_TESTS = ({}, [PROGRAM_PATH, '--gtest_list_tests']) | |
66 | COMMAND_WITH_COLOR = ({}, [PROGRAM_PATH, '--gtest_color=yes']) | |
67 | COMMAND_WITH_TIME = ({}, [PROGRAM_PATH, | |
68 | '--gtest_print_time', | |
69 | 'internal_skip_environment_and_ad_hoc_tests', | |
70 | '--gtest_filter=FatalFailureTest.*:LoggingTest.*']) | |
71 | COMMAND_WITH_DISABLED = ( | |
72 | {}, [PROGRAM_PATH, | |
73 | '--gtest_also_run_disabled_tests', | |
74 | 'internal_skip_environment_and_ad_hoc_tests', | |
75 | '--gtest_filter=*DISABLED_*']) | |
76 | COMMAND_WITH_SHARDING = ( | |
77 | {'GTEST_SHARD_INDEX': '1', 'GTEST_TOTAL_SHARDS': '2'}, | |
78 | [PROGRAM_PATH, | |
79 | 'internal_skip_environment_and_ad_hoc_tests', | |
80 | '--gtest_filter=PassingTest.*']) | |
81 | ||
82 | GOLDEN_PATH = os.path.join(gtest_test_utils.GetSourceDir(), GOLDEN_NAME) | |
83 | ||
84 | ||
85 | def ToUnixLineEnding(s): | |
86 | """Changes all Windows/Mac line endings in s to UNIX line endings.""" | |
87 | ||
88 | return s.replace('\r\n', '\n').replace('\r', '\n') | |
89 | ||
90 | ||
91 | def RemoveLocations(test_output): | |
92 | """Removes all file location info from a Google Test program's output. | |
93 | ||
94 | Args: | |
95 | test_output: the output of a Google Test program. | |
96 | ||
97 | Returns: | |
98 | output with all file location info (in the form of | |
99 | 'DIRECTORY/FILE_NAME:LINE_NUMBER: 'or | |
100 | 'DIRECTORY\\FILE_NAME(LINE_NUMBER): ') replaced by | |
101 | 'FILE_NAME:#: '. | |
102 | """ | |
103 | ||
1e59de90 TL |
104 | return re.sub(r'.*[/\\]((gtest_output_test_|gtest).cc)(\:\d+|\(\d+\))\: ', |
105 | r'\1:#: ', test_output) | |
31f18b77 FG |
106 | |
107 | ||
108 | def RemoveStackTraceDetails(output): | |
109 | """Removes all stack traces from a Google Test program's output.""" | |
110 | ||
111 | # *? means "find the shortest string that matches". | |
112 | return re.sub(r'Stack trace:(.|\n)*?\n\n', | |
113 | 'Stack trace: (omitted)\n\n', output) | |
114 | ||
115 | ||
116 | def RemoveStackTraces(output): | |
117 | """Removes all traces of stack traces from a Google Test program's output.""" | |
118 | ||
119 | # *? means "find the shortest string that matches". | |
120 | return re.sub(r'Stack trace:(.|\n)*?\n\n', '', output) | |
121 | ||
122 | ||
123 | def RemoveTime(output): | |
124 | """Removes all time information from a Google Test program's output.""" | |
125 | ||
126 | return re.sub(r'\(\d+ ms', '(? ms', output) | |
127 | ||
128 | ||
129 | def RemoveTypeInfoDetails(test_output): | |
130 | """Removes compiler-specific type info from Google Test program's output. | |
131 | ||
132 | Args: | |
133 | test_output: the output of a Google Test program. | |
134 | ||
135 | Returns: | |
136 | output with type information normalized to canonical form. | |
137 | """ | |
138 | ||
139 | # some compilers output the name of type 'unsigned int' as 'unsigned' | |
140 | return re.sub(r'unsigned int', 'unsigned', test_output) | |
141 | ||
142 | ||
143 | def NormalizeToCurrentPlatform(test_output): | |
144 | """Normalizes platform specific output details for easier comparison.""" | |
145 | ||
146 | if IS_WINDOWS: | |
147 | # Removes the color information that is not present on Windows. | |
148 | test_output = re.sub('\x1b\\[(0;3\d)?m', '', test_output) | |
149 | # Changes failure message headers into the Windows format. | |
150 | test_output = re.sub(r': Failure\n', r': error: ', test_output) | |
151 | # Changes file(line_number) to file:line_number. | |
152 | test_output = re.sub(r'((\w|\.)+)\((\d+)\):', r'\1:\3:', test_output) | |
153 | ||
154 | return test_output | |
155 | ||
156 | ||
157 | def RemoveTestCounts(output): | |
158 | """Removes test counts from a Google Test program's output.""" | |
159 | ||
160 | output = re.sub(r'\d+ tests?, listed below', | |
161 | '? tests, listed below', output) | |
162 | output = re.sub(r'\d+ FAILED TESTS', | |
163 | '? FAILED TESTS', output) | |
164 | output = re.sub(r'\d+ tests? from \d+ test cases?', | |
165 | '? tests from ? test cases', output) | |
166 | output = re.sub(r'\d+ tests? from ([a-zA-Z_])', | |
167 | r'? tests from \1', output) | |
168 | return re.sub(r'\d+ tests?\.', '? tests.', output) | |
169 | ||
170 | ||
171 | def RemoveMatchingTests(test_output, pattern): | |
172 | """Removes output of specified tests from a Google Test program's output. | |
173 | ||
174 | This function strips not only the beginning and the end of a test but also | |
175 | all output in between. | |
176 | ||
177 | Args: | |
178 | test_output: A string containing the test output. | |
179 | pattern: A regex string that matches names of test cases or | |
180 | tests to remove. | |
181 | ||
182 | Returns: | |
183 | Contents of test_output with tests whose names match pattern removed. | |
184 | """ | |
185 | ||
186 | test_output = re.sub( | |
187 | r'.*\[ RUN \] .*%s(.|\n)*?\[( FAILED | OK )\] .*%s.*\n' % ( | |
188 | pattern, pattern), | |
189 | '', | |
190 | test_output) | |
191 | return re.sub(r'.*%s.*\n' % pattern, '', test_output) | |
192 | ||
193 | ||
194 | def NormalizeOutput(output): | |
195 | """Normalizes output (the output of gtest_output_test_.exe).""" | |
196 | ||
197 | output = ToUnixLineEnding(output) | |
198 | output = RemoveLocations(output) | |
199 | output = RemoveStackTraceDetails(output) | |
200 | output = RemoveTime(output) | |
201 | return output | |
202 | ||
203 | ||
204 | def GetShellCommandOutput(env_cmd): | |
205 | """Runs a command in a sub-process, and returns its output in a string. | |
206 | ||
207 | Args: | |
208 | env_cmd: The shell command. A 2-tuple where element 0 is a dict of extra | |
209 | environment variables to set, and element 1 is a string with | |
210 | the command and any flags. | |
211 | ||
212 | Returns: | |
213 | A string with the command's combined standard and diagnostic output. | |
214 | """ | |
215 | ||
216 | # Spawns cmd in a sub-process, and gets its standard I/O file objects. | |
217 | # Set and save the environment properly. | |
218 | environ = os.environ.copy() | |
219 | environ.update(env_cmd[0]) | |
220 | p = gtest_test_utils.Subprocess(env_cmd[1], env=environ) | |
221 | ||
222 | return p.output | |
223 | ||
224 | ||
225 | def GetCommandOutput(env_cmd): | |
226 | """Runs a command and returns its output with all file location | |
227 | info stripped off. | |
228 | ||
229 | Args: | |
230 | env_cmd: The shell command. A 2-tuple where element 0 is a dict of extra | |
231 | environment variables to set, and element 1 is a string with | |
232 | the command and any flags. | |
233 | """ | |
234 | ||
235 | # Disables exception pop-ups on Windows. | |
236 | environ, cmdline = env_cmd | |
237 | environ = dict(environ) # Ensures we are modifying a copy. | |
238 | environ[CATCH_EXCEPTIONS_ENV_VAR_NAME] = '1' | |
239 | return NormalizeOutput(GetShellCommandOutput((environ, cmdline))) | |
240 | ||
241 | ||
242 | def GetOutputOfAllCommands(): | |
243 | """Returns concatenated output from several representative commands.""" | |
244 | ||
245 | return (GetCommandOutput(COMMAND_WITH_COLOR) + | |
246 | GetCommandOutput(COMMAND_WITH_TIME) + | |
247 | GetCommandOutput(COMMAND_WITH_DISABLED) + | |
248 | GetCommandOutput(COMMAND_WITH_SHARDING)) | |
249 | ||
250 | ||
251 | test_list = GetShellCommandOutput(COMMAND_LIST_TESTS) | |
252 | SUPPORTS_DEATH_TESTS = 'DeathTest' in test_list | |
253 | SUPPORTS_TYPED_TESTS = 'TypedTest' in test_list | |
254 | SUPPORTS_THREADS = 'ExpectFailureWithThreadsTest' in test_list | |
1e59de90 | 255 | SUPPORTS_STACK_TRACES = IS_LINUX |
31f18b77 FG |
256 | |
257 | CAN_GENERATE_GOLDEN_FILE = (SUPPORTS_DEATH_TESTS and | |
258 | SUPPORTS_TYPED_TESTS and | |
259 | SUPPORTS_THREADS and | |
1e59de90 | 260 | SUPPORTS_STACK_TRACES and |
31f18b77 FG |
261 | not IS_WINDOWS) |
262 | ||
263 | class GTestOutputTest(gtest_test_utils.TestCase): | |
264 | def RemoveUnsupportedTests(self, test_output): | |
265 | if not SUPPORTS_DEATH_TESTS: | |
266 | test_output = RemoveMatchingTests(test_output, 'DeathTest') | |
267 | if not SUPPORTS_TYPED_TESTS: | |
268 | test_output = RemoveMatchingTests(test_output, 'TypedTest') | |
269 | test_output = RemoveMatchingTests(test_output, 'TypedDeathTest') | |
270 | test_output = RemoveMatchingTests(test_output, 'TypeParamDeathTest') | |
271 | if not SUPPORTS_THREADS: | |
272 | test_output = RemoveMatchingTests(test_output, | |
273 | 'ExpectFailureWithThreadsTest') | |
274 | test_output = RemoveMatchingTests(test_output, | |
275 | 'ScopedFakeTestPartResultReporterTest') | |
276 | test_output = RemoveMatchingTests(test_output, | |
277 | 'WorksConcurrently') | |
278 | if not SUPPORTS_STACK_TRACES: | |
279 | test_output = RemoveStackTraces(test_output) | |
280 | ||
281 | return test_output | |
282 | ||
283 | def testOutput(self): | |
284 | output = GetOutputOfAllCommands() | |
285 | ||
1e59de90 | 286 | golden_file = open(GOLDEN_PATH, 'rb') |
31f18b77 FG |
287 | # A mis-configured source control system can cause \r appear in EOL |
288 | # sequences when we read the golden file irrespective of an operating | |
289 | # system used. Therefore, we need to strip those \r's from newlines | |
290 | # unconditionally. | |
291 | golden = ToUnixLineEnding(golden_file.read()) | |
292 | golden_file.close() | |
293 | ||
294 | # We want the test to pass regardless of certain features being | |
295 | # supported or not. | |
296 | ||
297 | # We still have to remove type name specifics in all cases. | |
298 | normalized_actual = RemoveTypeInfoDetails(output) | |
299 | normalized_golden = RemoveTypeInfoDetails(golden) | |
300 | ||
301 | if CAN_GENERATE_GOLDEN_FILE: | |
302 | self.assertEqual(normalized_golden, normalized_actual, | |
303 | '\n'.join(difflib.unified_diff( | |
304 | normalized_golden.split('\n'), | |
305 | normalized_actual.split('\n'), | |
306 | 'golden', 'actual'))) | |
307 | else: | |
308 | normalized_actual = NormalizeToCurrentPlatform( | |
309 | RemoveTestCounts(normalized_actual)) | |
310 | normalized_golden = NormalizeToCurrentPlatform( | |
311 | RemoveTestCounts(self.RemoveUnsupportedTests(normalized_golden))) | |
312 | ||
313 | # This code is very handy when debugging golden file differences: | |
314 | if os.getenv('DEBUG_GTEST_OUTPUT_TEST'): | |
315 | open(os.path.join( | |
316 | gtest_test_utils.GetSourceDir(), | |
317 | '_gtest_output_test_normalized_actual.txt'), 'wb').write( | |
318 | normalized_actual) | |
319 | open(os.path.join( | |
320 | gtest_test_utils.GetSourceDir(), | |
321 | '_gtest_output_test_normalized_golden.txt'), 'wb').write( | |
322 | normalized_golden) | |
323 | ||
324 | self.assertEqual(normalized_golden, normalized_actual) | |
325 | ||
326 | ||
327 | if __name__ == '__main__': | |
328 | if sys.argv[1:] == [GENGOLDEN_FLAG]: | |
329 | if CAN_GENERATE_GOLDEN_FILE: | |
330 | output = GetOutputOfAllCommands() | |
331 | golden_file = open(GOLDEN_PATH, 'wb') | |
332 | golden_file.write(output) | |
333 | golden_file.close() | |
334 | else: | |
335 | message = ( | |
336 | """Unable to write a golden file when compiled in an environment | |
1e59de90 TL |
337 | that does not support all the required features (death tests, |
338 | typed tests, stack traces, and multiple threads). | |
339 | Please build this test and generate the golden file using Blaze on Linux.""") | |
31f18b77 FG |
340 | |
341 | sys.stderr.write(message) | |
342 | sys.exit(1) | |
343 | else: | |
344 | gtest_test_utils.Main() |