]>
git.proxmox.com Git - ceph.git/blob - ceph/src/jaegertracing/opentelemetry-cpp/third_party/googletest/googletest/test/googletest-failfast-unittest.py
3 # Copyright 2020 Google Inc. All Rights Reserved.
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
9 # * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 # * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
15 # * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 """Unit test for Google Test fail_fast.
33 A user can specify if a Google Test program should continue test execution
34 after a test failure via the GTEST_FAIL_FAST environment variable or the
35 --gtest_fail_fast flag. The default value of the flag can also be changed
36 by Bazel fail fast environment variable TESTBRIDGE_TEST_RUNNER_FAIL_FAST.
38 This script tests such functionality by invoking googletest-failfast-unittest_
39 (a program written with Google Test) with different environments and command
44 import gtest_test_utils
48 # Bazel testbridge environment variable for fail fast
49 BAZEL_FAIL_FAST_ENV_VAR
= 'TESTBRIDGE_TEST_RUNNER_FAIL_FAST'
51 # The environment variable for specifying fail fast.
52 FAIL_FAST_ENV_VAR
= 'GTEST_FAIL_FAST'
54 # The command line flag for specifying fail fast.
55 FAIL_FAST_FLAG
= 'gtest_fail_fast'
57 # The command line flag to run disabled tests.
58 RUN_DISABLED_FLAG
= 'gtest_also_run_disabled_tests'
60 # The command line flag for specifying a filter.
61 FILTER_FLAG
= 'gtest_filter'
63 # Command to run the googletest-failfast-unittest_ program.
64 COMMAND
= gtest_test_utils
.GetTestExecutablePath(
65 'googletest-failfast-unittest_')
67 # The command line flag to tell Google Test to output the list of tests it
69 LIST_TESTS_FLAG
= '--gtest_list_tests'
71 # Indicates whether Google Test supports death tests.
72 SUPPORTS_DEATH_TESTS
= 'HasDeathTest' in gtest_test_utils
.Subprocess(
73 [COMMAND
, LIST_TESTS_FLAG
]).output
77 environ
= os
.environ
.copy()
80 def SetEnvVar(env_var
, value
):
81 """Sets the env variable to 'value'; unsets it when 'value' is None."""
84 environ
[env_var
] = value
85 elif env_var
in environ
:
89 def RunAndReturnOutput(test_suite
=None, fail_fast
=None, run_disabled
=False):
90 """Runs the test program and returns its output."""
93 xml_path
= os
.path
.join(gtest_test_utils
.GetTempDir(),
94 '.GTestFailFastUnitTest.xml')
95 args
+= ['--gtest_output=xml:' + xml_path
]
96 if fail_fast
is not None:
97 if isinstance(fail_fast
, str):
98 args
+= ['--%s=%s' % (FAIL_FAST_FLAG
, fail_fast
)]
100 args
+= ['--%s' % FAIL_FAST_FLAG
]
102 args
+= ['--no%s' % FAIL_FAST_FLAG
]
104 args
+= ['--%s=%s.*' % (FILTER_FLAG
, test_suite
)]
106 args
+= ['--%s' % RUN_DISABLED_FLAG
]
107 txt_out
= gtest_test_utils
.Subprocess([COMMAND
] + args
, env
=environ
).output
108 with
open(xml_path
) as xml_file
:
109 return txt_out
, xml_file
.read()
113 class GTestFailFastUnitTest(gtest_test_utils
.TestCase
):
114 """Tests the env variable or the command line flag for fail_fast."""
116 def testDefaultBehavior(self
):
117 """Tests the behavior of not specifying the fail_fast."""
119 txt
, _
= RunAndReturnOutput()
120 self
.assertIn('22 FAILED TEST', txt
)
122 def testGoogletestFlag(self
):
123 txt
, _
= RunAndReturnOutput(test_suite
='HasSimpleTest', fail_fast
=True)
124 self
.assertIn('1 FAILED TEST', txt
)
125 self
.assertIn('[ SKIPPED ] 3 tests', txt
)
127 txt
, _
= RunAndReturnOutput(test_suite
='HasSimpleTest', fail_fast
=False)
128 self
.assertIn('4 FAILED TEST', txt
)
129 self
.assertNotIn('[ SKIPPED ]', txt
)
131 def testGoogletestEnvVar(self
):
132 """Tests the behavior of specifying fail_fast via Googletest env var."""
135 SetEnvVar(FAIL_FAST_ENV_VAR
, '1')
136 txt
, _
= RunAndReturnOutput('HasSimpleTest')
137 self
.assertIn('1 FAILED TEST', txt
)
138 self
.assertIn('[ SKIPPED ] 3 tests', txt
)
140 SetEnvVar(FAIL_FAST_ENV_VAR
, '0')
141 txt
, _
= RunAndReturnOutput('HasSimpleTest')
142 self
.assertIn('4 FAILED TEST', txt
)
143 self
.assertNotIn('[ SKIPPED ]', txt
)
145 SetEnvVar(FAIL_FAST_ENV_VAR
, None)
147 def testBazelEnvVar(self
):
148 """Tests the behavior of specifying fail_fast via Bazel testbridge."""
151 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR
, '1')
152 txt
, _
= RunAndReturnOutput('HasSimpleTest')
153 self
.assertIn('1 FAILED TEST', txt
)
154 self
.assertIn('[ SKIPPED ] 3 tests', txt
)
156 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR
, '0')
157 txt
, _
= RunAndReturnOutput('HasSimpleTest')
158 self
.assertIn('4 FAILED TEST', txt
)
159 self
.assertNotIn('[ SKIPPED ]', txt
)
161 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR
, None)
163 def testFlagOverridesEnvVar(self
):
164 """Tests precedence of flag over env var."""
167 SetEnvVar(FAIL_FAST_ENV_VAR
, '0')
168 txt
, _
= RunAndReturnOutput('HasSimpleTest', True)
169 self
.assertIn('1 FAILED TEST', txt
)
170 self
.assertIn('[ SKIPPED ] 3 tests', txt
)
172 SetEnvVar(FAIL_FAST_ENV_VAR
, None)
174 def testGoogletestEnvVarOverridesBazelEnvVar(self
):
175 """Tests that the Googletest native env var over Bazel testbridge."""
178 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR
, '0')
179 SetEnvVar(FAIL_FAST_ENV_VAR
, '1')
180 txt
, _
= RunAndReturnOutput('HasSimpleTest')
181 self
.assertIn('1 FAILED TEST', txt
)
182 self
.assertIn('[ SKIPPED ] 3 tests', txt
)
184 SetEnvVar(FAIL_FAST_ENV_VAR
, None)
185 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR
, None)
187 def testEventListener(self
):
188 txt
, _
= RunAndReturnOutput(test_suite
='HasSkipTest', fail_fast
=True)
189 self
.assertIn('1 FAILED TEST', txt
)
190 self
.assertIn('[ SKIPPED ] 3 tests', txt
)
191 for expected_count
, callback
in [(1, 'OnTestSuiteStart'),
194 (5, 'OnTestPartResult'),
195 (1, 'OnTestSuiteEnd')]:
197 expected_count
, txt
.count(callback
),
198 'Expected %d calls to callback %s match count on output: %s ' %
199 (expected_count
, callback
, txt
))
201 txt
, _
= RunAndReturnOutput(test_suite
='HasSkipTest', fail_fast
=False)
202 self
.assertIn('3 FAILED TEST', txt
)
203 self
.assertIn('[ SKIPPED ] 1 test', txt
)
204 for expected_count
, callback
in [(1, 'OnTestSuiteStart'),
207 (5, 'OnTestPartResult'),
208 (1, 'OnTestSuiteEnd')]:
210 expected_count
, txt
.count(callback
),
211 'Expected %d calls to callback %s match count on output: %s ' %
212 (expected_count
, callback
, txt
))
214 def assertXmlResultCount(self
, result
, count
, xml
):
216 count
, xml
.count('result="%s"' % result
),
217 'Expected \'result="%s"\' match count of %s: %s ' %
218 (result
, count
, xml
))
220 def assertXmlStatusCount(self
, status
, count
, xml
):
222 count
, xml
.count('status="%s"' % status
),
223 'Expected \'status="%s"\' match count of %s: %s ' %
224 (status
, count
, xml
))
226 def assertFailFastXmlAndTxtOutput(self
,
234 """Assert XML and text output of a test execution."""
236 txt
, xml
= RunAndReturnOutput(test_suite
, fail_fast
, run_disabled
)
237 if failure_count
> 0:
238 self
.assertIn('%s FAILED TEST' % failure_count
, txt
)
239 if suppressed_count
> 0:
240 self
.assertIn('%s DISABLED TEST' % suppressed_count
, txt
)
241 if skipped_count
> 0:
242 self
.assertIn('[ SKIPPED ] %s tests' % skipped_count
, txt
)
243 self
.assertXmlStatusCount('run',
244 passed_count
+ failure_count
+ skipped_count
, xml
)
245 self
.assertXmlStatusCount('notrun', suppressed_count
, xml
)
246 self
.assertXmlResultCount('completed', passed_count
+ failure_count
, xml
)
247 self
.assertXmlResultCount('skipped', skipped_count
, xml
)
248 self
.assertXmlResultCount('suppressed', suppressed_count
, xml
)
250 def assertFailFastBehavior(self
,
257 """Assert --fail_fast via flag."""
259 for fail_fast
in ('true', '1', 't', True):
260 self
.assertFailFastXmlAndTxtOutput(fail_fast
, test_suite
, passed_count
,
261 failure_count
, skipped_count
,
262 suppressed_count
, run_disabled
)
264 def assertNotFailFastBehavior(self
,
271 """Assert --nofail_fast via flag."""
273 for fail_fast
in ('false', '0', 'f', False):
274 self
.assertFailFastXmlAndTxtOutput(fail_fast
, test_suite
, passed_count
,
275 failure_count
, skipped_count
,
276 suppressed_count
, run_disabled
)
278 def testFlag_HasFixtureTest(self
):
279 """Tests the behavior of fail_fast and TEST_F."""
280 self
.assertFailFastBehavior(
281 test_suite
='HasFixtureTest',
286 self
.assertNotFailFastBehavior(
287 test_suite
='HasFixtureTest',
293 def testFlag_HasSimpleTest(self
):
294 """Tests the behavior of fail_fast and TEST."""
295 self
.assertFailFastBehavior(
296 test_suite
='HasSimpleTest',
301 self
.assertNotFailFastBehavior(
302 test_suite
='HasSimpleTest',
308 def testFlag_HasParametersTest(self
):
309 """Tests the behavior of fail_fast and TEST_P."""
310 self
.assertFailFastBehavior(
311 test_suite
='HasParametersSuite/HasParametersTest',
316 self
.assertNotFailFastBehavior(
317 test_suite
='HasParametersSuite/HasParametersTest',
323 def testFlag_HasDisabledTest(self
):
324 """Tests the behavior of fail_fast and Disabled test cases."""
325 self
.assertFailFastBehavior(
326 test_suite
='HasDisabledTest',
332 self
.assertNotFailFastBehavior(
333 test_suite
='HasDisabledTest',
340 def testFlag_HasDisabledRunDisabledTest(self
):
341 """Tests the behavior of fail_fast and Disabled test cases enabled."""
342 self
.assertFailFastBehavior(
343 test_suite
='HasDisabledTest',
349 self
.assertNotFailFastBehavior(
350 test_suite
='HasDisabledTest',
357 def testFlag_HasDisabledSuiteTest(self
):
358 """Tests the behavior of fail_fast and Disabled test suites."""
359 self
.assertFailFastBehavior(
360 test_suite
='DISABLED_HasDisabledSuite',
366 self
.assertNotFailFastBehavior(
367 test_suite
='DISABLED_HasDisabledSuite',
374 def testFlag_HasDisabledSuiteRunDisabledTest(self
):
375 """Tests the behavior of fail_fast and Disabled test suites enabled."""
376 self
.assertFailFastBehavior(
377 test_suite
='DISABLED_HasDisabledSuite',
383 self
.assertNotFailFastBehavior(
384 test_suite
='DISABLED_HasDisabledSuite',
391 if SUPPORTS_DEATH_TESTS
:
393 def testFlag_HasDeathTest(self
):
394 """Tests the behavior of fail_fast and death tests."""
395 self
.assertFailFastBehavior(
396 test_suite
='HasDeathTest',
401 self
.assertNotFailFastBehavior(
402 test_suite
='HasDeathTest',
409 if __name__
== '__main__':
410 gtest_test_utils
.Main()