3 # Copyright 2005 Google Inc. All Rights Reserved.
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
9 # * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 # * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
15 # * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 """Unit test for Google Test test filters.
33 A user can specify which test(s) in a Google Test program to run via either
34 the GTEST_FILTER environment variable or the --gtest_filter flag.
35 This script tests such functionality by invoking
36 gtest_filter_unittest_ (a program written with Google Test) with different
37 environments and command line flags.
39 Note that test sharding may also influence which tests are filtered. Therefore,
40 we test that here also.
43 __author__
= 'wan@google.com (Zhanyong Wan)'
48 from sets
import Set
as set # For Python 2.3 compatibility
53 import gtest_test_utils
57 # Checks if this platform can pass empty environment variables to child
58 # processes. We set an env variable to an empty string and invoke a python
59 # script in a subprocess to print whether the variable is STILL in
60 # os.environ. We then use 'eval' to parse the child's output so that an
61 # exception is thrown if the input is anything other than 'True' nor 'False'.
62 os
.environ
['EMPTY_VAR'] = ''
63 child
= gtest_test_utils
.Subprocess(
64 [sys
.executable
, '-c', 'import os; print(\'EMPTY_VAR\' in os.environ)'])
65 CAN_PASS_EMPTY_ENV
= eval(child
.output
)
68 # Check if this platform can unset environment variables in child processes.
69 # We set an env variable to a non-empty string, unset it, and invoke
70 # a python script in a subprocess to print whether the variable
71 # is NO LONGER in os.environ.
72 # We use 'eval' to parse the child's output so that an exception
73 # is thrown if the input is neither 'True' nor 'False'.
74 os
.environ
['UNSET_VAR'] = 'X'
75 del os
.environ
['UNSET_VAR']
76 child
= gtest_test_utils
.Subprocess(
77 [sys
.executable
, '-c', 'import os; print(\'UNSET_VAR\' not in os.environ)'])
78 CAN_UNSET_ENV
= eval(child
.output
)
81 # Checks if we should test with an empty filter. This doesn't
82 # make sense on platforms that cannot pass empty env variables (Win32)
83 # and on platforms that cannot unset variables (since we cannot tell
84 # the difference between "" and NULL -- Borland and Solaris < 5.10)
85 CAN_TEST_EMPTY_FILTER
= (CAN_PASS_EMPTY_ENV
and CAN_UNSET_ENV
)
88 # The environment variable for specifying the test filters.
89 FILTER_ENV_VAR
= 'GTEST_FILTER'
91 # The environment variables for test sharding.
92 TOTAL_SHARDS_ENV_VAR
= 'GTEST_TOTAL_SHARDS'
93 SHARD_INDEX_ENV_VAR
= 'GTEST_SHARD_INDEX'
94 SHARD_STATUS_FILE_ENV_VAR
= 'GTEST_SHARD_STATUS_FILE'
96 # The command line flag for specifying the test filters.
97 FILTER_FLAG
= 'gtest_filter'
99 # The command line flag for including disabled tests.
100 ALSO_RUN_DISABED_TESTS_FLAG
= 'gtest_also_run_disabled_tests'
102 # Command to run the gtest_filter_unittest_ program.
103 COMMAND
= gtest_test_utils
.GetTestExecutablePath('gtest_filter_unittest_')
105 # Regex for determining whether parameterized tests are enabled in the binary.
106 PARAM_TEST_REGEX
= re
.compile(r
'/ParamTest')
108 # Regex for parsing test case names from Google Test's output.
109 TEST_CASE_REGEX
= re
.compile(r
'^\[\-+\] \d+ tests? from (\w+(/\w+)?)')
111 # Regex for parsing test names from Google Test's output.
112 TEST_REGEX
= re
.compile(r
'^\[\s*RUN\s*\].*\.(\w+(/\w+)?)')
114 # The command line flag to tell Google Test to output the list of tests it
116 LIST_TESTS_FLAG
= '--gtest_list_tests'
118 # Indicates whether Google Test supports death tests.
119 SUPPORTS_DEATH_TESTS
= 'HasDeathTest' in gtest_test_utils
.Subprocess(
120 [COMMAND
, LIST_TESTS_FLAG
]).output
122 # Full names of all tests in gtest_filter_unittests_.
124 'SeqP/ParamTest.TestX/0',
125 'SeqP/ParamTest.TestX/1',
126 'SeqP/ParamTest.TestY/0',
127 'SeqP/ParamTest.TestY/1',
128 'SeqQ/ParamTest.TestX/0',
129 'SeqQ/ParamTest.TestX/1',
130 'SeqQ/ParamTest.TestY/0',
131 'SeqQ/ParamTest.TestY/1',
135 'BarTest.DISABLED_TestFour',
136 'BarTest.DISABLED_TestFive',
137 'BazTest.DISABLED_TestC',
138 'DISABLED_FoobarTest.Test1',
139 'DISABLED_FoobarTest.DISABLED_Test2',
140 'DISABLED_FoobarbazTest.TestA',
143 if SUPPORTS_DEATH_TESTS
:
145 'HasDeathTest.Test1',
146 'HasDeathTest.Test2',
151 # All the non-disabled tests.
163 ] + DEATH_TESTS
+ PARAM_TESTS
165 param_tests_present
= None
169 environ
= os
.environ
.copy()
172 def SetEnvVar(env_var
, value
):
173 """Sets the env variable to 'value'; unsets it when 'value' is None."""
175 if value
is not None:
176 environ
[env_var
] = value
177 elif env_var
in environ
:
181 def RunAndReturnOutput(args
= None):
182 """Runs the test program and returns its output."""
184 return gtest_test_utils
.Subprocess([COMMAND
] + (args
or []),
188 def RunAndExtractTestList(args
= None):
189 """Runs the test program and returns its exit code and a list of tests run."""
191 p
= gtest_test_utils
.Subprocess([COMMAND
] + (args
or []), env
=environ
)
195 for line
in p
.output
.split('\n'):
196 match
= TEST_CASE_REGEX
.match(line
)
197 if match
is not None:
198 test_case
= match
.group(1)
200 match
= TEST_REGEX
.match(line
)
201 if match
is not None:
202 test
= match
.group(1)
203 tests_run
.append(test_case
+ '.' + test
)
204 return (tests_run
, p
.exit_code
)
207 def InvokeWithModifiedEnv(extra_env
, function
, *args
, **kwargs
):
208 """Runs the given function and arguments in a modified environment."""
210 original_env
= environ
.copy()
211 environ
.update(extra_env
)
212 return function(*args
, **kwargs
)
215 environ
.update(original_env
)
218 def RunWithSharding(total_shards
, shard_index
, command
):
219 """Runs a test program shard and returns exit code and a list of tests run."""
221 extra_env
= {SHARD_INDEX_ENV_VAR
: str(shard_index
),
222 TOTAL_SHARDS_ENV_VAR
: str(total_shards
)}
223 return InvokeWithModifiedEnv(extra_env
, RunAndExtractTestList
, command
)
228 class GTestFilterUnitTest(gtest_test_utils
.TestCase
):
229 """Tests the env variable or the command line flag to filter tests."""
233 def AssertSetEqual(self
, lhs
, rhs
):
234 """Asserts that two sets are equal."""
237 self
.assert_(elem
in rhs
, '%s in %s' % (elem
, rhs
))
240 self
.assert_(elem
in lhs
, '%s in %s' % (elem
, lhs
))
242 def AssertPartitionIsValid(self
, set_var
, list_of_sets
):
243 """Asserts that list_of_sets is a valid partition of set_var."""
246 for slice_var
in list_of_sets
:
247 full_partition
.extend(slice_var
)
248 self
.assertEqual(len(set_var
), len(full_partition
))
249 self
.assertEqual(set(set_var
), set(full_partition
))
251 def AdjustForParameterizedTests(self
, tests_to_run
):
252 """Adjust tests_to_run in case value parameterized tests are disabled."""
254 global param_tests_present
255 if not param_tests_present
:
256 return list(set(tests_to_run
) - set(PARAM_TESTS
))
260 def RunAndVerify(self
, gtest_filter
, tests_to_run
):
261 """Checks that the binary runs correct set of tests for a given filter."""
263 tests_to_run
= self
.AdjustForParameterizedTests(tests_to_run
)
265 # First, tests using the environment variable.
267 # Windows removes empty variables from the environment when passing it
268 # to a new process. This means it is impossible to pass an empty filter
269 # into a process using the environment variable. However, we can still
270 # test the case when the variable is not supplied (i.e., gtest_filter is
272 # pylint: disable-msg=C6403
273 if CAN_TEST_EMPTY_FILTER
or gtest_filter
!= '':
274 SetEnvVar(FILTER_ENV_VAR
, gtest_filter
)
275 tests_run
= RunAndExtractTestList()[0]
276 SetEnvVar(FILTER_ENV_VAR
, None)
277 self
.AssertSetEqual(tests_run
, tests_to_run
)
278 # pylint: enable-msg=C6403
280 # Next, tests using the command line flag.
282 if gtest_filter
is None:
285 args
= ['--%s=%s' % (FILTER_FLAG
, gtest_filter
)]
287 tests_run
= RunAndExtractTestList(args
)[0]
288 self
.AssertSetEqual(tests_run
, tests_to_run
)
290 def RunAndVerifyWithSharding(self
, gtest_filter
, total_shards
, tests_to_run
,
291 args
=None, check_exit_0
=False):
292 """Checks that binary runs correct tests for the given filter and shard.
294 Runs all shards of gtest_filter_unittest_ with the given filter, and
295 verifies that the right set of tests were run. The union of tests run
296 on each shard should be identical to tests_to_run, without duplicates.
299 gtest_filter: A filter to apply to the tests.
300 total_shards: A total number of shards to split test run into.
301 tests_to_run: A set of tests expected to run.
302 args : Arguments to pass to the to the test binary.
303 check_exit_0: When set to a true value, make sure that all shards
307 tests_to_run
= self
.AdjustForParameterizedTests(tests_to_run
)
309 # Windows removes empty variables from the environment when passing it
310 # to a new process. This means it is impossible to pass an empty filter
311 # into a process using the environment variable. However, we can still
312 # test the case when the variable is not supplied (i.e., gtest_filter is
314 # pylint: disable-msg=C6403
315 if CAN_TEST_EMPTY_FILTER
or gtest_filter
!= '':
316 SetEnvVar(FILTER_ENV_VAR
, gtest_filter
)
318 for i
in range(0, total_shards
):
319 (tests_run
, exit_code
) = RunWithSharding(total_shards
, i
, args
)
321 self
.assertEqual(0, exit_code
)
322 partition
.append(tests_run
)
324 self
.AssertPartitionIsValid(tests_to_run
, partition
)
325 SetEnvVar(FILTER_ENV_VAR
, None)
326 # pylint: enable-msg=C6403
328 def RunAndVerifyAllowingDisabled(self
, gtest_filter
, tests_to_run
):
329 """Checks that the binary runs correct set of tests for the given filter.
331 Runs gtest_filter_unittest_ with the given filter, and enables
332 disabled tests. Verifies that the right set of tests were run.
335 gtest_filter: A filter to apply to the tests.
336 tests_to_run: A set of tests expected to run.
339 tests_to_run
= self
.AdjustForParameterizedTests(tests_to_run
)
341 # Construct the command line.
342 args
= ['--%s' % ALSO_RUN_DISABED_TESTS_FLAG
]
343 if gtest_filter
is not None:
344 args
.append('--%s=%s' % (FILTER_FLAG
, gtest_filter
))
346 tests_run
= RunAndExtractTestList(args
)[0]
347 self
.AssertSetEqual(tests_run
, tests_to_run
)
350 """Sets up test case.
352 Determines whether value-parameterized tests are enabled in the binary and
353 sets the flags accordingly.
356 global param_tests_present
357 if param_tests_present
is None:
358 param_tests_present
= PARAM_TEST_REGEX
.search(
359 RunAndReturnOutput()) is not None
361 def testDefaultBehavior(self
):
362 """Tests the behavior of not specifying the filter."""
364 self
.RunAndVerify(None, ACTIVE_TESTS
)
366 def testDefaultBehaviorWithShards(self
):
367 """Tests the behavior without the filter, with sharding enabled."""
369 self
.RunAndVerifyWithSharding(None, 1, ACTIVE_TESTS
)
370 self
.RunAndVerifyWithSharding(None, 2, ACTIVE_TESTS
)
371 self
.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS
) - 1, ACTIVE_TESTS
)
372 self
.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS
), ACTIVE_TESTS
)
373 self
.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS
) + 1, ACTIVE_TESTS
)
375 def testEmptyFilter(self
):
376 """Tests an empty filter."""
378 self
.RunAndVerify('', [])
379 self
.RunAndVerifyWithSharding('', 1, [])
380 self
.RunAndVerifyWithSharding('', 2, [])
382 def testBadFilter(self
):
383 """Tests a filter that matches nothing."""
385 self
.RunAndVerify('BadFilter', [])
386 self
.RunAndVerifyAllowingDisabled('BadFilter', [])
388 def testFullName(self
):
389 """Tests filtering by full name."""
391 self
.RunAndVerify('FooTest.Xyz', ['FooTest.Xyz'])
392 self
.RunAndVerifyAllowingDisabled('FooTest.Xyz', ['FooTest.Xyz'])
393 self
.RunAndVerifyWithSharding('FooTest.Xyz', 5, ['FooTest.Xyz'])
395 def testUniversalFilters(self
):
396 """Tests filters that match everything."""
398 self
.RunAndVerify('*', ACTIVE_TESTS
)
399 self
.RunAndVerify('*.*', ACTIVE_TESTS
)
400 self
.RunAndVerifyWithSharding('*.*', len(ACTIVE_TESTS
) - 3, ACTIVE_TESTS
)
401 self
.RunAndVerifyAllowingDisabled('*', ACTIVE_TESTS
+ DISABLED_TESTS
)
402 self
.RunAndVerifyAllowingDisabled('*.*', ACTIVE_TESTS
+ DISABLED_TESTS
)
404 def testFilterByTestCase(self
):
405 """Tests filtering by test case name."""
407 self
.RunAndVerify('FooTest.*', ['FooTest.Abc', 'FooTest.Xyz'])
409 BAZ_TESTS
= ['BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB']
410 self
.RunAndVerify('BazTest.*', BAZ_TESTS
)
411 self
.RunAndVerifyAllowingDisabled('BazTest.*',
412 BAZ_TESTS
+ ['BazTest.DISABLED_TestC'])
414 def testFilterByTest(self
):
415 """Tests filtering by test name."""
417 self
.RunAndVerify('*.TestOne', ['BarTest.TestOne', 'BazTest.TestOne'])
419 def testFilterDisabledTests(self
):
420 """Select only the disabled tests to run."""
422 self
.RunAndVerify('DISABLED_FoobarTest.Test1', [])
423 self
.RunAndVerifyAllowingDisabled('DISABLED_FoobarTest.Test1',
424 ['DISABLED_FoobarTest.Test1'])
426 self
.RunAndVerify('*DISABLED_*', [])
427 self
.RunAndVerifyAllowingDisabled('*DISABLED_*', DISABLED_TESTS
)
429 self
.RunAndVerify('*.DISABLED_*', [])
430 self
.RunAndVerifyAllowingDisabled('*.DISABLED_*', [
431 'BarTest.DISABLED_TestFour',
432 'BarTest.DISABLED_TestFive',
433 'BazTest.DISABLED_TestC',
434 'DISABLED_FoobarTest.DISABLED_Test2',
437 self
.RunAndVerify('DISABLED_*', [])
438 self
.RunAndVerifyAllowingDisabled('DISABLED_*', [
439 'DISABLED_FoobarTest.Test1',
440 'DISABLED_FoobarTest.DISABLED_Test2',
441 'DISABLED_FoobarbazTest.TestA',
444 def testWildcardInTestCaseName(self
):
445 """Tests using wildcard in the test case name."""
447 self
.RunAndVerify('*a*.*', [
454 'BazTest.TestB', ] + DEATH_TESTS
+ PARAM_TESTS
)
456 def testWildcardInTestName(self
):
457 """Tests using wildcard in the test name."""
459 self
.RunAndVerify('*.*A*', ['FooTest.Abc', 'BazTest.TestA'])
461 def testFilterWithoutDot(self
):
462 """Tests a filter that has no '.' in it."""
464 self
.RunAndVerify('*z*', [
472 def testTwoPatterns(self
):
473 """Tests filters that consist of two patterns."""
475 self
.RunAndVerify('Foo*.*:*A*', [
482 # An empty pattern + a non-empty one
483 self
.RunAndVerify(':*A*', ['FooTest.Abc', 'BazTest.TestA'])
485 def testThreePatterns(self
):
486 """Tests filters that consist of three patterns."""
488 self
.RunAndVerify('*oo*:*A*:*One', [
498 # The 2nd pattern is empty.
499 self
.RunAndVerify('*oo*::*One', [
508 # The last 2 patterns are empty.
509 self
.RunAndVerify('*oo*::', [
514 def testNegativeFilters(self
):
515 self
.RunAndVerify('*-BazTest.TestOne', [
525 ] + DEATH_TESTS
+ PARAM_TESTS
)
527 self
.RunAndVerify('*-FooTest.Abc:BazTest.*', [
533 ] + DEATH_TESTS
+ PARAM_TESTS
)
535 self
.RunAndVerify('BarTest.*-BarTest.TestOne', [
540 # Tests without leading '*'.
541 self
.RunAndVerify('-FooTest.Abc:FooTest.Xyz:BazTest.*', [
545 ] + DEATH_TESTS
+ PARAM_TESTS
)
547 # Value parameterized tests.
548 self
.RunAndVerify('*/*', PARAM_TESTS
)
550 # Value parameterized tests filtering by the sequence name.
551 self
.RunAndVerify('SeqP/*', [
552 'SeqP/ParamTest.TestX/0',
553 'SeqP/ParamTest.TestX/1',
554 'SeqP/ParamTest.TestY/0',
555 'SeqP/ParamTest.TestY/1',
558 # Value parameterized tests filtering by the test name.
559 self
.RunAndVerify('*/0', [
560 'SeqP/ParamTest.TestX/0',
561 'SeqP/ParamTest.TestY/0',
562 'SeqQ/ParamTest.TestX/0',
563 'SeqQ/ParamTest.TestY/0',
566 def testFlagOverridesEnvVar(self
):
567 """Tests that the filter flag overrides the filtering env. variable."""
569 SetEnvVar(FILTER_ENV_VAR
, 'Foo*')
570 args
= ['--%s=%s' % (FILTER_FLAG
, '*One')]
571 tests_run
= RunAndExtractTestList(args
)[0]
572 SetEnvVar(FILTER_ENV_VAR
, None)
574 self
.AssertSetEqual(tests_run
, ['BarTest.TestOne', 'BazTest.TestOne'])
576 def testShardStatusFileIsCreated(self
):
577 """Tests that the shard file is created if specified in the environment."""
579 shard_status_file
= os
.path
.join(gtest_test_utils
.GetTempDir(),
581 self
.assert_(not os
.path
.exists(shard_status_file
))
583 extra_env
= {SHARD_STATUS_FILE_ENV_VAR
: shard_status_file
}
585 InvokeWithModifiedEnv(extra_env
, RunAndReturnOutput
)
587 self
.assert_(os
.path
.exists(shard_status_file
))
588 os
.remove(shard_status_file
)
590 def testShardStatusFileIsCreatedWithListTests(self
):
591 """Tests that the shard file is created with the "list_tests" flag."""
593 shard_status_file
= os
.path
.join(gtest_test_utils
.GetTempDir(),
594 'shard_status_file2')
595 self
.assert_(not os
.path
.exists(shard_status_file
))
597 extra_env
= {SHARD_STATUS_FILE_ENV_VAR
: shard_status_file
}
599 output
= InvokeWithModifiedEnv(extra_env
,
603 # This assertion ensures that Google Test enumerated the tests as
604 # opposed to running them.
605 self
.assert_('[==========]' not in output
,
606 'Unexpected output during test enumeration.\n'
607 'Please ensure that LIST_TESTS_FLAG is assigned the\n'
608 'correct flag value for listing Google Test tests.')
610 self
.assert_(os
.path
.exists(shard_status_file
))
611 os
.remove(shard_status_file
)
613 if SUPPORTS_DEATH_TESTS
:
614 def testShardingWorksWithDeathTests(self
):
615 """Tests integration with death tests and sharding."""
617 gtest_filter
= 'HasDeathTest.*:SeqP/*'
619 'HasDeathTest.Test1',
620 'HasDeathTest.Test2',
622 'SeqP/ParamTest.TestX/0',
623 'SeqP/ParamTest.TestX/1',
624 'SeqP/ParamTest.TestY/0',
625 'SeqP/ParamTest.TestY/1',
628 for flag
in ['--gtest_death_test_style=threadsafe',
629 '--gtest_death_test_style=fast']:
630 self
.RunAndVerifyWithSharding(gtest_filter
, 3, expected_tests
,
631 check_exit_0
=True, args
=[flag
])
632 self
.RunAndVerifyWithSharding(gtest_filter
, 5, expected_tests
,
633 check_exit_0
=True, args
=[flag
])
635 if __name__
== '__main__':
636 gtest_test_utils
.Main()