]>
Commit | Line | Data |
---|---|---|
31f18b77 FG |
1 | #!/usr/bin/env python |
2 | # | |
3 | # Copyright 2009 Google Inc. All Rights Reserved. | |
4 | # | |
5 | # Redistribution and use in source and binary forms, with or without | |
6 | # modification, are permitted provided that the following conditions are | |
7 | # met: | |
8 | # | |
9 | # * Redistributions of source code must retain the above copyright | |
10 | # notice, this list of conditions and the following disclaimer. | |
11 | # * Redistributions in binary form must reproduce the above | |
12 | # copyright notice, this list of conditions and the following disclaimer | |
13 | # in the documentation and/or other materials provided with the | |
14 | # distribution. | |
15 | # * Neither the name of Google Inc. nor the names of its | |
16 | # contributors may be used to endorse or promote products derived from | |
17 | # this software without specific prior written permission. | |
18 | # | |
19 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
20 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
21 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
22 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
23 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
24 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
25 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
26 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
27 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
28 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
29 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
30 | ||
31 | """Verifies that test shuffling works.""" | |
32 | ||
33 | __author__ = 'wan@google.com (Zhanyong Wan)' | |
34 | ||
35 | import os | |
36 | import gtest_test_utils | |
37 | ||
38 | # Command to run the gtest_shuffle_test_ program. | |
39 | COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_') | |
40 | ||
41 | # The environment variables for test sharding. | |
42 | TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' | |
43 | SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' | |
44 | ||
45 | TEST_FILTER = 'A*.A:A*.B:C*' | |
46 | ||
47 | ALL_TESTS = [] | |
48 | ACTIVE_TESTS = [] | |
49 | FILTERED_TESTS = [] | |
50 | SHARDED_TESTS = [] | |
51 | ||
52 | SHUFFLED_ALL_TESTS = [] | |
53 | SHUFFLED_ACTIVE_TESTS = [] | |
54 | SHUFFLED_FILTERED_TESTS = [] | |
55 | SHUFFLED_SHARDED_TESTS = [] | |
56 | ||
57 | ||
58 | def AlsoRunDisabledTestsFlag(): | |
59 | return '--gtest_also_run_disabled_tests' | |
60 | ||
61 | ||
62 | def FilterFlag(test_filter): | |
63 | return '--gtest_filter=%s' % (test_filter,) | |
64 | ||
65 | ||
66 | def RepeatFlag(n): | |
67 | return '--gtest_repeat=%s' % (n,) | |
68 | ||
69 | ||
70 | def ShuffleFlag(): | |
71 | return '--gtest_shuffle' | |
72 | ||
73 | ||
74 | def RandomSeedFlag(n): | |
75 | return '--gtest_random_seed=%s' % (n,) | |
76 | ||
77 | ||
78 | def RunAndReturnOutput(extra_env, args): | |
79 | """Runs the test program and returns its output.""" | |
80 | ||
81 | environ_copy = os.environ.copy() | |
82 | environ_copy.update(extra_env) | |
83 | ||
84 | return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output | |
85 | ||
86 | ||
87 | def GetTestsForAllIterations(extra_env, args): | |
88 | """Runs the test program and returns a list of test lists. | |
89 | ||
90 | Args: | |
91 | extra_env: a map from environment variables to their values | |
92 | args: command line flags to pass to gtest_shuffle_test_ | |
93 | ||
94 | Returns: | |
95 | A list where the i-th element is the list of tests run in the i-th | |
96 | test iteration. | |
97 | """ | |
98 | ||
99 | test_iterations = [] | |
100 | for line in RunAndReturnOutput(extra_env, args).split('\n'): | |
101 | if line.startswith('----'): | |
102 | tests = [] | |
103 | test_iterations.append(tests) | |
104 | elif line.strip(): | |
105 | tests.append(line.strip()) # 'TestCaseName.TestName' | |
106 | ||
107 | return test_iterations | |
108 | ||
109 | ||
110 | def GetTestCases(tests): | |
111 | """Returns a list of test cases in the given full test names. | |
112 | ||
113 | Args: | |
114 | tests: a list of full test names | |
115 | ||
116 | Returns: | |
117 | A list of test cases from 'tests', in their original order. | |
118 | Consecutive duplicates are removed. | |
119 | """ | |
120 | ||
121 | test_cases = [] | |
122 | for test in tests: | |
123 | test_case = test.split('.')[0] | |
124 | if not test_case in test_cases: | |
125 | test_cases.append(test_case) | |
126 | ||
127 | return test_cases | |
128 | ||
129 | ||
130 | def CalculateTestLists(): | |
131 | """Calculates the list of tests run under different flags.""" | |
132 | ||
133 | if not ALL_TESTS: | |
134 | ALL_TESTS.extend( | |
135 | GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0]) | |
136 | ||
137 | if not ACTIVE_TESTS: | |
138 | ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0]) | |
139 | ||
140 | if not FILTERED_TESTS: | |
141 | FILTERED_TESTS.extend( | |
142 | GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0]) | |
143 | ||
144 | if not SHARDED_TESTS: | |
145 | SHARDED_TESTS.extend( | |
146 | GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
147 | SHARD_INDEX_ENV_VAR: '1'}, | |
148 | [])[0]) | |
149 | ||
150 | if not SHUFFLED_ALL_TESTS: | |
151 | SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations( | |
152 | {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0]) | |
153 | ||
154 | if not SHUFFLED_ACTIVE_TESTS: | |
155 | SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations( | |
156 | {}, [ShuffleFlag(), RandomSeedFlag(1)])[0]) | |
157 | ||
158 | if not SHUFFLED_FILTERED_TESTS: | |
159 | SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations( | |
160 | {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0]) | |
161 | ||
162 | if not SHUFFLED_SHARDED_TESTS: | |
163 | SHUFFLED_SHARDED_TESTS.extend( | |
164 | GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
165 | SHARD_INDEX_ENV_VAR: '1'}, | |
166 | [ShuffleFlag(), RandomSeedFlag(1)])[0]) | |
167 | ||
168 | ||
169 | class GTestShuffleUnitTest(gtest_test_utils.TestCase): | |
170 | """Tests test shuffling.""" | |
171 | ||
172 | def setUp(self): | |
173 | CalculateTestLists() | |
174 | ||
175 | def testShufflePreservesNumberOfTests(self): | |
176 | self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS)) | |
177 | self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS)) | |
178 | self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS)) | |
179 | self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS)) | |
180 | ||
181 | def testShuffleChangesTestOrder(self): | |
182 | self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS) | |
183 | self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS) | |
184 | self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, | |
185 | SHUFFLED_FILTERED_TESTS) | |
186 | self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, | |
187 | SHUFFLED_SHARDED_TESTS) | |
188 | ||
189 | def testShuffleChangesTestCaseOrder(self): | |
190 | self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS), | |
191 | GetTestCases(SHUFFLED_ALL_TESTS)) | |
192 | self.assert_( | |
193 | GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS), | |
194 | GetTestCases(SHUFFLED_ACTIVE_TESTS)) | |
195 | self.assert_( | |
196 | GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS), | |
197 | GetTestCases(SHUFFLED_FILTERED_TESTS)) | |
198 | self.assert_( | |
199 | GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS), | |
200 | GetTestCases(SHUFFLED_SHARDED_TESTS)) | |
201 | ||
202 | def testShuffleDoesNotRepeatTest(self): | |
203 | for test in SHUFFLED_ALL_TESTS: | |
204 | self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test), | |
205 | '%s appears more than once' % (test,)) | |
206 | for test in SHUFFLED_ACTIVE_TESTS: | |
207 | self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test), | |
208 | '%s appears more than once' % (test,)) | |
209 | for test in SHUFFLED_FILTERED_TESTS: | |
210 | self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test), | |
211 | '%s appears more than once' % (test,)) | |
212 | for test in SHUFFLED_SHARDED_TESTS: | |
213 | self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test), | |
214 | '%s appears more than once' % (test,)) | |
215 | ||
216 | def testShuffleDoesNotCreateNewTest(self): | |
217 | for test in SHUFFLED_ALL_TESTS: | |
218 | self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,)) | |
219 | for test in SHUFFLED_ACTIVE_TESTS: | |
220 | self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,)) | |
221 | for test in SHUFFLED_FILTERED_TESTS: | |
222 | self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,)) | |
223 | for test in SHUFFLED_SHARDED_TESTS: | |
224 | self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,)) | |
225 | ||
226 | def testShuffleIncludesAllTests(self): | |
227 | for test in ALL_TESTS: | |
228 | self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,)) | |
229 | for test in ACTIVE_TESTS: | |
230 | self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,)) | |
231 | for test in FILTERED_TESTS: | |
232 | self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,)) | |
233 | for test in SHARDED_TESTS: | |
234 | self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,)) | |
235 | ||
236 | def testShuffleLeavesDeathTestsAtFront(self): | |
237 | non_death_test_found = False | |
238 | for test in SHUFFLED_ACTIVE_TESTS: | |
239 | if 'DeathTest.' in test: | |
240 | self.assert_(not non_death_test_found, | |
241 | '%s appears after a non-death test' % (test,)) | |
242 | else: | |
243 | non_death_test_found = True | |
244 | ||
245 | def _VerifyTestCasesDoNotInterleave(self, tests): | |
246 | test_cases = [] | |
247 | for test in tests: | |
248 | [test_case, _] = test.split('.') | |
249 | if test_cases and test_cases[-1] != test_case: | |
250 | test_cases.append(test_case) | |
251 | self.assertEqual(1, test_cases.count(test_case), | |
252 | 'Test case %s is not grouped together in %s' % | |
253 | (test_case, tests)) | |
254 | ||
255 | def testShuffleDoesNotInterleaveTestCases(self): | |
256 | self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS) | |
257 | self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS) | |
258 | self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS) | |
259 | self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS) | |
260 | ||
261 | def testShuffleRestoresOrderAfterEachIteration(self): | |
262 | # Get the test lists in all 3 iterations, using random seed 1, 2, | |
263 | # and 3 respectively. Google Test picks a different seed in each | |
264 | # iteration, and this test depends on the current implementation | |
265 | # picking successive numbers. This dependency is not ideal, but | |
266 | # makes the test much easier to write. | |
267 | [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( | |
268 | GetTestsForAllIterations( | |
269 | {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) | |
270 | ||
271 | # Make sure running the tests with random seed 1 gets the same | |
272 | # order as in iteration 1 above. | |
273 | [tests_with_seed1] = GetTestsForAllIterations( | |
274 | {}, [ShuffleFlag(), RandomSeedFlag(1)]) | |
275 | self.assertEqual(tests_in_iteration1, tests_with_seed1) | |
276 | ||
277 | # Make sure running the tests with random seed 2 gets the same | |
278 | # order as in iteration 2 above. Success means that Google Test | |
279 | # correctly restores the test order before re-shuffling at the | |
280 | # beginning of iteration 2. | |
281 | [tests_with_seed2] = GetTestsForAllIterations( | |
282 | {}, [ShuffleFlag(), RandomSeedFlag(2)]) | |
283 | self.assertEqual(tests_in_iteration2, tests_with_seed2) | |
284 | ||
285 | # Make sure running the tests with random seed 3 gets the same | |
286 | # order as in iteration 3 above. Success means that Google Test | |
287 | # correctly restores the test order before re-shuffling at the | |
288 | # beginning of iteration 3. | |
289 | [tests_with_seed3] = GetTestsForAllIterations( | |
290 | {}, [ShuffleFlag(), RandomSeedFlag(3)]) | |
291 | self.assertEqual(tests_in_iteration3, tests_with_seed3) | |
292 | ||
293 | def testShuffleGeneratesNewOrderInEachIteration(self): | |
294 | [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( | |
295 | GetTestsForAllIterations( | |
296 | {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) | |
297 | ||
298 | self.assert_(tests_in_iteration1 != tests_in_iteration2, | |
299 | tests_in_iteration1) | |
300 | self.assert_(tests_in_iteration1 != tests_in_iteration3, | |
301 | tests_in_iteration1) | |
302 | self.assert_(tests_in_iteration2 != tests_in_iteration3, | |
303 | tests_in_iteration2) | |
304 | ||
305 | def testShuffleShardedTestsPreservesPartition(self): | |
306 | # If we run M tests on N shards, the same M tests should be run in | |
307 | # total, regardless of the random seeds used by the shards. | |
308 | [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
309 | SHARD_INDEX_ENV_VAR: '0'}, | |
310 | [ShuffleFlag(), RandomSeedFlag(1)]) | |
311 | [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
312 | SHARD_INDEX_ENV_VAR: '1'}, | |
313 | [ShuffleFlag(), RandomSeedFlag(20)]) | |
314 | [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
315 | SHARD_INDEX_ENV_VAR: '2'}, | |
316 | [ShuffleFlag(), RandomSeedFlag(25)]) | |
317 | sorted_sharded_tests = tests1 + tests2 + tests3 | |
318 | sorted_sharded_tests.sort() | |
319 | sorted_active_tests = [] | |
320 | sorted_active_tests.extend(ACTIVE_TESTS) | |
321 | sorted_active_tests.sort() | |
322 | self.assertEqual(sorted_active_tests, sorted_sharded_tests) | |
323 | ||
324 | if __name__ == '__main__': | |
325 | gtest_test_utils.Main() |