]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | import logging |
f6b5b4d7 | 2 | import random |
7c673cae FG |
3 | import time |
4 | from tasks.cephfs.fuse_mount import FuseMount | |
5 | from tasks.cephfs.cephfs_test_case import CephFSTestCase | |
f6b5b4d7 | 6 | from teuthology.orchestra.run import CommandFailedError |
7c673cae FG |
7 | |
8 | log = logging.getLogger(__name__) | |
9 | ||
10 | class TestExports(CephFSTestCase): | |
31f18b77 | 11 | MDSS_REQUIRED = 2 |
28e407b8 | 12 | CLIENTS_REQUIRED = 2 |
31f18b77 | 13 | |
f6b5b4d7 TL |
14 | def test_session_race(self): |
15 | """ | |
16 | Test session creation race. | |
17 | ||
18 | See: https://tracker.ceph.com/issues/24072#change-113056 | |
19 | """ | |
20 | ||
7c673cae | 21 | self.fs.set_max_mds(2) |
f6b5b4d7 | 22 | status = self.fs.wait_for_daemons() |
7c673cae | 23 | |
f6b5b4d7 | 24 | rank1 = self.fs.get_rank(rank=1, status=status) |
7c673cae | 25 | |
f6b5b4d7 TL |
26 | # Create a directory that is pre-exported to rank 1 |
27 | self.mount_a.run_shell(["mkdir", "-p", "a/aa"]) | |
28 | self.mount_a.setfattr("a", "ceph.dir.pin", "1") | |
29 | self._wait_subtrees([('/a', 1)], status=status, rank=1) | |
7c673cae | 30 | |
f6b5b4d7 TL |
31 | # Now set the mds config to allow the race |
32 | self.fs.rank_asok(["config", "set", "mds_inject_migrator_session_race", "true"], rank=1) | |
7c673cae | 33 | |
f6b5b4d7 TL |
34 | # Now create another directory and try to export it |
35 | self.mount_b.run_shell(["mkdir", "-p", "b/bb"]) | |
36 | self.mount_b.setfattr("b", "ceph.dir.pin", "1") | |
7c673cae | 37 | |
f6b5b4d7 | 38 | time.sleep(5) |
7c673cae | 39 | |
f6b5b4d7 TL |
40 | # Now turn off the race so that it doesn't wait again |
41 | self.fs.rank_asok(["config", "set", "mds_inject_migrator_session_race", "false"], rank=1) | |
7c673cae | 42 | |
f6b5b4d7 TL |
43 | # Now try to create a session with rank 1 by accessing a dir known to |
44 | # be there, if buggy, this should cause the rank 1 to crash: | |
45 | self.mount_b.run_shell(["ls", "a"]) | |
7c673cae | 46 | |
f6b5b4d7 TL |
47 | # Check if rank1 changed (standby tookover?) |
48 | new_rank1 = self.fs.get_rank(rank=1) | |
49 | self.assertEqual(rank1['gid'], new_rank1['gid']) | |
7c673cae | 50 | |
f6b5b4d7 TL |
51 | class TestExportPin(CephFSTestCase): |
52 | MDSS_REQUIRED = 3 | |
53 | CLIENTS_REQUIRED = 1 | |
7c673cae | 54 | |
f6b5b4d7 TL |
55 | def setUp(self): |
56 | CephFSTestCase.setUp(self) | |
7c673cae | 57 | |
f6b5b4d7 TL |
58 | self.fs.set_max_mds(3) |
59 | self.status = self.fs.wait_for_daemons() | |
7c673cae | 60 | |
f6b5b4d7 | 61 | self.mount_a.run_shell_payload("mkdir -p 1/2/3/4") |
7c673cae | 62 | |
f6b5b4d7 TL |
63 | def test_noop(self): |
64 | self.mount_a.setfattr("1", "ceph.dir.pin", "-1") | |
65 | time.sleep(30) # for something to not happen | |
66 | self._wait_subtrees([], status=self.status) | |
11fdf7f2 | 67 | |
f6b5b4d7 TL |
68 | def test_negative(self): |
69 | self.mount_a.setfattr("1", "ceph.dir.pin", "-2341") | |
70 | time.sleep(30) # for something to not happen | |
71 | self._wait_subtrees([], status=self.status) | |
92f5a8d4 | 72 | |
f6b5b4d7 TL |
73 | def test_empty_pin(self): |
74 | self.mount_a.setfattr("1/2/3/4", "ceph.dir.pin", "1") | |
75 | time.sleep(30) # for something to not happen | |
76 | self._wait_subtrees([], status=self.status) | |
92f5a8d4 | 77 | |
f6b5b4d7 TL |
78 | def test_trivial(self): |
79 | self.mount_a.setfattr("1", "ceph.dir.pin", "1") | |
80 | self._wait_subtrees([('/1', 1)], status=self.status, rank=1) | |
92f5a8d4 | 81 | |
f6b5b4d7 | 82 | def test_export_targets(self): |
92f5a8d4 | 83 | self.mount_a.setfattr("1", "ceph.dir.pin", "1") |
f6b5b4d7 TL |
84 | self._wait_subtrees([('/1', 1)], status=self.status, rank=1) |
85 | self.status = self.fs.status() | |
86 | r0 = self.status.get_rank(self.fs.id, 0) | |
87 | self.assertTrue(sorted(r0['export_targets']) == [1]) | |
92f5a8d4 | 88 | |
f6b5b4d7 TL |
89 | def test_redundant(self): |
90 | # redundant pin /1/2 to rank 1 | |
91 | self.mount_a.setfattr("1", "ceph.dir.pin", "1") | |
92 | self._wait_subtrees([('/1', 1)], status=self.status, rank=1) | |
92f5a8d4 | 93 | self.mount_a.setfattr("1/2", "ceph.dir.pin", "1") |
f6b5b4d7 | 94 | self._wait_subtrees([('/1', 1), ('/1/2', 1)], status=self.status, rank=1) |
92f5a8d4 | 95 | |
f6b5b4d7 TL |
96 | def test_reassignment(self): |
97 | self.mount_a.setfattr("1/2", "ceph.dir.pin", "1") | |
98 | self._wait_subtrees([('/1/2', 1)], status=self.status, rank=1) | |
92f5a8d4 | 99 | self.mount_a.setfattr("1/2", "ceph.dir.pin", "0") |
f6b5b4d7 | 100 | self._wait_subtrees([('/1/2', 0)], status=self.status, rank=0) |
92f5a8d4 | 101 | |
f6b5b4d7 TL |
102 | def test_phantom_rank(self): |
103 | self.mount_a.setfattr("1", "ceph.dir.pin", "0") | |
104 | self.mount_a.setfattr("1/2", "ceph.dir.pin", "10") | |
105 | time.sleep(30) # wait for nothing weird to happen | |
106 | self._wait_subtrees([('/1', 0)], status=self.status) | |
107 | ||
108 | def test_nested(self): | |
109 | self.mount_a.setfattr("1", "ceph.dir.pin", "1") | |
110 | self.mount_a.setfattr("1/2", "ceph.dir.pin", "0") | |
92f5a8d4 | 111 | self.mount_a.setfattr("1/2/3", "ceph.dir.pin", "2") |
f6b5b4d7 TL |
112 | self._wait_subtrees([('/1', 1), ('/1/2', 0), ('/1/2/3', 2)], status=self.status, rank=2) |
113 | ||
114 | def test_nested_unset(self): | |
115 | self.mount_a.setfattr("1", "ceph.dir.pin", "1") | |
116 | self.mount_a.setfattr("1/2", "ceph.dir.pin", "2") | |
117 | self._wait_subtrees([('/1', 1), ('/1/2', 2)], status=self.status, rank=1) | |
118 | self.mount_a.setfattr("1/2", "ceph.dir.pin", "-1") | |
119 | self._wait_subtrees([('/1', 1)], status=self.status, rank=1) | |
120 | ||
121 | def test_rename(self): | |
122 | self.mount_a.setfattr("1", "ceph.dir.pin", "1") | |
123 | self.mount_a.run_shell_payload("mkdir -p 9/8/7") | |
124 | self.mount_a.setfattr("9/8", "ceph.dir.pin", "0") | |
125 | self._wait_subtrees([('/1', 1), ("/9/8", 0)], status=self.status, rank=0) | |
126 | self.mount_a.run_shell_payload("mv 9/8 1/2") | |
127 | self._wait_subtrees([('/1', 1), ("/1/2/8", 0)], status=self.status, rank=0) | |
92f5a8d4 | 128 | |
f6b5b4d7 TL |
129 | def test_getfattr(self): |
130 | # pin /1 to rank 0 | |
131 | self.mount_a.setfattr("1", "ceph.dir.pin", "1") | |
132 | self.mount_a.setfattr("1/2", "ceph.dir.pin", "0") | |
133 | self._wait_subtrees([('/1', 1), ('/1/2', 0)], status=self.status, rank=1) | |
92f5a8d4 TL |
134 | |
135 | if not isinstance(self.mount_a, FuseMount): | |
e306af50 | 136 | p = self.mount_a.client_remote.sh('uname -r', wait=True) |
92f5a8d4 TL |
137 | dir_pin = self.mount_a.getfattr("1", "ceph.dir.pin") |
138 | log.debug("mount.getfattr('1','ceph.dir.pin'): %s " % dir_pin) | |
e306af50 | 139 | if str(p) < "5" and not(dir_pin): |
9f95a23c | 140 | self.skipTest("Kernel does not support getting the extended attribute ceph.dir.pin") |
e306af50 TL |
141 | self.assertEqual(self.mount_a.getfattr("1", "ceph.dir.pin"), '1') |
142 | self.assertEqual(self.mount_a.getfattr("1/2", "ceph.dir.pin"), '0') | |
28e407b8 | 143 | |
f6b5b4d7 TL |
144 | def test_export_pin_cache_drop(self): |
145 | """ | |
146 | That the export pin does not prevent empty (nothing in cache) subtree merging. | |
28e407b8 | 147 | """ |
28e407b8 | 148 | |
f6b5b4d7 TL |
149 | self.mount_a.setfattr("1", "ceph.dir.pin", "0") |
150 | self.mount_a.setfattr("1/2", "ceph.dir.pin", "1") | |
151 | self._wait_subtrees([('/1', 0), ('/1/2', 1)], status=self.status) | |
152 | self.mount_a.umount_wait() # release all caps | |
153 | def _drop(): | |
154 | self.fs.ranks_tell(["cache", "drop"], status=self.status) | |
155 | # drop cache multiple times to clear replica pins | |
156 | self._wait_subtrees([], status=self.status, action=_drop) | |
157 | ||
158 | class TestEphemeralPins(CephFSTestCase): | |
159 | MDSS_REQUIRED = 3 | |
160 | CLIENTS_REQUIRED = 1 | |
161 | ||
162 | def setUp(self): | |
163 | CephFSTestCase.setUp(self) | |
164 | ||
165 | self.config_set('mds', 'mds_export_ephemeral_random', True) | |
166 | self.config_set('mds', 'mds_export_ephemeral_distributed', True) | |
167 | self.config_set('mds', 'mds_export_ephemeral_random_max', 1.0) | |
168 | ||
169 | self.mount_a.run_shell_payload(""" | |
170 | set -e | |
171 | ||
172 | # Use up a random number of inode numbers so the ephemeral pinning is not the same every test. | |
173 | mkdir .inode_number_thrash | |
174 | count=$((RANDOM % 1024)) | |
175 | for ((i = 0; i < count; i++)); do touch .inode_number_thrash/$i; done | |
176 | rm -rf .inode_number_thrash | |
177 | """) | |
178 | ||
179 | self.fs.set_max_mds(3) | |
180 | self.status = self.fs.wait_for_daemons() | |
181 | ||
182 | def _setup_tree(self, path="tree", export=-1, distributed=False, random=0.0, count=100, wait=True): | |
183 | return self.mount_a.run_shell_payload(f""" | |
184 | set -e | |
185 | mkdir -p {path} | |
186 | {f"setfattr -n ceph.dir.pin -v {export} {path}" if export >= 0 else ""} | |
187 | {f"setfattr -n ceph.dir.pin.distributed -v 1 {path}" if distributed else ""} | |
188 | {f"setfattr -n ceph.dir.pin.random -v {random} {path}" if random > 0.0 else ""} | |
189 | for ((i = 0; i < {count}; i++)); do | |
190 | mkdir -p "{path}/$i" | |
191 | echo file > "{path}/$i/file" | |
192 | done | |
193 | """, wait=wait) | |
194 | ||
195 | def test_ephemeral_pin_dist_override(self): | |
196 | """ | |
197 | That an ephemeral distributed pin overrides a normal export pin. | |
28e407b8 AA |
198 | """ |
199 | ||
f6b5b4d7 TL |
200 | self._setup_tree(distributed=True) |
201 | subtrees = self._wait_distributed_subtrees(100, status=self.status, rank="all") | |
202 | for s in subtrees: | |
203 | path = s['dir']['path'] | |
204 | if path == '/tree': | |
205 | self.assertEqual(s['export_pin'], 0) | |
206 | self.assertEqual(s['auth_first'], 0) | |
207 | elif path.startswith('/tree/'): | |
208 | self.assertEqual(s['export_pin'], -1) | |
209 | self.assertTrue(s['distributed_ephemeral_pin']) | |
210 | ||
211 | def test_ephemeral_pin_dist_override_pin(self): | |
212 | """ | |
213 | That an export pin overrides an ephemerally pinned directory. | |
214 | """ | |
28e407b8 | 215 | |
f6b5b4d7 TL |
216 | self._setup_tree(distributed=True, export=0) |
217 | subtrees = self._wait_distributed_subtrees(100, status=self.status, rank="all", path="/tree/") | |
218 | which = None | |
219 | for s in subtrees: | |
220 | if s['auth_first'] == 1: | |
221 | path = s['dir']['path'] | |
222 | self.mount_a.setfattr(path[1:], "ceph.dir.pin", "0") | |
223 | which = path | |
224 | break | |
225 | self.assertIsNotNone(which) | |
226 | time.sleep(15) | |
227 | subtrees = self._get_subtrees(status=self.status, rank=0) | |
228 | for s in subtrees: | |
229 | path = s['dir']['path'] | |
230 | if path == which: | |
231 | self.assertEqual(s['auth_first'], 0) | |
232 | self.assertFalse(s['distributed_ephemeral_pin']) | |
233 | return | |
234 | # it has been merged into /tree | |
235 | ||
236 | def test_ephemeral_pin_dist_off(self): | |
237 | """ | |
238 | That turning off ephemeral distributed pin merges subtrees. | |
239 | """ | |
28e407b8 | 240 | |
f6b5b4d7 TL |
241 | self._setup_tree(distributed=True, export=0) |
242 | self._wait_distributed_subtrees(100, status=self.status, rank="all") | |
243 | self.mount_a.setfattr("tree", "ceph.dir.pin.distributed", "0") | |
244 | self._wait_subtrees([('/tree', 0)], status=self.status) | |
28e407b8 | 245 | |
f6b5b4d7 TL |
246 | def test_ephemeral_pin_dist_conf_off(self): |
247 | """ | |
248 | That turning off ephemeral distributed pin config prevents distribution. | |
249 | """ | |
28e407b8 | 250 | |
f6b5b4d7 TL |
251 | self._setup_tree(export=0) |
252 | self.config_set('mds', 'mds_export_ephemeral_distributed', False) | |
253 | self.mount_a.setfattr("tree", "ceph.dir.pin.distributed", "1") | |
254 | time.sleep(30) | |
255 | self._wait_subtrees([('/tree', 0)], status=self.status) | |
28e407b8 | 256 | |
f6b5b4d7 TL |
257 | def test_ephemeral_pin_dist_conf_off_merge(self): |
258 | """ | |
259 | That turning off ephemeral distributed pin config merges subtrees. | |
260 | """ | |
28e407b8 | 261 | |
f6b5b4d7 TL |
262 | self._setup_tree(distributed=True, export=0) |
263 | self._wait_distributed_subtrees(100, status=self.status) | |
264 | self.config_set('mds', 'mds_export_ephemeral_distributed', False) | |
265 | self._wait_subtrees([('/tree', 0)], timeout=60, status=self.status) | |
28e407b8 | 266 | |
f6b5b4d7 TL |
267 | def test_ephemeral_pin_dist_override_before(self): |
268 | """ | |
269 | That a conventional export pin overrides the distributed policy _before_ distributed policy is set. | |
270 | """ | |
28e407b8 | 271 | |
f6b5b4d7 TL |
272 | count = 10 |
273 | self._setup_tree(count=count) | |
274 | test = [] | |
275 | for i in range(count): | |
276 | path = f"tree/{i}" | |
277 | self.mount_a.setfattr(path, "ceph.dir.pin", "1") | |
278 | test.append(("/"+path, 1)) | |
279 | self.mount_a.setfattr("tree", "ceph.dir.pin.distributed", "1") | |
280 | time.sleep(10) # for something to not happen... | |
281 | self._wait_subtrees(test, timeout=60, status=self.status, rank="all", path="/tree/") | |
282 | ||
283 | def test_ephemeral_pin_dist_override_after(self): | |
284 | """ | |
285 | That a conventional export pin overrides the distributed policy _after_ distributed policy is set. | |
286 | """ | |
287 | ||
288 | self._setup_tree(count=10, distributed=True) | |
289 | subtrees = self._wait_distributed_subtrees(10, status=self.status, rank="all") | |
290 | victim = None | |
291 | test = [] | |
292 | for s in subtrees: | |
293 | path = s['dir']['path'] | |
294 | auth = s['auth_first'] | |
295 | if auth in (0, 2) and victim is None: | |
296 | victim = path | |
297 | self.mount_a.setfattr(victim[1:], "ceph.dir.pin", "1") | |
298 | test.append((victim, 1)) | |
299 | else: | |
300 | test.append((path, auth)) | |
301 | self.assertIsNotNone(victim) | |
302 | self._wait_subtrees(test, status=self.status, rank="all", path="/tree/") | |
303 | ||
304 | def test_ephemeral_pin_dist_failover(self): | |
305 | """ | |
306 | That MDS failover does not cause unnecessary migrations. | |
307 | """ | |
308 | ||
309 | # pin /tree so it does not export during failover | |
310 | self._setup_tree(distributed=True, export=0) | |
311 | self._wait_distributed_subtrees(100, status=self.status, rank="all") | |
312 | #test = [(s['dir']['path'], s['auth_first']) for s in subtrees] | |
313 | before = self.fs.ranks_perf(lambda p: p['mds']['exported']) | |
314 | log.info(f"export stats: {before}") | |
315 | self.fs.rank_fail(rank=1) | |
316 | self.status = self.fs.wait_for_daemons() | |
317 | time.sleep(10) # waiting for something to not happen | |
318 | after = self.fs.ranks_perf(lambda p: p['mds']['exported']) | |
319 | log.info(f"export stats: {after}") | |
320 | self.assertEqual(before, after) | |
321 | ||
322 | def test_ephemeral_pin_distribution(self): | |
323 | """ | |
324 | That ephemerally pinned subtrees are somewhat evenly distributed. | |
325 | """ | |
326 | ||
327 | self.fs.set_max_mds(3) | |
328 | self.status = self.fs.wait_for_daemons() | |
329 | ||
330 | count = 1000 | |
331 | self._setup_tree(count=count, distributed=True) | |
332 | subtrees = self._wait_distributed_subtrees(count, status=self.status, rank="all") | |
333 | nsubtrees = len(subtrees) | |
334 | ||
335 | # Check if distribution is uniform | |
336 | rank0 = list(filter(lambda x: x['auth_first'] == 0, subtrees)) | |
337 | rank1 = list(filter(lambda x: x['auth_first'] == 1, subtrees)) | |
338 | rank2 = list(filter(lambda x: x['auth_first'] == 2, subtrees)) | |
339 | self.assertGreaterEqual(len(rank0)/nsubtrees, 0.2) | |
340 | self.assertGreaterEqual(len(rank1)/nsubtrees, 0.2) | |
341 | self.assertGreaterEqual(len(rank2)/nsubtrees, 0.2) | |
342 | ||
343 | def test_ephemeral_random(self): | |
344 | """ | |
345 | That 100% randomness causes all children to be pinned. | |
346 | """ | |
347 | self._setup_tree(random=1.0) | |
348 | self._wait_random_subtrees(100, status=self.status, rank="all") | |
349 | ||
350 | def test_ephemeral_random_max(self): | |
351 | """ | |
352 | That the config mds_export_ephemeral_random_max is not exceeded. | |
353 | """ | |
354 | ||
355 | r = 0.5 | |
356 | count = 1000 | |
357 | self._setup_tree(count=count, random=r) | |
358 | subtrees = self._wait_random_subtrees(int(r*count*.75), status=self.status, rank="all") | |
359 | self.config_set('mds', 'mds_export_ephemeral_random_max', 0.01) | |
360 | self._setup_tree(path="tree/new", count=count) | |
361 | time.sleep(30) # for something not to happen... | |
362 | subtrees = self._get_subtrees(status=self.status, rank="all", path="tree/new/") | |
363 | self.assertLessEqual(len(subtrees), int(.01*count*1.25)) | |
364 | ||
365 | def test_ephemeral_random_max_config(self): | |
366 | """ | |
367 | That the config mds_export_ephemeral_random_max config rejects new OOB policies. | |
368 | """ | |
369 | ||
370 | self.config_set('mds', 'mds_export_ephemeral_random_max', 0.01) | |
371 | try: | |
372 | p = self._setup_tree(count=1, random=0.02, wait=False) | |
373 | p.wait() | |
374 | except CommandFailedError as e: | |
375 | log.info(f"{e}") | |
376 | self.assertIn("Invalid", p.stderr.getvalue()) | |
377 | else: | |
378 | raise RuntimeError("mds_export_ephemeral_random_max ignored!") | |
379 | ||
380 | def test_ephemeral_random_dist(self): | |
381 | """ | |
382 | That ephemeral random and distributed can coexist with each other. | |
383 | """ | |
384 | ||
385 | self._setup_tree(random=1.0, distributed=True, export=0) | |
386 | self._wait_distributed_subtrees(100, status=self.status) | |
387 | self._wait_random_subtrees(100, status=self.status) | |
388 | ||
389 | def test_ephemeral_random_pin_override_before(self): | |
390 | """ | |
391 | That a conventional export pin overrides the random policy before creating new directories. | |
392 | """ | |
393 | ||
394 | self._setup_tree(count=0, random=1.0) | |
395 | self._setup_tree(path="tree/pin", count=10, export=1) | |
396 | self._wait_subtrees([("/tree/pin", 1)], status=self.status, rank=1, path="/tree/pin") | |
397 | ||
398 | def test_ephemeral_random_pin_override_after(self): | |
399 | """ | |
400 | That a conventional export pin overrides the random policy after creating new directories. | |
401 | """ | |
402 | ||
403 | count = 10 | |
404 | self._setup_tree(count=0, random=1.0) | |
405 | self._setup_tree(path="tree/pin", count=count) | |
406 | self._wait_random_subtrees(count+1, status=self.status, rank="all") | |
407 | self.mount_a.setfattr("tree/pin", "ceph.dir.pin", "1") | |
408 | self._wait_subtrees([("/tree/pin", 1)], status=self.status, rank=1, path="/tree/pin") | |
409 | ||
410 | def test_ephemeral_randomness(self): | |
411 | """ | |
412 | That the randomness is reasonable. | |
413 | """ | |
414 | ||
415 | r = random.uniform(0.25, 0.75) # ratios don't work for small r! | |
416 | count = 1000 | |
417 | self._setup_tree(count=count, random=r) | |
418 | subtrees = self._wait_random_subtrees(int(r*count*.50), status=self.status, rank="all") | |
419 | time.sleep(30) # for max to not be exceeded | |
420 | subtrees = self._wait_random_subtrees(int(r*count*.50), status=self.status, rank="all") | |
421 | self.assertLessEqual(len(subtrees), int(r*count*1.50)) | |
422 | ||
423 | def test_ephemeral_random_cache_drop(self): | |
424 | """ | |
425 | That the random ephemeral pin does not prevent empty (nothing in cache) subtree merging. | |
426 | """ | |
427 | ||
428 | count = 100 | |
429 | self._setup_tree(count=count, random=1.0) | |
430 | self._wait_random_subtrees(count, status=self.status, rank="all") | |
431 | self.mount_a.umount_wait() # release all caps | |
432 | def _drop(): | |
433 | self.fs.ranks_tell(["cache", "drop"], status=self.status) | |
434 | self._wait_subtrees([], status=self.status, action=_drop) | |
435 | ||
436 | def test_ephemeral_random_failover(self): | |
437 | """ | |
438 | That the random ephemeral pins stay pinned across MDS failover. | |
439 | """ | |
440 | ||
441 | count = 100 | |
442 | r = 0.5 | |
443 | self._setup_tree(count=count, random=r, export=0) | |
444 | # wait for all random subtrees to be created, not a specific count | |
445 | time.sleep(30) | |
446 | subtrees = self._wait_random_subtrees(1, status=self.status, rank=1) | |
447 | test = [(s['dir']['path'], s['auth_first']) for s in subtrees] | |
448 | before = self.fs.ranks_perf(lambda p: p['mds']['exported']) | |
449 | log.info(f"export stats: {before}") | |
450 | self.fs.rank_fail(rank=1) | |
451 | self.status = self.fs.wait_for_daemons() | |
452 | time.sleep(30) # waiting for something to not happen | |
453 | self._wait_subtrees(test, status=self.status, rank=1) | |
454 | after = self.fs.ranks_perf(lambda p: p['mds']['exported']) | |
455 | log.info(f"export stats: {after}") | |
456 | self.assertEqual(before, after) | |
457 | ||
458 | def test_ephemeral_pin_grow_mds(self): | |
459 | """ | |
460 | That consistent hashing works to reduce the number of migrations. | |
461 | """ | |
462 | ||
463 | self.fs.set_max_mds(2) | |
464 | self.status = self.fs.wait_for_daemons() | |
465 | ||
466 | self._setup_tree(distributed=True) | |
467 | subtrees_old = self._wait_distributed_subtrees(100, status=self.status, rank="all") | |
468 | ||
469 | self.fs.set_max_mds(3) | |
470 | self.status = self.fs.wait_for_daemons() | |
471 | ||
472 | # Sleeping for a while to allow the ephemeral pin migrations to complete | |
473 | time.sleep(30) | |
474 | ||
475 | subtrees_new = self._wait_distributed_subtrees(100, status=self.status, rank="all") | |
476 | count = 0 | |
477 | for old_subtree in subtrees_old: | |
478 | for new_subtree in subtrees_new: | |
479 | if (old_subtree['dir']['path'] == new_subtree['dir']['path']) and (old_subtree['auth_first'] != new_subtree['auth_first']): | |
480 | count = count + 1 | |
481 | break | |
482 | ||
483 | log.info("{0} migrations have occured due to the cluster resizing".format(count)) | |
484 | # ~50% of subtrees from the two rank will migrate to another rank | |
485 | self.assertLessEqual((count/len(subtrees_old)), (0.5)*1.25) # with 25% overbudget | |
486 | ||
487 | def test_ephemeral_pin_shrink_mds(self): | |
488 | """ | |
489 | That consistent hashing works to reduce the number of migrations. | |
490 | """ | |
491 | ||
492 | self.fs.set_max_mds(3) | |
493 | self.status = self.fs.wait_for_daemons() | |
494 | ||
495 | self._setup_tree(distributed=True) | |
496 | subtrees_old = self._wait_distributed_subtrees(100, status=self.status, rank="all") | |
497 | ||
498 | self.fs.set_max_mds(2) | |
499 | self.status = self.fs.wait_for_daemons() | |
500 | time.sleep(30) | |
501 | ||
502 | subtrees_new = self._wait_distributed_subtrees(100, status=self.status, rank="all") | |
503 | count = 0 | |
504 | for old_subtree in subtrees_old: | |
505 | for new_subtree in subtrees_new: | |
506 | if (old_subtree['dir']['path'] == new_subtree['dir']['path']) and (old_subtree['auth_first'] != new_subtree['auth_first']): | |
507 | count = count + 1 | |
508 | break | |
509 | ||
510 | log.info("{0} migrations have occured due to the cluster resizing".format(count)) | |
511 | # rebalancing from 3 -> 2 may cause half of rank 0/1 to move and all of rank 2 | |
512 | self.assertLessEqual((count/len(subtrees_old)), (1.0/3.0/2.0 + 1.0/3.0/2.0 + 1.0/3.0)*1.25) # aka .66 with 25% overbudget |