]>
Commit | Line | Data |
---|---|---|
f67539c2 TL |
1 | import os |
2 | import json | |
3 | import time | |
4 | import errno | |
5 | import logging | |
6 | ||
7 | from tasks.cephfs.cephfs_test_case import CephFSTestCase | |
8 | from teuthology.exceptions import CommandFailedError | |
9 | from datetime import datetime, timedelta | |
10 | ||
11 | log = logging.getLogger(__name__) | |
12 | ||
13 | def extract_schedule_and_retention_spec(spec=[]): | |
14 | schedule = set([s[0] for s in spec]) | |
15 | retention = set([s[1] for s in spec]) | |
16 | return (schedule, retention) | |
17 | ||
18 | def seconds_upto_next_schedule(time_from, timo): | |
19 | ts = int(time_from) | |
20 | return ((int(ts / 60) * 60) + timo) - ts | |
21 | ||
2a845540 | 22 | class TestSnapSchedulesHelper(CephFSTestCase): |
f67539c2 TL |
23 | CLIENTS_REQUIRED = 1 |
24 | ||
25 | TEST_VOLUME_NAME = 'snap_vol' | |
26 | TEST_DIRECTORY = 'snap_test_dir1' | |
27 | ||
28 | # this should be in sync with snap_schedule format | |
29 | SNAPSHOT_TS_FORMAT = '%Y-%m-%d-%H_%M_%S' | |
30 | ||
31 | def check_scheduled_snapshot(self, exec_time, timo): | |
32 | now = time.time() | |
33 | delta = now - exec_time | |
34 | log.debug(f'exec={exec_time}, now = {now}, timo = {timo}') | |
35 | # tolerate snapshot existance in the range [-5,+5] | |
36 | self.assertTrue((delta <= timo + 5) and (delta >= timo - 5)) | |
37 | ||
38 | def _fs_cmd(self, *args): | |
39 | return self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", *args) | |
40 | ||
b3b6e05e | 41 | def fs_snap_schedule_cmd(self, *args, **kwargs): |
aee94f69 TL |
42 | if 'fs' in kwargs: |
43 | fs = kwargs.pop('fs') | |
44 | args += ('--fs', fs) | |
20effc67 TL |
45 | if 'format' in kwargs: |
46 | fmt = kwargs.pop('format') | |
47 | args += ('--format', fmt) | |
b3b6e05e | 48 | for name, val in kwargs.items(): |
20effc67 | 49 | args += (str(val),) |
f67539c2 TL |
50 | res = self._fs_cmd('snap-schedule', *args) |
51 | log.debug(f'res={res}') | |
52 | return res | |
53 | ||
54 | def _create_or_reuse_test_volume(self): | |
55 | result = json.loads(self._fs_cmd("volume", "ls")) | |
56 | if len(result) == 0: | |
57 | self.vol_created = True | |
2a845540 | 58 | self.volname = TestSnapSchedulesHelper.TEST_VOLUME_NAME |
f67539c2 TL |
59 | self._fs_cmd("volume", "create", self.volname) |
60 | else: | |
61 | self.volname = result[0]['name'] | |
62 | ||
63 | def _enable_snap_schedule(self): | |
64 | return self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "module", "enable", "snap_schedule") | |
65 | ||
66 | def _disable_snap_schedule(self): | |
67 | return self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "module", "disable", "snap_schedule") | |
68 | ||
69 | def _allow_minute_granularity_snapshots(self): | |
70 | self.config_set('mgr', 'mgr/snap_schedule/allow_m_granularity', True) | |
71 | ||
20effc67 TL |
72 | def _dump_on_update(self): |
73 | self.config_set('mgr', 'mgr/snap_schedule/dump_on_update', True) | |
74 | ||
f67539c2 | 75 | def setUp(self): |
2a845540 | 76 | super(TestSnapSchedulesHelper, self).setUp() |
f67539c2 TL |
77 | self.volname = None |
78 | self.vol_created = False | |
79 | self._create_or_reuse_test_volume() | |
80 | self.create_cbks = [] | |
81 | self.remove_cbks = [] | |
82 | # used to figure out which snapshots are created/deleted | |
83 | self.snapshots = set() | |
84 | self._enable_snap_schedule() | |
85 | self._allow_minute_granularity_snapshots() | |
20effc67 | 86 | self._dump_on_update() |
f67539c2 TL |
87 | |
88 | def tearDown(self): | |
89 | if self.vol_created: | |
90 | self._delete_test_volume() | |
91 | self._disable_snap_schedule() | |
2a845540 | 92 | super(TestSnapSchedulesHelper, self).tearDown() |
f67539c2 TL |
93 | |
94 | def _schedule_to_timeout(self, schedule): | |
95 | mult = schedule[-1] | |
96 | period = int(schedule[0:-1]) | |
97 | if mult == 'M': | |
98 | return period * 60 | |
99 | elif mult == 'h': | |
100 | return period * 60 * 60 | |
101 | elif mult == 'd': | |
102 | return period * 60 * 60 * 24 | |
103 | elif mult == 'w': | |
104 | return period * 60 * 60 * 24 * 7 | |
105 | else: | |
106 | raise RuntimeError('schedule multiplier not recognized') | |
107 | ||
108 | def add_snap_create_cbk(self, cbk): | |
109 | self.create_cbks.append(cbk) | |
110 | def remove_snap_create_cbk(self, cbk): | |
111 | self.create_cbks.remove(cbk) | |
112 | ||
113 | def add_snap_remove_cbk(self, cbk): | |
114 | self.remove_cbks.append(cbk) | |
115 | def remove_snap_remove_cbk(self, cbk): | |
116 | self.remove_cbks.remove(cbk) | |
117 | ||
118 | def assert_if_not_verified(self): | |
20effc67 TL |
119 | self.assertListEqual(self.create_cbks, []) |
120 | self.assertListEqual(self.remove_cbks, []) | |
f67539c2 TL |
121 | |
122 | def verify(self, dir_path, max_trials): | |
123 | trials = 0 | |
2a845540 | 124 | snap_path = f'{dir_path}/.snap' |
f67539c2 TL |
125 | while (len(self.create_cbks) or len(self.remove_cbks)) and trials < max_trials: |
126 | snapshots = set(self.mount_a.ls(path=snap_path)) | |
2a845540 | 127 | log.info(f'snapshots: {snapshots}') |
f67539c2 | 128 | added = snapshots - self.snapshots |
2a845540 | 129 | log.info(f'added: {added}') |
f67539c2 | 130 | removed = self.snapshots - snapshots |
2a845540 | 131 | log.info(f'removed: {removed}') |
f67539c2 TL |
132 | if added: |
133 | for cbk in list(self.create_cbks): | |
134 | res = cbk(list(added)) | |
135 | if res: | |
136 | self.remove_snap_create_cbk(cbk) | |
137 | break | |
138 | if removed: | |
139 | for cbk in list(self.remove_cbks): | |
140 | res = cbk(list(removed)) | |
141 | if res: | |
142 | self.remove_snap_remove_cbk(cbk) | |
143 | break | |
144 | self.snapshots = snapshots | |
145 | trials += 1 | |
146 | time.sleep(1) | |
147 | ||
148 | def calc_wait_time_and_snap_name(self, snap_sched_exec_epoch, schedule): | |
149 | timo = self._schedule_to_timeout(schedule) | |
150 | # calculate wait time upto the next minute | |
151 | wait_timo = seconds_upto_next_schedule(snap_sched_exec_epoch, timo) | |
152 | ||
153 | # expected "scheduled" snapshot name | |
154 | ts_name = (datetime.utcfromtimestamp(snap_sched_exec_epoch) | |
2a845540 | 155 | + timedelta(seconds=wait_timo)).strftime(TestSnapSchedulesHelper.SNAPSHOT_TS_FORMAT) |
f67539c2 TL |
156 | return (wait_timo, ts_name) |
157 | ||
158 | def verify_schedule(self, dir_path, schedules, retentions=[]): | |
159 | log.debug(f'expected_schedule: {schedules}, expected_retention: {retentions}') | |
160 | ||
b3b6e05e | 161 | result = self.fs_snap_schedule_cmd('list', path=dir_path, format='json') |
f67539c2 TL |
162 | json_res = json.loads(result) |
163 | log.debug(f'json_res: {json_res}') | |
164 | ||
165 | for schedule in schedules: | |
166 | self.assertTrue(schedule in json_res['schedule']) | |
167 | for retention in retentions: | |
168 | self.assertTrue(retention in json_res['retention']) | |
2a845540 TL |
169 | |
170 | class TestSnapSchedules(TestSnapSchedulesHelper): | |
f67539c2 TL |
171 | def remove_snapshots(self, dir_path): |
172 | snap_path = f'{dir_path}/.snap' | |
173 | ||
174 | snapshots = self.mount_a.ls(path=snap_path) | |
175 | for snapshot in snapshots: | |
176 | snapshot_path = os.path.join(snap_path, snapshot) | |
177 | log.debug(f'removing snapshot: {snapshot_path}') | |
178 | self.mount_a.run_shell(['rmdir', snapshot_path]) | |
179 | ||
180 | def test_non_existent_snap_schedule_list(self): | |
181 | """Test listing snap schedules on a non-existing filesystem path failure""" | |
182 | try: | |
b3b6e05e | 183 | self.fs_snap_schedule_cmd('list', path=TestSnapSchedules.TEST_DIRECTORY) |
f67539c2 TL |
184 | except CommandFailedError as ce: |
185 | if ce.exitstatus != errno.ENOENT: | |
186 | raise RuntimeError('incorrect errno when listing a non-existing snap schedule') | |
187 | else: | |
188 | raise RuntimeError('expected "fs snap-schedule list" to fail') | |
189 | ||
190 | def test_non_existent_schedule(self): | |
191 | """Test listing non-existing snap schedules failure""" | |
192 | self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY]) | |
193 | ||
194 | try: | |
b3b6e05e | 195 | self.fs_snap_schedule_cmd('list', path=TestSnapSchedules.TEST_DIRECTORY) |
f67539c2 TL |
196 | except CommandFailedError as ce: |
197 | if ce.exitstatus != errno.ENOENT: | |
198 | raise RuntimeError('incorrect errno when listing a non-existing snap schedule') | |
199 | else: | |
200 | raise RuntimeError('expected "fs snap-schedule list" returned fail') | |
201 | ||
202 | self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY]) | |
203 | ||
204 | def test_snap_schedule_list_post_schedule_remove(self): | |
205 | """Test listing snap schedules post removal of a schedule""" | |
206 | self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY]) | |
207 | ||
b3b6e05e | 208 | self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='1h') |
f67539c2 | 209 | |
b3b6e05e | 210 | self.fs_snap_schedule_cmd('remove', path=TestSnapSchedules.TEST_DIRECTORY) |
f67539c2 TL |
211 | |
212 | try: | |
b3b6e05e | 213 | self.fs_snap_schedule_cmd('list', path=TestSnapSchedules.TEST_DIRECTORY) |
f67539c2 TL |
214 | except CommandFailedError as ce: |
215 | if ce.exitstatus != errno.ENOENT: | |
216 | raise RuntimeError('incorrect errno when listing a non-existing snap schedule') | |
217 | else: | |
218 | raise RuntimeError('"fs snap-schedule list" returned error') | |
219 | ||
220 | self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY]) | |
221 | ||
222 | def test_snap_schedule(self): | |
223 | """Test existence of a scheduled snapshot""" | |
224 | self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY]) | |
225 | ||
226 | # set a schedule on the dir | |
b3b6e05e | 227 | self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='1M') |
f67539c2 TL |
228 | exec_time = time.time() |
229 | ||
230 | timo, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M') | |
231 | log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx} in ~{timo}s...') | |
232 | to_wait = timo + 2 # some leeway to avoid false failures... | |
233 | ||
234 | # verify snapshot schedule | |
235 | self.verify_schedule(TestSnapSchedules.TEST_DIRECTORY, ['1M']) | |
236 | ||
237 | def verify_added(snaps_added): | |
238 | log.debug(f'snapshots added={snaps_added}') | |
20effc67 | 239 | self.assertEqual(len(snaps_added), 1) |
f67539c2 | 240 | snapname = snaps_added[0] |
20effc67 TL |
241 | if snapname.startswith('scheduled-'): |
242 | if snapname[10:26] == snap_sfx[:16]: | |
243 | self.check_scheduled_snapshot(exec_time, timo) | |
244 | return True | |
f67539c2 TL |
245 | return False |
246 | self.add_snap_create_cbk(verify_added) | |
247 | self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait) | |
248 | self.assert_if_not_verified() | |
249 | ||
250 | # remove snapshot schedule | |
b3b6e05e | 251 | self.fs_snap_schedule_cmd('remove', path=TestSnapSchedules.TEST_DIRECTORY) |
f67539c2 TL |
252 | |
253 | # remove all scheduled snapshots | |
254 | self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY) | |
255 | ||
256 | self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY]) | |
257 | ||
258 | def test_multi_snap_schedule(self): | |
259 | """Test exisitence of multiple scheduled snapshots""" | |
260 | self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY]) | |
261 | ||
262 | # set schedules on the dir | |
b3b6e05e TL |
263 | self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='1M') |
264 | self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='2M') | |
f67539c2 TL |
265 | exec_time = time.time() |
266 | ||
267 | timo_1, snap_sfx_1 = self.calc_wait_time_and_snap_name(exec_time, '1M') | |
268 | log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx_1} in ~{timo_1}s...') | |
269 | timo_2, snap_sfx_2 = self.calc_wait_time_and_snap_name(exec_time, '2M') | |
270 | log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx_2} in ~{timo_2}s...') | |
271 | to_wait = timo_2 + 2 # use max timeout | |
272 | ||
273 | # verify snapshot schedule | |
274 | self.verify_schedule(TestSnapSchedules.TEST_DIRECTORY, ['1M', '2M']) | |
275 | ||
276 | def verify_added_1(snaps_added): | |
277 | log.debug(f'snapshots added={snaps_added}') | |
20effc67 | 278 | self.assertEqual(len(snaps_added), 1) |
f67539c2 | 279 | snapname = snaps_added[0] |
20effc67 TL |
280 | if snapname.startswith('scheduled-'): |
281 | if snapname[10:26] == snap_sfx_1[:16]: | |
282 | self.check_scheduled_snapshot(exec_time, timo_1) | |
283 | return True | |
f67539c2 TL |
284 | return False |
285 | def verify_added_2(snaps_added): | |
286 | log.debug(f'snapshots added={snaps_added}') | |
20effc67 | 287 | self.assertEqual(len(snaps_added), 1) |
f67539c2 | 288 | snapname = snaps_added[0] |
20effc67 TL |
289 | if snapname.startswith('scheduled-'): |
290 | if snapname[10:26] == snap_sfx_2[:16]: | |
291 | self.check_scheduled_snapshot(exec_time, timo_2) | |
292 | return True | |
f67539c2 TL |
293 | return False |
294 | self.add_snap_create_cbk(verify_added_1) | |
295 | self.add_snap_create_cbk(verify_added_2) | |
296 | self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait) | |
297 | self.assert_if_not_verified() | |
298 | ||
299 | # remove snapshot schedule | |
b3b6e05e | 300 | self.fs_snap_schedule_cmd('remove', path=TestSnapSchedules.TEST_DIRECTORY) |
f67539c2 TL |
301 | |
302 | # remove all scheduled snapshots | |
303 | self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY) | |
304 | ||
305 | self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY]) | |
306 | ||
307 | def test_snap_schedule_with_retention(self): | |
308 | """Test scheduled snapshots along with rentention policy""" | |
309 | self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY]) | |
310 | ||
311 | # set a schedule on the dir | |
b3b6e05e TL |
312 | self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='1M') |
313 | self.fs_snap_schedule_cmd('retention', 'add', path=TestSnapSchedules.TEST_DIRECTORY, retention_spec_or_period='1M') | |
f67539c2 TL |
314 | exec_time = time.time() |
315 | ||
316 | timo_1, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M') | |
317 | log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx} in ~{timo_1}s...') | |
318 | to_wait = timo_1 + 2 # some leeway to avoid false failures... | |
319 | ||
320 | # verify snapshot schedule | |
321 | self.verify_schedule(TestSnapSchedules.TEST_DIRECTORY, ['1M'], retentions=[{'M':1}]) | |
322 | ||
323 | def verify_added(snaps_added): | |
324 | log.debug(f'snapshots added={snaps_added}') | |
20effc67 | 325 | self.assertEqual(len(snaps_added), 1) |
f67539c2 | 326 | snapname = snaps_added[0] |
20effc67 TL |
327 | if snapname.startswith('scheduled-'): |
328 | if snapname[10:26] == snap_sfx[:16]: | |
329 | self.check_scheduled_snapshot(exec_time, timo_1) | |
330 | return True | |
f67539c2 TL |
331 | return False |
332 | self.add_snap_create_cbk(verify_added) | |
333 | self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait) | |
334 | self.assert_if_not_verified() | |
335 | ||
336 | timo_2 = timo_1 + 60 # expected snapshot removal timeout | |
337 | def verify_removed(snaps_removed): | |
338 | log.debug(f'snapshots removed={snaps_removed}') | |
20effc67 | 339 | self.assertEqual(len(snaps_removed), 1) |
f67539c2 | 340 | snapname = snaps_removed[0] |
20effc67 TL |
341 | if snapname.startswith('scheduled-'): |
342 | if snapname[10:26] == snap_sfx[:16]: | |
343 | self.check_scheduled_snapshot(exec_time, timo_2) | |
344 | return True | |
f67539c2 TL |
345 | return False |
346 | log.debug(f'expecting removal of snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx} in ~{timo_2}s...') | |
347 | to_wait = timo_2 | |
348 | self.add_snap_remove_cbk(verify_removed) | |
349 | self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait+2) | |
350 | self.assert_if_not_verified() | |
351 | ||
352 | # remove snapshot schedule | |
b3b6e05e | 353 | self.fs_snap_schedule_cmd('remove', path=TestSnapSchedules.TEST_DIRECTORY) |
f67539c2 TL |
354 | |
355 | # remove all scheduled snapshots | |
356 | self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY) | |
357 | ||
358 | self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY]) | |
20effc67 | 359 | |
1d09f67e | 360 | def get_snap_stats(self, dir_path): |
20effc67 TL |
361 | snap_path = f"{dir_path}/.snap"[1:] |
362 | snapshots = self.mount_a.ls(path=snap_path) | |
363 | fs_count = len(snapshots) | |
2a845540 | 364 | log.debug(f'snapshots: {snapshots}') |
20effc67 | 365 | |
1d09f67e | 366 | result = self.fs_snap_schedule_cmd('status', path=dir_path, |
39ae355f | 367 | format='json') |
20effc67 TL |
368 | json_res = json.loads(result)[0] |
369 | db_count = int(json_res['created_count']) | |
370 | log.debug(f'json_res: {json_res}') | |
371 | ||
1d09f67e TL |
372 | snap_stats = dict() |
373 | snap_stats['fs_count'] = fs_count | |
374 | snap_stats['db_count'] = db_count | |
375 | ||
aee94f69 TL |
376 | log.debug(f'fs_count: {fs_count}') |
377 | log.debug(f'db_count: {db_count}') | |
378 | ||
1d09f67e TL |
379 | return snap_stats |
380 | ||
381 | def verify_snap_stats(self, dir_path): | |
382 | snap_stats = self.get_snap_stats(dir_path) | |
383 | self.assertTrue(snap_stats['fs_count'] == snap_stats['db_count']) | |
20effc67 TL |
384 | |
385 | def test_concurrent_snap_creates(self): | |
1d09f67e | 386 | """Test concurrent snap creates in same file-system without db issues""" |
20effc67 TL |
387 | """ |
388 | Test snap creates at same cadence on same fs to verify correct stats. | |
389 | A single SQLite DB Connection handle cannot be used to run concurrent | |
390 | transactions and results transaction aborts. This test makes sure that | |
391 | proper care has been taken in the code to avoid such situation by | |
392 | verifying number of dirs created on the file system with the | |
393 | created_count in the schedule_meta table for the specific path. | |
394 | """ | |
395 | self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY]) | |
396 | ||
397 | testdirs = [] | |
398 | for d in range(10): | |
399 | testdirs.append(os.path.join("/", TestSnapSchedules.TEST_DIRECTORY, "dir" + str(d))) | |
400 | ||
401 | for d in testdirs: | |
402 | self.mount_a.run_shell(['mkdir', '-p', d[1:]]) | |
403 | self.fs_snap_schedule_cmd('add', path=d, snap_schedule='1M') | |
404 | ||
405 | exec_time = time.time() | |
406 | timo_1, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M') | |
407 | ||
408 | for d in testdirs: | |
409 | self.fs_snap_schedule_cmd('activate', path=d, snap_schedule='1M') | |
410 | ||
411 | # we wait for 10 snaps to be taken | |
412 | wait_time = timo_1 + 10 * 60 + 15 | |
413 | time.sleep(wait_time) | |
414 | ||
415 | for d in testdirs: | |
416 | self.fs_snap_schedule_cmd('deactivate', path=d, snap_schedule='1M') | |
417 | ||
418 | for d in testdirs: | |
419 | self.verify_snap_stats(d) | |
420 | ||
421 | for d in testdirs: | |
422 | self.fs_snap_schedule_cmd('remove', path=d, snap_schedule='1M') | |
423 | self.remove_snapshots(d[1:]) | |
424 | self.mount_a.run_shell(['rmdir', d[1:]]) | |
1d09f67e TL |
425 | |
426 | def test_snap_schedule_with_mgr_restart(self): | |
427 | """Test that snap schedule is resumed after mgr restart""" | |
428 | self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY]) | |
429 | testdir = os.path.join("/", TestSnapSchedules.TEST_DIRECTORY, "test_restart") | |
430 | self.mount_a.run_shell(['mkdir', '-p', testdir[1:]]) | |
431 | self.fs_snap_schedule_cmd('add', path=testdir, snap_schedule='1M') | |
432 | ||
433 | exec_time = time.time() | |
434 | timo_1, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M') | |
435 | ||
436 | self.fs_snap_schedule_cmd('activate', path=testdir, snap_schedule='1M') | |
437 | ||
438 | # we wait for 10 snaps to be taken | |
439 | wait_time = timo_1 + 10 * 60 + 15 | |
440 | time.sleep(wait_time) | |
441 | ||
442 | old_stats = self.get_snap_stats(testdir) | |
443 | self.assertTrue(old_stats['fs_count'] == old_stats['db_count']) | |
444 | self.assertTrue(old_stats['fs_count'] > 9) | |
445 | ||
446 | # restart mgr | |
447 | active_mgr = self.mgr_cluster.mon_manager.get_mgr_dump()['active_name'] | |
448 | log.debug(f'restarting active mgr: {active_mgr}') | |
449 | self.mgr_cluster.mon_manager.revive_mgr(active_mgr) | |
450 | time.sleep(300) # sleep for 5 minutes | |
451 | self.fs_snap_schedule_cmd('deactivate', path=testdir, snap_schedule='1M') | |
452 | ||
453 | new_stats = self.get_snap_stats(testdir) | |
454 | self.assertTrue(new_stats['fs_count'] == new_stats['db_count']) | |
455 | self.assertTrue(new_stats['fs_count'] > old_stats['fs_count']) | |
456 | self.assertTrue(new_stats['db_count'] > old_stats['db_count']) | |
457 | ||
458 | # cleanup | |
459 | self.fs_snap_schedule_cmd('remove', path=testdir, snap_schedule='1M') | |
460 | self.remove_snapshots(testdir[1:]) | |
2a845540 TL |
461 | self.mount_a.run_shell(['rmdir', testdir[1:]]) |
462 | ||
1e59de90 TL |
463 | def test_schedule_auto_deactivation_for_non_existent_path(self): |
464 | """ | |
465 | Test that a non-existent path leads to schedule deactivation after a few retries. | |
466 | """ | |
467 | self.fs_snap_schedule_cmd('add', path="/bad-path", snap_schedule='1M') | |
468 | start_time = time.time() | |
469 | ||
470 | while time.time() - start_time < 60.0: | |
471 | s = self.fs_snap_schedule_cmd('status', path="/bad-path", format='json') | |
472 | json_status = json.loads(s)[0] | |
473 | ||
474 | self.assertTrue(int(json_status['active']) == 1) | |
475 | time.sleep(60) | |
476 | ||
477 | s = self.fs_snap_schedule_cmd('status', path="/bad-path", format='json') | |
478 | json_status = json.loads(s)[0] | |
479 | self.assertTrue(int(json_status['active']) == 0) | |
480 | ||
481 | # remove snapshot schedule | |
482 | self.fs_snap_schedule_cmd('remove', path="/bad-path") | |
483 | ||
aee94f69 TL |
484 | def test_snap_schedule_for_number_of_snaps_retention(self): |
485 | """ | |
486 | Test that number of snaps retained are as per user spec. | |
487 | """ | |
488 | total_snaps = 55 | |
489 | test_dir = '/' + TestSnapSchedules.TEST_DIRECTORY | |
490 | ||
491 | self.mount_a.run_shell(['mkdir', '-p', test_dir[1:]]) | |
492 | ||
493 | # set a schedule on the dir | |
494 | self.fs_snap_schedule_cmd('add', path=test_dir, snap_schedule='1M') | |
495 | self.fs_snap_schedule_cmd('retention', 'add', path=test_dir, | |
496 | retention_spec_or_period=f'{total_snaps}n') | |
497 | exec_time = time.time() | |
498 | ||
499 | timo_1, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M') | |
500 | ||
501 | # verify snapshot schedule | |
502 | self.verify_schedule(test_dir, ['1M']) | |
503 | ||
504 | # we wait for total_snaps snaps to be taken | |
505 | wait_time = timo_1 + total_snaps * 60 + 15 | |
506 | time.sleep(wait_time) | |
507 | ||
508 | snap_stats = self.get_snap_stats(test_dir) | |
509 | self.assertTrue(snap_stats['fs_count'] == total_snaps) | |
510 | self.assertTrue(snap_stats['db_count'] >= total_snaps) | |
511 | ||
512 | # remove snapshot schedule | |
513 | self.fs_snap_schedule_cmd('remove', path=test_dir) | |
514 | ||
515 | # remove all scheduled snapshots | |
516 | self.remove_snapshots(test_dir[1:]) | |
517 | ||
518 | self.mount_a.run_shell(['rmdir', test_dir[1:]]) | |
519 | ||
1e59de90 | 520 | |
2a845540 TL |
521 | class TestSnapSchedulesSnapdir(TestSnapSchedulesHelper): |
522 | def remove_snapshots(self, dir_path, sdn): | |
523 | snap_path = f'{dir_path}/{sdn}' | |
524 | ||
525 | snapshots = self.mount_a.ls(path=snap_path) | |
526 | for snapshot in snapshots: | |
527 | snapshot_path = os.path.join(snap_path, snapshot) | |
528 | log.debug(f'removing snapshot: {snapshot_path}') | |
529 | self.mount_a.run_shell(['rmdir', snapshot_path]) | |
530 | ||
531 | def get_snap_dir_name(self): | |
532 | from tasks.cephfs.fuse_mount import FuseMount | |
533 | from tasks.cephfs.kernel_mount import KernelMount | |
534 | ||
535 | if isinstance(self.mount_a, KernelMount): | |
536 | sdn = self.mount_a.client_config.get('snapdirname', '.snap') | |
537 | elif isinstance(self.mount_a, FuseMount): | |
538 | sdn = self.mount_a.client_config.get('client_snapdir', '.snap') | |
539 | self.fs.set_ceph_conf('client', 'client snapdir', sdn) | |
540 | self.mount_a.remount() | |
541 | return sdn | |
542 | ||
543 | def test_snap_dir_name(self): | |
544 | """Test the correctness of snap directory name""" | |
545 | self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedulesSnapdir.TEST_DIRECTORY]) | |
546 | ||
547 | # set a schedule on the dir | |
548 | self.fs_snap_schedule_cmd('add', path=TestSnapSchedulesSnapdir.TEST_DIRECTORY, snap_schedule='1M') | |
549 | self.fs_snap_schedule_cmd('retention', 'add', path=TestSnapSchedulesSnapdir.TEST_DIRECTORY, retention_spec_or_period='1M') | |
550 | exec_time = time.time() | |
551 | ||
552 | timo, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M') | |
553 | sdn = self.get_snap_dir_name() | |
554 | log.info(f'expecting snap {TestSnapSchedulesSnapdir.TEST_DIRECTORY}/{sdn}/scheduled-{snap_sfx} in ~{timo}s...') | |
555 | ||
556 | # verify snapshot schedule | |
557 | self.verify_schedule(TestSnapSchedulesSnapdir.TEST_DIRECTORY, ['1M'], retentions=[{'M':1}]) | |
558 | ||
559 | # remove snapshot schedule | |
560 | self.fs_snap_schedule_cmd('remove', path=TestSnapSchedulesSnapdir.TEST_DIRECTORY) | |
561 | ||
562 | # remove all scheduled snapshots | |
563 | self.remove_snapshots(TestSnapSchedulesSnapdir.TEST_DIRECTORY, sdn) | |
564 | ||
565 | self.mount_a.run_shell(['rmdir', TestSnapSchedulesSnapdir.TEST_DIRECTORY]) | |
aee94f69 TL |
566 | |
567 | ||
568 | """ | |
569 | Note that the class TestSnapSchedulesMandatoryFSArgument tests snap-schedule | |
570 | commands only for multi-fs scenario. Commands for a single default fs should | |
571 | pass for tests defined above or elsewhere. | |
572 | """ | |
573 | ||
574 | ||
575 | class TestSnapSchedulesMandatoryFSArgument(TestSnapSchedulesHelper): | |
576 | REQUIRE_BACKUP_FILESYSTEM = True | |
577 | TEST_DIRECTORY = 'mandatory_fs_argument_test_dir' | |
578 | ||
579 | def test_snap_schedule_without_fs_argument(self): | |
580 | """Test command fails without --fs argument in presence of multiple fs""" | |
581 | test_path = TestSnapSchedulesMandatoryFSArgument.TEST_DIRECTORY | |
582 | self.mount_a.run_shell(['mkdir', '-p', test_path]) | |
583 | ||
584 | # try setting a schedule on the dir; this should fail now that we are | |
585 | # working with mutliple fs; we need the --fs argument if there are more | |
586 | # than one fs hosted by the same cluster | |
587 | with self.assertRaises(CommandFailedError): | |
588 | self.fs_snap_schedule_cmd('add', test_path, snap_schedule='1M') | |
589 | ||
590 | self.mount_a.run_shell(['rmdir', test_path]) | |
591 | ||
592 | def test_snap_schedule_for_non_default_fs(self): | |
593 | """Test command succes with --fs argument for non-default fs""" | |
594 | test_path = TestSnapSchedulesMandatoryFSArgument.TEST_DIRECTORY | |
595 | self.mount_a.run_shell(['mkdir', '-p', test_path]) | |
596 | ||
597 | # use the backup fs as the second fs; all these commands must pass | |
598 | self.fs_snap_schedule_cmd('add', test_path, snap_schedule='1M', fs='backup_fs') | |
599 | self.fs_snap_schedule_cmd('activate', test_path, snap_schedule='1M', fs='backup_fs') | |
600 | self.fs_snap_schedule_cmd('retention', 'add', test_path, retention_spec_or_period='1M', fs='backup_fs') | |
601 | self.fs_snap_schedule_cmd('list', test_path, fs='backup_fs', format='json') | |
602 | self.fs_snap_schedule_cmd('status', test_path, fs='backup_fs', format='json') | |
603 | self.fs_snap_schedule_cmd('retention', 'remove', test_path, retention_spec_or_period='1M', fs='backup_fs') | |
604 | self.fs_snap_schedule_cmd('deactivate', test_path, snap_schedule='1M', fs='backup_fs') | |
605 | self.fs_snap_schedule_cmd('remove', test_path, snap_schedule='1M', fs='backup_fs') | |
606 | ||
607 | self.mount_a.run_shell(['rmdir', test_path]) |