ceph/qa/tasks/cephfs/test_snap_schedules.py

   1 import os
   2 import json
   3 import time
   4 import errno
   5 import logging
   6
   7 from tasks.cephfs.cephfs_test_case import CephFSTestCase
   8 from teuthology.exceptions import CommandFailedError
   9 from datetime import datetime, timedelta
  10
  11 log = logging.getLogger(__name__)
  12
  13 def extract_schedule_and_retention_spec(spec=[]):
  14     schedule = set([s[0] for s in spec])
  15     retention = set([s[1] for s in spec])
  16     return (schedule, retention)
  17
  18 def seconds_upto_next_schedule(time_from, timo):
  19     ts = int(time_from)
  20     return ((int(ts / 60) * 60) + timo) - ts
  21
  22 class TestSnapSchedules(CephFSTestCase):
  23     CLIENTS_REQUIRED = 1
  24
  25     TEST_VOLUME_NAME = 'snap_vol'
  26     TEST_DIRECTORY = 'snap_test_dir1'
  27
  28     # this should be in sync with snap_schedule format
  29     SNAPSHOT_TS_FORMAT = '%Y-%m-%d-%H_%M_%S'
  30
  31     def check_scheduled_snapshot(self, exec_time, timo):
  32         now = time.time()
  33         delta = now - exec_time
  34         log.debug(f'exec={exec_time}, now = {now}, timo = {timo}')
  35         # tolerate snapshot existance in the range [-5,+5]
  36         self.assertTrue((delta <= timo + 5) and (delta >= timo - 5))
  37
  38     def _fs_cmd(self, *args):
  39         return self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", *args)
  40
  41     def fs_snap_schedule_cmd(self, *args, **kwargs):
  42         fs = kwargs.pop('fs', self.volname)
  43         args += ('--fs', fs)
  44         for name, val in kwargs.items():
  45             args += (f'--{name}', str(val))
  46         res = self._fs_cmd('snap-schedule', *args)
  47         log.debug(f'res={res}')
  48         return res
  49
  50     def _create_or_reuse_test_volume(self):
  51         result = json.loads(self._fs_cmd("volume", "ls"))
  52         if len(result) == 0:
  53             self.vol_created = True
  54             self.volname = TestSnapSchedules.TEST_VOLUME_NAME
  55             self._fs_cmd("volume", "create", self.volname)
  56         else:
  57             self.volname = result[0]['name']
  58
  59     def _enable_snap_schedule(self):
  60         return self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "module", "enable", "snap_schedule")
  61
  62     def _disable_snap_schedule(self):
  63         return self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "module", "disable", "snap_schedule")
  64
  65     def _allow_minute_granularity_snapshots(self):
  66         self.config_set('mgr', 'mgr/snap_schedule/allow_m_granularity', True)
  67
  68     def setUp(self):
  69         super(TestSnapSchedules, self).setUp()
  70         self.volname = None
  71         self.vol_created = False
  72         self._create_or_reuse_test_volume()
  73         self.create_cbks = []
  74         self.remove_cbks = []
  75         # used to figure out which snapshots are created/deleted
  76         self.snapshots = set()
  77         self._enable_snap_schedule()
  78         self._allow_minute_granularity_snapshots()
  79
  80     def tearDown(self):
  81         if self.vol_created:
  82             self._delete_test_volume()
  83         self._disable_snap_schedule()
  84         super(TestSnapSchedules, self).tearDown()
  85
  86     def _schedule_to_timeout(self, schedule):
  87         mult = schedule[-1]
  88         period = int(schedule[0:-1])
  89         if mult == 'M':
  90             return period * 60
  91         elif mult == 'h':
  92             return period * 60 * 60
  93         elif mult == 'd':
  94             return period * 60 * 60 * 24
  95         elif mult == 'w':
  96             return period * 60 * 60 * 24 * 7
  97         else:
  98             raise RuntimeError('schedule multiplier not recognized')
  99
 100     def add_snap_create_cbk(self, cbk):
 101         self.create_cbks.append(cbk)
 102     def remove_snap_create_cbk(self, cbk):
 103         self.create_cbks.remove(cbk)
 104
 105     def add_snap_remove_cbk(self, cbk):
 106         self.remove_cbks.append(cbk)
 107     def remove_snap_remove_cbk(self, cbk):
 108         self.remove_cbks.remove(cbk)
 109
 110     def assert_if_not_verified(self):
 111         self.assertTrue(len(self.create_cbks) == 0 and len(self.remove_cbks) == 0)
 112
 113     def verify(self, dir_path, max_trials):
 114         trials = 0
 115         snap_path = "{0}/.snap".format(dir_path)
 116         while (len(self.create_cbks) or len(self.remove_cbks)) and trials < max_trials:
 117             snapshots = set(self.mount_a.ls(path=snap_path))
 118             added = snapshots - self.snapshots
 119             removed = self.snapshots - snapshots
 120             if added:
 121                 for cbk in list(self.create_cbks):
 122                     res = cbk(list(added))
 123                     if res:
 124                         self.remove_snap_create_cbk(cbk)
 125                         break
 126             if removed:
 127                 for cbk in list(self.remove_cbks):
 128                     res = cbk(list(removed))
 129                     if res:
 130                         self.remove_snap_remove_cbk(cbk)
 131                         break
 132             self.snapshots = snapshots
 133             trials += 1
 134             time.sleep(1)
 135
 136     def calc_wait_time_and_snap_name(self, snap_sched_exec_epoch, schedule):
 137         timo = self._schedule_to_timeout(schedule)
 138         # calculate wait time upto the next minute
 139         wait_timo = seconds_upto_next_schedule(snap_sched_exec_epoch, timo)
 140
 141         # expected "scheduled" snapshot name
 142         ts_name = (datetime.utcfromtimestamp(snap_sched_exec_epoch)
 143                    + timedelta(seconds=wait_timo)).strftime(TestSnapSchedules.SNAPSHOT_TS_FORMAT)
 144         return (wait_timo, ts_name)
 145
 146     def verify_schedule(self, dir_path, schedules, retentions=[]):
 147         log.debug(f'expected_schedule: {schedules}, expected_retention: {retentions}')
 148
 149         result = self.fs_snap_schedule_cmd('list', path=dir_path, format='json')
 150         json_res = json.loads(result)
 151         log.debug(f'json_res: {json_res}')
 152
 153         for schedule in schedules:
 154             self.assertTrue(schedule in json_res['schedule'])
 155         for retention in retentions:
 156             self.assertTrue(retention in json_res['retention'])
 157
 158     def remove_snapshots(self, dir_path):
 159         snap_path = f'{dir_path}/.snap'
 160
 161         snapshots = self.mount_a.ls(path=snap_path)
 162         for snapshot in snapshots:
 163             snapshot_path = os.path.join(snap_path, snapshot)
 164             log.debug(f'removing snapshot: {snapshot_path}')
 165             self.mount_a.run_shell(['rmdir', snapshot_path])
 166
 167     def test_non_existent_snap_schedule_list(self):
 168         """Test listing snap schedules on a non-existing filesystem path failure"""
 169         try:
 170             self.fs_snap_schedule_cmd('list', path=TestSnapSchedules.TEST_DIRECTORY)
 171         except CommandFailedError as ce:
 172             if ce.exitstatus != errno.ENOENT:
 173                 raise RuntimeError('incorrect errno when listing a non-existing snap schedule')
 174         else:
 175             raise RuntimeError('expected "fs snap-schedule list" to fail')
 176
 177     def test_non_existent_schedule(self):
 178         """Test listing non-existing snap schedules failure"""
 179         self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])
 180
 181         try:
 182             self.fs_snap_schedule_cmd('list', path=TestSnapSchedules.TEST_DIRECTORY)
 183         except CommandFailedError as ce:
 184             if ce.exitstatus != errno.ENOENT:
 185                 raise RuntimeError('incorrect errno when listing a non-existing snap schedule')
 186         else:
 187             raise RuntimeError('expected "fs snap-schedule list" returned fail')
 188
 189         self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])
 190
 191     def test_snap_schedule_list_post_schedule_remove(self):
 192         """Test listing snap schedules post removal of a schedule"""
 193         self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])
 194
 195         self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='1h')
 196
 197         self.fs_snap_schedule_cmd('remove', path=TestSnapSchedules.TEST_DIRECTORY)
 198
 199         try:
 200             self.fs_snap_schedule_cmd('list', path=TestSnapSchedules.TEST_DIRECTORY)
 201         except CommandFailedError as ce:
 202             if ce.exitstatus != errno.ENOENT:
 203                 raise RuntimeError('incorrect errno when listing a non-existing snap schedule')
 204         else:
 205             raise RuntimeError('"fs snap-schedule list" returned error')
 206
 207         self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])
 208
 209     def test_snap_schedule(self):
 210         """Test existence of a scheduled snapshot"""
 211         self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])
 212
 213         # set a schedule on the dir
 214         self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='1M')
 215         exec_time = time.time()
 216
 217         timo, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M')
 218         log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx} in ~{timo}s...')
 219         to_wait = timo + 2 # some leeway to avoid false failures...
 220
 221         # verify snapshot schedule
 222         self.verify_schedule(TestSnapSchedules.TEST_DIRECTORY, ['1M'])
 223
 224         def verify_added(snaps_added):
 225             log.debug(f'snapshots added={snaps_added}')
 226             self.assertEquals(len(snaps_added), 1)
 227             snapname = snaps_added[0]
 228             if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx:
 229                 self.check_scheduled_snapshot(exec_time, timo)
 230                 return True
 231             return False
 232         self.add_snap_create_cbk(verify_added)
 233         self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait)
 234         self.assert_if_not_verified()
 235
 236         # remove snapshot schedule
 237         self.fs_snap_schedule_cmd('remove', path=TestSnapSchedules.TEST_DIRECTORY)
 238
 239         # remove all scheduled snapshots
 240         self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY)
 241
 242         self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])
 243
 244     def test_multi_snap_schedule(self):
 245         """Test exisitence of multiple scheduled snapshots"""
 246         self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])
 247
 248         # set schedules on the dir
 249         self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='1M')
 250         self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='2M')
 251         exec_time = time.time()
 252
 253         timo_1, snap_sfx_1 = self.calc_wait_time_and_snap_name(exec_time, '1M')
 254         log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx_1} in ~{timo_1}s...')
 255         timo_2, snap_sfx_2 = self.calc_wait_time_and_snap_name(exec_time, '2M')
 256         log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx_2} in ~{timo_2}s...')
 257         to_wait = timo_2 + 2 # use max timeout
 258
 259         # verify snapshot schedule
 260         self.verify_schedule(TestSnapSchedules.TEST_DIRECTORY, ['1M', '2M'])
 261
 262         def verify_added_1(snaps_added):
 263             log.debug(f'snapshots added={snaps_added}')
 264             self.assertEquals(len(snaps_added), 1)
 265             snapname = snaps_added[0]
 266             if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx_1:
 267                 self.check_scheduled_snapshot(exec_time, timo_1)
 268                 return True
 269             return False
 270         def verify_added_2(snaps_added):
 271             log.debug(f'snapshots added={snaps_added}')
 272             self.assertEquals(len(snaps_added), 1)
 273             snapname = snaps_added[0]
 274             if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx_2:
 275                 self.check_scheduled_snapshot(exec_time, timo_2)
 276                 return True
 277             return False
 278         self.add_snap_create_cbk(verify_added_1)
 279         self.add_snap_create_cbk(verify_added_2)
 280         self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait)
 281         self.assert_if_not_verified()
 282
 283         # remove snapshot schedule
 284         self.fs_snap_schedule_cmd('remove', path=TestSnapSchedules.TEST_DIRECTORY)
 285
 286         # remove all scheduled snapshots
 287         self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY)
 288
 289         self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])
 290
 291     def test_snap_schedule_with_retention(self):
 292         """Test scheduled snapshots along with rentention policy"""
 293         self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])
 294
 295         # set a schedule on the dir
 296         self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='1M')
 297         self.fs_snap_schedule_cmd('retention', 'add', path=TestSnapSchedules.TEST_DIRECTORY, retention_spec_or_period='1M')
 298         exec_time = time.time()
 299
 300         timo_1, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M')
 301         log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx} in ~{timo_1}s...')
 302         to_wait = timo_1 + 2 # some leeway to avoid false failures...
 303
 304         # verify snapshot schedule
 305         self.verify_schedule(TestSnapSchedules.TEST_DIRECTORY, ['1M'], retentions=[{'M':1}])
 306
 307         def verify_added(snaps_added):
 308             log.debug(f'snapshots added={snaps_added}')
 309             self.assertEquals(len(snaps_added), 1)
 310             snapname = snaps_added[0]
 311             if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx:
 312                 self.check_scheduled_snapshot(exec_time, timo_1)
 313                 return True
 314             return False
 315         self.add_snap_create_cbk(verify_added)
 316         self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait)
 317         self.assert_if_not_verified()
 318
 319         timo_2 = timo_1 + 60 # expected snapshot removal timeout
 320         def verify_removed(snaps_removed):
 321             log.debug(f'snapshots removed={snaps_removed}')
 322             self.assertEquals(len(snaps_removed), 1)
 323             snapname = snaps_removed[0]
 324             if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx:
 325                 self.check_scheduled_snapshot(exec_time, timo_2)
 326                 return True
 327             return False
 328         log.debug(f'expecting removal of snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx} in ~{timo_2}s...')
 329         to_wait = timo_2
 330         self.add_snap_remove_cbk(verify_removed)
 331         self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait+2)
 332         self.assert_if_not_verified()
 333
 334         # remove snapshot schedule
 335         self.fs_snap_schedule_cmd('remove', path=TestSnapSchedules.TEST_DIRECTORY)
 336
 337         # remove all scheduled snapshots
 338         self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY)
 339
 340         self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])