[ceph.git] / ceph / qa / tasks / cephfs / test_snap_schedules.py

import os
import json
import time
import errno
import logging

from tasks.cephfs.cephfs_test_case import CephFSTestCase
from teuthology.exceptions import CommandFailedError
from datetime import datetime, timedelta

log = logging.getLogger(__name__)

def extract_schedule_and_retention_spec(spec=[]):
    schedule = set([s[0] for s in spec])
    retention = set([s[1] for s in spec])
    return (schedule, retention)

def seconds_upto_next_schedule(time_from, timo):
    ts = int(time_from)
    return ((int(ts / 60) * 60) + timo) - ts

class TestSnapSchedules(CephFSTestCase):
    CLIENTS_REQUIRED = 1

    TEST_VOLUME_NAME = 'snap_vol'
    TEST_DIRECTORY = 'snap_test_dir1'

    # this should be in sync with snap_schedule format
    SNAPSHOT_TS_FORMAT = '%Y-%m-%d-%H_%M_%S'

    def check_scheduled_snapshot(self, exec_time, timo):
        now = time.time()
        delta = now - exec_time
        log.debug(f'exec={exec_time}, now = {now}, timo = {timo}')
        # tolerate snapshot existance in the range [-5,+5]
        self.assertTrue((delta <= timo + 5) and (delta >= timo - 5))

    def _fs_cmd(self, *args):
        return self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", *args)

    def fs_snap_schedule_cmd(self, *args):
        args = list(args)
        args.append(f'fs={self.volname}')
        res = self._fs_cmd('snap-schedule', *args)
        log.debug(f'res={res}')
        return res

    def _create_or_reuse_test_volume(self):
        result = json.loads(self._fs_cmd("volume", "ls"))
        if len(result) == 0:
            self.vol_created = True
            self.volname = TestSnapSchedules.TEST_VOLUME_NAME
            self._fs_cmd("volume", "create", self.volname)
        else:
            self.volname = result[0]['name']

    def _enable_snap_schedule(self):
        return self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "module", "enable", "snap_schedule")

    def _disable_snap_schedule(self):
        return self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "module", "disable", "snap_schedule")

    def _allow_minute_granularity_snapshots(self):
        self.config_set('mgr', 'mgr/snap_schedule/allow_m_granularity', True)

    def setUp(self):
        super(TestSnapSchedules, self).setUp()
        self.volname = None
        self.vol_created = False
        self._create_or_reuse_test_volume()
        self.create_cbks = []
        self.remove_cbks = []
        # used to figure out which snapshots are created/deleted
        self.snapshots = set()
        self._enable_snap_schedule()
        self._allow_minute_granularity_snapshots()

    def tearDown(self):
        if self.vol_created:
            self._delete_test_volume()
        self._disable_snap_schedule()
        super(TestSnapSchedules, self).tearDown()

    def _schedule_to_timeout(self, schedule):
        mult = schedule[-1]
        period = int(schedule[0:-1])
        if mult == 'M':
            return period * 60
        elif mult == 'h':
            return period * 60 * 60
        elif mult == 'd':
            return period * 60 * 60 * 24
        elif mult == 'w':
            return period * 60 * 60 * 24 * 7
        else:
            raise RuntimeError('schedule multiplier not recognized')

    def add_snap_create_cbk(self, cbk):
        self.create_cbks.append(cbk)
    def remove_snap_create_cbk(self, cbk):
        self.create_cbks.remove(cbk)

    def add_snap_remove_cbk(self, cbk):
        self.remove_cbks.append(cbk)
    def remove_snap_remove_cbk(self, cbk):
        self.remove_cbks.remove(cbk)

    def assert_if_not_verified(self):
        self.assertTrue(len(self.create_cbks) == 0 and len(self.remove_cbks) == 0)

    def verify(self, dir_path, max_trials):
        trials = 0
        snap_path = "{0}/.snap".format(dir_path)
        while (len(self.create_cbks) or len(self.remove_cbks)) and trials < max_trials:
            snapshots = set(self.mount_a.ls(path=snap_path))
            added = snapshots - self.snapshots
            removed = self.snapshots - snapshots
            if added:
                for cbk in list(self.create_cbks):
                    res = cbk(list(added))
                    if res:
                        self.remove_snap_create_cbk(cbk)
                        break
            if removed:
                for cbk in list(self.remove_cbks):
                    res = cbk(list(removed))
                    if res:
                        self.remove_snap_remove_cbk(cbk)
                        break
            self.snapshots = snapshots
            trials += 1
            time.sleep(1)

    def calc_wait_time_and_snap_name(self, snap_sched_exec_epoch, schedule):
        timo = self._schedule_to_timeout(schedule)
        # calculate wait time upto the next minute
        wait_timo = seconds_upto_next_schedule(snap_sched_exec_epoch, timo)

        # expected "scheduled" snapshot name
        ts_name = (datetime.utcfromtimestamp(snap_sched_exec_epoch)
                   + timedelta(seconds=wait_timo)).strftime(TestSnapSchedules.SNAPSHOT_TS_FORMAT)
        return (wait_timo, ts_name)

    def verify_schedule(self, dir_path, schedules, retentions=[]):
        log.debug(f'expected_schedule: {schedules}, expected_retention: {retentions}')

        result = self.fs_snap_schedule_cmd('list', f'path={dir_path}', 'format=json')
        json_res = json.loads(result)
        log.debug(f'json_res: {json_res}')

        for schedule in schedules:
            self.assertTrue(schedule in json_res['schedule'])
        for retention in retentions:
            self.assertTrue(retention in json_res['retention'])

    def remove_snapshots(self, dir_path):
        snap_path = f'{dir_path}/.snap'

        snapshots = self.mount_a.ls(path=snap_path)
        for snapshot in snapshots:
            snapshot_path = os.path.join(snap_path, snapshot)
            log.debug(f'removing snapshot: {snapshot_path}')
            self.mount_a.run_shell(['rmdir', snapshot_path])

    def test_non_existent_snap_schedule_list(self):
        """Test listing snap schedules on a non-existing filesystem path failure"""
        try:
            self.fs_snap_schedule_cmd('list', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'format=json')
        except CommandFailedError as ce:
            if ce.exitstatus != errno.ENOENT:
                raise RuntimeError('incorrect errno when listing a non-existing snap schedule')
        else:
            raise RuntimeError('expected "fs snap-schedule list" to fail')

    def test_non_existent_schedule(self):
        """Test listing non-existing snap schedules failure"""
        self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])

        try:
            self.fs_snap_schedule_cmd('list', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'format=json')
        except CommandFailedError as ce:
            if ce.exitstatus != errno.ENOENT:
                raise RuntimeError('incorrect errno when listing a non-existing snap schedule')
        else:
            raise RuntimeError('expected "fs snap-schedule list" returned fail')

        self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])

    def test_snap_schedule_list_post_schedule_remove(self):
        """Test listing snap schedules post removal of a schedule"""
        self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])

        self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=1h')

        self.fs_snap_schedule_cmd('remove', f'path={TestSnapSchedules.TEST_DIRECTORY}')

        try:
            self.fs_snap_schedule_cmd('list', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'format=json')
        except CommandFailedError as ce:
            if ce.exitstatus != errno.ENOENT:
                raise RuntimeError('incorrect errno when listing a non-existing snap schedule')
        else:
            raise RuntimeError('"fs snap-schedule list" returned error')

        self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])

    def test_snap_schedule(self):
        """Test existence of a scheduled snapshot"""
        self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])

        # set a schedule on the dir
        self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=1M')
        exec_time = time.time()

        timo, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M')
        log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx} in ~{timo}s...')
        to_wait = timo + 2 # some leeway to avoid false failures...

        # verify snapshot schedule
        self.verify_schedule(TestSnapSchedules.TEST_DIRECTORY, ['1M'])

        def verify_added(snaps_added):
            log.debug(f'snapshots added={snaps_added}')
            self.assertEquals(len(snaps_added), 1)
            snapname = snaps_added[0]
            if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx:
                self.check_scheduled_snapshot(exec_time, timo)
                return True
            return False
        self.add_snap_create_cbk(verify_added)
        self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait)
        self.assert_if_not_verified()

        # remove snapshot schedule
        self.fs_snap_schedule_cmd('remove', f'path={TestSnapSchedules.TEST_DIRECTORY}')

        # remove all scheduled snapshots
        self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY)

        self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])

    def test_multi_snap_schedule(self):
        """Test exisitence of multiple scheduled snapshots"""
        self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])

        # set schedules on the dir
        self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=1M')
        self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=2M')
        exec_time = time.time()

        timo_1, snap_sfx_1 = self.calc_wait_time_and_snap_name(exec_time, '1M')
        log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx_1} in ~{timo_1}s...')
        timo_2, snap_sfx_2 = self.calc_wait_time_and_snap_name(exec_time, '2M')
        log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx_2} in ~{timo_2}s...')
        to_wait = timo_2 + 2 # use max timeout

        # verify snapshot schedule
        self.verify_schedule(TestSnapSchedules.TEST_DIRECTORY, ['1M', '2M'])

        def verify_added_1(snaps_added):
            log.debug(f'snapshots added={snaps_added}')
            self.assertEquals(len(snaps_added), 1)
            snapname = snaps_added[0]
            if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx_1:
                self.check_scheduled_snapshot(exec_time, timo_1)
                return True
            return False
        def verify_added_2(snaps_added):
            log.debug(f'snapshots added={snaps_added}')
            self.assertEquals(len(snaps_added), 1)
            snapname = snaps_added[0]
            if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx_2:
                self.check_scheduled_snapshot(exec_time, timo_2)
                return True
            return False
        self.add_snap_create_cbk(verify_added_1)
        self.add_snap_create_cbk(verify_added_2)
        self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait)
        self.assert_if_not_verified()

        # remove snapshot schedule
        self.fs_snap_schedule_cmd('remove', f'path={TestSnapSchedules.TEST_DIRECTORY}')

        # remove all scheduled snapshots
        self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY)

        self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])

    def test_snap_schedule_with_retention(self):
        """Test scheduled snapshots along with rentention policy"""
        self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])

        # set a schedule on the dir
        self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=1M')
        self.fs_snap_schedule_cmd('retention', 'add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'retention-spec-or-period=1M')
        exec_time = time.time()

        timo_1, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M')
        log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx} in ~{timo_1}s...')
        to_wait = timo_1 + 2 # some leeway to avoid false failures...

        # verify snapshot schedule
        self.verify_schedule(TestSnapSchedules.TEST_DIRECTORY, ['1M'], retentions=[{'M':1}])

        def verify_added(snaps_added):
            log.debug(f'snapshots added={snaps_added}')
            self.assertEquals(len(snaps_added), 1)
            snapname = snaps_added[0]
            if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx:
                self.check_scheduled_snapshot(exec_time, timo_1)
                return True
            return False
        self.add_snap_create_cbk(verify_added)
        self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait)
        self.assert_if_not_verified()

        timo_2 = timo_1 + 60 # expected snapshot removal timeout
        def verify_removed(snaps_removed):
            log.debug(f'snapshots removed={snaps_removed}')
            self.assertEquals(len(snaps_removed), 1)
            snapname = snaps_removed[0]
            if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx:
                self.check_scheduled_snapshot(exec_time, timo_2)
                return True
            return False
        log.debug(f'expecting removal of snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx} in ~{timo_2}s...')
        to_wait = timo_2
        self.add_snap_remove_cbk(verify_removed)
        self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait+2)
        self.assert_if_not_verified()

        # remove snapshot schedule
        self.fs_snap_schedule_cmd('remove', f'path={TestSnapSchedules.TEST_DIRECTORY}')

        # remove all scheduled snapshots
        self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY)

        self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])
Commit	Line	Data
f67539c2 TL	1	import os
	2	import json
	3	import time
	4	import errno
	5	import logging
	6
	7	from tasks.cephfs.cephfs_test_case import CephFSTestCase
	8	from teuthology.exceptions import CommandFailedError
	9	from datetime import datetime, timedelta
	10
	11	log = logging.getLogger(__name__)
	12
	13	def extract_schedule_and_retention_spec(spec=[]):
	14	schedule = set([s[0] for s in spec])
	15	retention = set([s[1] for s in spec])
	16	return (schedule, retention)
	17
	18	def seconds_upto_next_schedule(time_from, timo):
	19	ts = int(time_from)
	20	return ((int(ts / 60) * 60) + timo) - ts
	21
	22	class TestSnapSchedules(CephFSTestCase):
	23	CLIENTS_REQUIRED = 1
	24
	25	TEST_VOLUME_NAME = 'snap_vol'
	26	TEST_DIRECTORY = 'snap_test_dir1'
	27
	28	# this should be in sync with snap_schedule format
	29	SNAPSHOT_TS_FORMAT = '%Y-%m-%d-%H_%M_%S'
	30
	31	def check_scheduled_snapshot(self, exec_time, timo):
	32	now = time.time()
	33	delta = now - exec_time
	34	log.debug(f'exec={exec_time}, now = {now}, timo = {timo}')
	35	# tolerate snapshot existance in the range [-5,+5]
	36	self.assertTrue((delta <= timo + 5) and (delta >= timo - 5))
	37
	38	def _fs_cmd(self, *args):
	39	return self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", *args)
	40
	41	def fs_snap_schedule_cmd(self, *args):
	42	args = list(args)
	43	args.append(f'fs={self.volname}')
	44	res = self._fs_cmd('snap-schedule', *args)
	45	log.debug(f'res={res}')
	46	return res
	47
	48	def _create_or_reuse_test_volume(self):
	49	result = json.loads(self._fs_cmd("volume", "ls"))
	50	if len(result) == 0:
	51	self.vol_created = True
	52	self.volname = TestSnapSchedules.TEST_VOLUME_NAME
	53	self._fs_cmd("volume", "create", self.volname)
	54	else:
	55	self.volname = result[0]['name']
	56
	57	def _enable_snap_schedule(self):
	58	return self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "module", "enable", "snap_schedule")
	59
	60	def _disable_snap_schedule(self):
	61	return self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "module", "disable", "snap_schedule")
	62
	63	def _allow_minute_granularity_snapshots(self):
	64	self.config_set('mgr', 'mgr/snap_schedule/allow_m_granularity', True)
65
66	def setUp(self):
67	super(TestSnapSchedules, self).setUp()
68	self.volname = None
69	self.vol_created = False
70	self._create_or_reuse_test_volume()
71	self.create_cbks = []
72	self.remove_cbks = []
73	# used to figure out which snapshots are created/deleted
74	self.snapshots = set()
75	self._enable_snap_schedule()
76	self._allow_minute_granularity_snapshots()
77
78	def tearDown(self):
79	if self.vol_created:
80	self._delete_test_volume()
81	self._disable_snap_schedule()
82	super(TestSnapSchedules, self).tearDown()
83
84	def _schedule_to_timeout(self, schedule):
85	mult = schedule[-1]
86	period = int(schedule[0:-1])
87	if mult == 'M':
88	return period * 60
89	elif mult == 'h':
90	return period * 60 * 60
91	elif mult == 'd':
92	return period * 60 * 60 * 24
93	elif mult == 'w':
94	return period * 60 * 60 * 24 * 7
95	else:
96	raise RuntimeError('schedule multiplier not recognized')
97
98	def add_snap_create_cbk(self, cbk):
99	self.create_cbks.append(cbk)
100	def remove_snap_create_cbk(self, cbk):
101	self.create_cbks.remove(cbk)
102
103	def add_snap_remove_cbk(self, cbk):
104	self.remove_cbks.append(cbk)
105	def remove_snap_remove_cbk(self, cbk):
106	self.remove_cbks.remove(cbk)
107
108	def assert_if_not_verified(self):
109	self.assertTrue(len(self.create_cbks) == 0 and len(self.remove_cbks) == 0)
110
111	def verify(self, dir_path, max_trials):
112	trials = 0
113	snap_path = "{0}/.snap".format(dir_path)
114	while (len(self.create_cbks) or len(self.remove_cbks)) and trials < max_trials:
115	snapshots = set(self.mount_a.ls(path=snap_path))
116	added = snapshots - self.snapshots
117	removed = self.snapshots - snapshots
118	if added:
119	for cbk in list(self.create_cbks):
120	res = cbk(list(added))
121	if res:
122	self.remove_snap_create_cbk(cbk)
123	break
124	if removed:
125	for cbk in list(self.remove_cbks):
126	res = cbk(list(removed))
127	if res:
128	self.remove_snap_remove_cbk(cbk)
129	break
130	self.snapshots = snapshots
131	trials += 1
132	time.sleep(1)
133
134	def calc_wait_time_and_snap_name(self, snap_sched_exec_epoch, schedule):
135	timo = self._schedule_to_timeout(schedule)
136	# calculate wait time upto the next minute
137	wait_timo = seconds_upto_next_schedule(snap_sched_exec_epoch, timo)
138
139	# expected "scheduled" snapshot name
140	ts_name = (datetime.utcfromtimestamp(snap_sched_exec_epoch)
141	+ timedelta(seconds=wait_timo)).strftime(TestSnapSchedules.SNAPSHOT_TS_FORMAT)
142	return (wait_timo, ts_name)
143
144	def verify_schedule(self, dir_path, schedules, retentions=[]):
145	log.debug(f'expected_schedule: {schedules}, expected_retention: {retentions}')
146
147	result = self.fs_snap_schedule_cmd('list', f'path={dir_path}', 'format=json')
148	json_res = json.loads(result)
149	log.debug(f'json_res: {json_res}')
150
151	for schedule in schedules:
152	self.assertTrue(schedule in json_res['schedule'])
153	for retention in retentions:
154	self.assertTrue(retention in json_res['retention'])
155
156	def remove_snapshots(self, dir_path):
157	snap_path = f'{dir_path}/.snap'
158
159	snapshots = self.mount_a.ls(path=snap_path)
160	for snapshot in snapshots:
161	snapshot_path = os.path.join(snap_path, snapshot)
162	log.debug(f'removing snapshot: {snapshot_path}')
163	self.mount_a.run_shell(['rmdir', snapshot_path])
164
165	def test_non_existent_snap_schedule_list(self):
166	"""Test listing snap schedules on a non-existing filesystem path failure"""
167	try:
168	self.fs_snap_schedule_cmd('list', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'format=json')
169	except CommandFailedError as ce:
170	if ce.exitstatus != errno.ENOENT:
171	raise RuntimeError('incorrect errno when listing a non-existing snap schedule')
172	else:
173	raise RuntimeError('expected "fs snap-schedule list" to fail')
174
175	def test_non_existent_schedule(self):
176	"""Test listing non-existing snap schedules failure"""
177	self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])
178
179	try:
180	self.fs_snap_schedule_cmd('list', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'format=json')
181	except CommandFailedError as ce:
182	if ce.exitstatus != errno.ENOENT:
183	raise RuntimeError('incorrect errno when listing a non-existing snap schedule')
184	else:
185	raise RuntimeError('expected "fs snap-schedule list" returned fail')
186
187	self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])
188
189	def test_snap_schedule_list_post_schedule_remove(self):
190	"""Test listing snap schedules post removal of a schedule"""
191	self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])
192
193	self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=1h')
194
195	self.fs_snap_schedule_cmd('remove', f'path={TestSnapSchedules.TEST_DIRECTORY}')
196
197	try:
198	self.fs_snap_schedule_cmd('list', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'format=json')
199	except CommandFailedError as ce:
200	if ce.exitstatus != errno.ENOENT:
201	raise RuntimeError('incorrect errno when listing a non-existing snap schedule')
202	else:
203	raise RuntimeError('"fs snap-schedule list" returned error')
204
205	self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])
206
207	def test_snap_schedule(self):
208	"""Test existence of a scheduled snapshot"""
209	self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])
210
211	# set a schedule on the dir
212	self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=1M')
213	exec_time = time.time()
214
215	timo, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M')
216	log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx} in ~{timo}s...')
217	to_wait = timo + 2 # some leeway to avoid false failures...
218
219	# verify snapshot schedule
220	self.verify_schedule(TestSnapSchedules.TEST_DIRECTORY, ['1M'])
221
222	def verify_added(snaps_added):
223	log.debug(f'snapshots added={snaps_added}')
224	self.assertEquals(len(snaps_added), 1)
225	snapname = snaps_added[0]
226	if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx:
227	self.check_scheduled_snapshot(exec_time, timo)
228	return True
229	return False
230	self.add_snap_create_cbk(verify_added)
231	self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait)
232	self.assert_if_not_verified()
233
234	# remove snapshot schedule
235	self.fs_snap_schedule_cmd('remove', f'path={TestSnapSchedules.TEST_DIRECTORY}')
236
237	# remove all scheduled snapshots
238	self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY)
239
240	self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])
241
242	def test_multi_snap_schedule(self):
243	"""Test exisitence of multiple scheduled snapshots"""
244	self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])
245
246	# set schedules on the dir
247	self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=1M')
248	self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=2M')
249	exec_time = time.time()
250
251	timo_1, snap_sfx_1 = self.calc_wait_time_and_snap_name(exec_time, '1M')
252	log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx_1} in ~{timo_1}s...')
253	timo_2, snap_sfx_2 = self.calc_wait_time_and_snap_name(exec_time, '2M')
254	log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx_2} in ~{timo_2}s...')
255	to_wait = timo_2 + 2 # use max timeout
256
257	# verify snapshot schedule
258	self.verify_schedule(TestSnapSchedules.TEST_DIRECTORY, ['1M', '2M'])
259
260	def verify_added_1(snaps_added):
261	log.debug(f'snapshots added={snaps_added}')
262	self.assertEquals(len(snaps_added), 1)
263	snapname = snaps_added[0]
264	if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx_1:
265	self.check_scheduled_snapshot(exec_time, timo_1)
266	return True
267	return False
268	def verify_added_2(snaps_added):
269	log.debug(f'snapshots added={snaps_added}')
270	self.assertEquals(len(snaps_added), 1)
271	snapname = snaps_added[0]
272	if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx_2:
273	self.check_scheduled_snapshot(exec_time, timo_2)
274	return True
275	return False
276	self.add_snap_create_cbk(verify_added_1)
277	self.add_snap_create_cbk(verify_added_2)
278	self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait)
279	self.assert_if_not_verified()
280
281	# remove snapshot schedule
282	self.fs_snap_schedule_cmd('remove', f'path={TestSnapSchedules.TEST_DIRECTORY}')
283
284	# remove all scheduled snapshots
285	self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY)
286
287	self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])
288
289	def test_snap_schedule_with_retention(self):
290	"""Test scheduled snapshots along with rentention policy"""
291	self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY])
292
293	# set a schedule on the dir
294	self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=1M')
295	self.fs_snap_schedule_cmd('retention', 'add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'retention-spec-or-period=1M')
296	exec_time = time.time()
297
298	timo_1, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M')
299	log.debug(f'expecting snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx} in ~{timo_1}s...')
300	to_wait = timo_1 + 2 # some leeway to avoid false failures...
301
302	# verify snapshot schedule
303	self.verify_schedule(TestSnapSchedules.TEST_DIRECTORY, ['1M'], retentions=[{'M':1}])
304
305	def verify_added(snaps_added):
306	log.debug(f'snapshots added={snaps_added}')
307	self.assertEquals(len(snaps_added), 1)
308	snapname = snaps_added[0]
309	if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx:
310	self.check_scheduled_snapshot(exec_time, timo_1)
311	return True
312	return False
313	self.add_snap_create_cbk(verify_added)
314	self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait)
315	self.assert_if_not_verified()
316
317	timo_2 = timo_1 + 60 # expected snapshot removal timeout
318	def verify_removed(snaps_removed):
319	log.debug(f'snapshots removed={snaps_removed}')
320	self.assertEquals(len(snaps_removed), 1)
321	snapname = snaps_removed[0]
322	if snapname.startswith('scheduled-') and snapname[10:] == snap_sfx:
323	self.check_scheduled_snapshot(exec_time, timo_2)
324	return True
325	return False
326	log.debug(f'expecting removal of snap {TestSnapSchedules.TEST_DIRECTORY}/.snap/scheduled-{snap_sfx} in ~{timo_2}s...')
327	to_wait = timo_2
328	self.add_snap_remove_cbk(verify_removed)
329	self.verify(TestSnapSchedules.TEST_DIRECTORY, to_wait+2)
330	self.assert_if_not_verified()
331
332	# remove snapshot schedule
333	self.fs_snap_schedule_cmd('remove', f'path={TestSnapSchedules.TEST_DIRECTORY}')
334
335	# remove all scheduled snapshots
336	self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY)
337
338	self.mount_a.run_shell(['rmdir', TestSnapSchedules.TEST_DIRECTORY])