ceph/qa/tasks/mgr/test_progress.py

   1
   2 import json
   3 import logging
   4 import time
   5
   6 from .mgr_test_case import MgrTestCase
   7
   8
   9 log = logging.getLogger(__name__)
  10
  11
  12 class TestProgress(MgrTestCase):
  13     POOL = "progress_data"
  14
  15     # How long we expect to wait at most between taking an OSD out
  16     # and seeing the progress event pop up.
  17     EVENT_CREATION_PERIOD = 15
  18
  19     WRITE_PERIOD = 30
  20
  21     # Generous period for OSD recovery, should be same order of magnitude
  22     # to how long it took to write the data to begin with
  23     RECOVERY_PERIOD = WRITE_PERIOD * 4
  24
  25     def _get_progress(self):
  26         out = self.mgr_cluster.mon_manager.raw_cluster_cmd("progress", "json")
  27         return json.loads(out)
  28
  29     def _all_events(self):
  30         """
  31         To avoid racing on completion, we almost always want to look
  32         for events in the total list of active and complete, so
  33         munge them into a single list.
  34         """
  35         p = self._get_progress()
  36         log.info(json.dumps(p, indent=2))
  37         return p['events'] + p['completed']
  38
  39     def _events_in_progress(self):
  40         """
  41         this function returns all events that are in progress
  42         """
  43         p = self._get_progress()
  44         log.info(json.dumps(p, indent=2))
  45         return p['events']
  46
  47     def _completed_events(self):
  48         """
  49         This function returns all events that are completed
  50         """
  51         p = self._get_progress()
  52         log.info(json.dumps(p, indent=2))
  53         return p['completed']
  54
  55     def is_osd_marked_out(self, ev):
  56         return ev['message'].endswith('marked out')
  57
  58     def is_osd_marked_in(self, ev):
  59         return ev['message'].endswith('marked in')
  60
  61     def _get_osd_in_out_events(self, marked='both'):
  62         """
  63         Return the event that deals with OSDs being
  64         marked in, out or both
  65         """
  66
  67         marked_in_events = []
  68         marked_out_events = []
  69
  70         events_in_progress = self._events_in_progress()
  71         for ev in events_in_progress:
  72             if self.is_osd_marked_out(ev):
  73                 marked_out_events.append(ev)
  74             elif self.is_osd_marked_in(ev):
  75                 marked_in_events.append(ev)
  76
  77         if marked == 'both':
  78             return [marked_in_events] + [marked_out_events]
  79         elif marked == 'in':
  80             return marked_in_events
  81         else:
  82             return marked_out_events
  83
  84     def _osd_in_out_events_count(self, marked='both'):
  85         """
  86         Count the number of on going recovery events that deals with
  87         OSDs being marked in, out or both.
  88         """
  89         events_in_progress = self._events_in_progress()
  90         marked_in_count = 0
  91         marked_out_count = 0
  92
  93         for ev in events_in_progress:
  94             if self.is_osd_marked_out(ev):
  95                 marked_out_count += 1
  96             elif self.is_osd_marked_in(ev):
  97                 marked_in_count += 1
  98
  99         if marked == 'both':
 100             return marked_in_count + marked_out_count
 101         elif marked == 'in':
 102             return marked_in_count
 103         else:
 104             return marked_out_count
 105
 106     def _setup_pool(self, size=None):
 107         self.mgr_cluster.mon_manager.create_pool(self.POOL)
 108         if size is not None:
 109             self.mgr_cluster.mon_manager.raw_cluster_cmd(
 110                 'osd', 'pool', 'set', self.POOL, 'size', str(size))
 111
 112     def _osd_in_out_completed_events_count(self, marked='both'):
 113         """
 114         Count the number of completed recovery events that deals with
 115         OSDs being marked in, out, or both.
 116         """
 117
 118         completed_events = self._completed_events()
 119         marked_in_count = 0
 120         marked_out_count = 0
 121
 122         for ev in completed_events:
 123             if self.is_osd_marked_out(ev):
 124                 marked_out_count += 1
 125             elif self.is_osd_marked_in(ev):
 126                 marked_in_count += 1
 127
 128         if marked == 'both':
 129             return marked_in_count + marked_out_count
 130         elif marked == 'in':
 131             return marked_in_count
 132         else:
 133             return marked_out_count
 134
 135     def _write_some_data(self, t):
 136         """
 137         To adapt to test systems of varying performance, we write
 138         data for a defined time period, rather than to a defined
 139         capacity.  This will hopefully result in a similar timescale
 140         for PG recovery after an OSD failure.
 141         """
 142
 143         args = [
 144             "rados", "-p", self.POOL, "bench", str(t), "write", "-t", "16"]
 145
 146         self.mgr_cluster.admin_remote.run(args=args, wait=True)
 147
 148     def _osd_count(self):
 149         osd_map = self.mgr_cluster.mon_manager.get_osd_dump_json()
 150         return len(osd_map['osds'])
 151
 152     def setUp(self):
 153         super(TestProgress, self).setUp()
 154         # Ensure we have at least four OSDs
 155         if self._osd_count() < 4:
 156             self.skipTest("Not enough OSDS!")
 157
 158         # Remove any filesystems so that we can remove their pools
 159         if self.mds_cluster:
 160             self.mds_cluster.mds_stop()
 161             self.mds_cluster.mds_fail()
 162             self.mds_cluster.delete_all_filesystems()
 163
 164         # Remove all other pools
 165         for pool in self.mgr_cluster.mon_manager.get_osd_dump_json()['pools']:
 166             self.mgr_cluster.mon_manager.remove_pool(pool['pool_name'])
 167
 168         self._load_module("progress")
 169         self.mgr_cluster.mon_manager.raw_cluster_cmd('progress', 'clear')
 170
 171     def _simulate_failure(self, osd_ids=None):
 172         """
 173         Common lead-in to several tests: get some data in the cluster,
 174         then mark an OSD out to trigger the start of a progress event.
 175
 176         Return the JSON representation of the failure event.
 177         """
 178
 179         if osd_ids is None:
 180             osd_ids = [0]
 181
 182         self._setup_pool()
 183         self._write_some_data(self.WRITE_PERIOD)
 184
 185         for osd_id in osd_ids:
 186             self.mgr_cluster.mon_manager.raw_cluster_cmd(
 187                 'osd', 'out', str(osd_id))
 188
 189         # Wait for a progress event to pop up
 190         self.wait_until_equal(lambda: self._osd_in_out_events_count('out'), 1,
 191                               timeout=self.EVENT_CREATION_PERIOD*2,
 192                               period=1)
 193         ev = self._get_osd_in_out_events('out')[0]
 194         log.info(json.dumps(ev, indent=1))
 195         self.assertIn("Rebalancing after osd.0 marked out", ev['message'])
 196         return ev
 197
 198     def _simulate_back_in(self, osd_ids, initial_event):
 199         for osd_id in osd_ids:
 200             self.mgr_cluster.mon_manager.raw_cluster_cmd(
 201                     'osd', 'in', str(osd_id))
 202
 203         # First Event should complete promptly
 204         self.wait_until_true(lambda: self._is_complete(initial_event['id']),
 205                              timeout=self.EVENT_CREATION_PERIOD)
 206         try:
 207             # Wait for progress event marked in to pop up
 208             self.wait_until_equal(lambda: self._osd_in_out_events_count('in'), 1,
 209                                   timeout=self.EVENT_CREATION_PERIOD*2,
 210                                   period=1)
 211         except RuntimeError as ex:
 212             if not "Timed out after" in str(ex):
 213                 raise ex
 214
 215             log.info("There was no PGs affected by osd being marked in")
 216             return None
 217
 218         new_event = self._get_osd_in_out_events('in')[0]
 219         return new_event
 220
 221     def _no_events_anywhere(self):
 222         """
 223         Whether there are any live or completed events
 224         """
 225         p = self._get_progress()
 226         total_events = len(p['events']) + len(p['completed'])
 227         return total_events == 0
 228
 229     def _is_quiet(self):
 230         """
 231         Whether any progress events are live.
 232         """
 233         return len(self._get_progress()['events']) == 0
 234
 235     def _is_complete(self, ev_id):
 236         progress = self._get_progress()
 237         live_ids = [ev['id'] for ev in progress['events']]
 238         complete_ids = [ev['id'] for ev in progress['completed']]
 239         if ev_id in complete_ids:
 240             assert ev_id not in live_ids
 241             return True
 242         else:
 243             assert ev_id in live_ids
 244             return False
 245
 246     def tearDown(self):
 247         if self.POOL in self.mgr_cluster.mon_manager.pools:
 248             self.mgr_cluster.mon_manager.remove_pool(self.POOL)
 249
 250         osd_map = self.mgr_cluster.mon_manager.get_osd_dump_json()
 251         for osd in osd_map['osds']:
 252             if osd['weight'] == 0.0:
 253                 self.mgr_cluster.mon_manager.raw_cluster_cmd(
 254                     'osd', 'in', str(osd['osd']))
 255
 256         super(TestProgress, self).tearDown()
 257
 258     def test_osd_healthy_recovery(self):
 259         """
 260         The simple recovery case: an OSD goes down, its PGs get a new
 261         placement, and we wait for the PG to get healthy in its new
 262         locations.
 263         """
 264         ev = self._simulate_failure()
 265
 266         # Wait for progress event to ultimately reach completion
 267         self.wait_until_true(lambda: self._is_complete(ev['id']),
 268                              timeout=self.RECOVERY_PERIOD)
 269         self.assertEqual(self._osd_in_out_events_count(), 0)
 270
 271     def test_pool_removal(self):
 272         """
 273         That a pool removed during OSD recovery causes the
 274         progress event to be correctly marked complete once there
 275         is no more data to move.
 276         """
 277         ev = self._simulate_failure()
 278
 279         self.mgr_cluster.mon_manager.remove_pool(self.POOL)
 280
 281         # Event should complete promptly
 282         self.wait_until_true(lambda: self._is_complete(ev['id']),
 283                              timeout=self.EVENT_CREATION_PERIOD)
 284         self.assertEqual(self._osd_in_out_events_count(), 0)
 285
 286     def test_osd_came_back(self):
 287         """
 288         When a recovery is underway, but then the out OSD
 289         comes back in, such that recovery is no longer necessary.
 290         It should create another event for when osd is marked in
 291         and cancel the one that is still ongoing.
 292         """
 293         ev1 = self._simulate_failure()
 294
 295         ev2 = self._simulate_back_in([0], ev1)
 296
 297         if ev2 is not None:
 298             # Wait for progress event to ultimately complete
 299             self.wait_until_true(lambda: self._is_complete(ev2['id']),
 300                                  timeout=self.RECOVERY_PERIOD)
 301
 302         self.assertEqual(self._osd_in_out_events_count(), 0)
 303
 304     def test_osd_cannot_recover(self):
 305         """
 306         When the cluster cannot recover from a lost OSD, e.g.
 307         because there is no suitable new placement for it.
 308         (a size=3 pool when there are only 2 OSDs left)
 309         (a size=3 pool when the remaining osds are only on 2 hosts)
 310
 311         Progress event should not be created.
 312         """
 313
 314         pool_size = 3
 315
 316         self._setup_pool(size=pool_size)
 317         self._write_some_data(self.WRITE_PERIOD)
 318
 319         # Fail enough OSDs so there are less than N_replicas OSDs
 320         # available.
 321         osd_count = self._osd_count()
 322
 323         # First do some failures that will result in a normal rebalance
 324         # (Assumption: we're in a test environment that is configured
 325         #  not to require replicas be on different hosts, like teuthology)
 326         for osd_id in range(0, osd_count - pool_size):
 327             self.mgr_cluster.mon_manager.raw_cluster_cmd(
 328                 'osd', 'out', str(osd_id))
 329
 330         # We should see an event for each of the OSDs we took out
 331         self.wait_until_equal(
 332             lambda: self._osd_in_out_events_count('out'),
 333             osd_count - pool_size,
 334             timeout=self.EVENT_CREATION_PERIOD*(osd_count - pool_size))
 335
 336         # Those should complete cleanly
 337         self.wait_until_equal(
 338             lambda: self._osd_in_out_completed_events_count('out'),
 339             osd_count - pool_size,
 340             timeout=self.RECOVERY_PERIOD*(osd_count - pool_size)
 341         )
 342
 343         # Fail one last OSD, at the point the PGs have nowhere to go
 344         victim_osd = osd_count - pool_size
 345         self.mgr_cluster.mon_manager.raw_cluster_cmd(
 346                 'osd', 'out', str(victim_osd))
 347
 348         # Check that no event is created
 349         time.sleep(self.EVENT_CREATION_PERIOD)
 350
 351         self.assertEqual(
 352             self._osd_in_out_completed_events_count('out'),
 353             osd_count - pool_size)
 354
 355     def test_turn_off_module(self):
 356         """
 357         When the the module is turned off, there should not
 358         be any on going events or completed events.
 359         Also module should not accept any kind of Remote Event
 360         coming in from other module, however, once it is turned
 361         back, on creating an event should be working as it is.
 362         """
 363
 364         pool_size = 3
 365         self._setup_pool(size=pool_size)
 366         self._write_some_data(self.WRITE_PERIOD)
 367
 368         self.mgr_cluster.mon_manager.raw_cluster_cmd("progress", "off")
 369
 370         self.mgr_cluster.mon_manager.raw_cluster_cmd(
 371                 'osd', 'out', '0')
 372
 373         time.sleep(self.EVENT_CREATION_PERIOD)
 374
 375         self.mgr_cluster.mon_manager.raw_cluster_cmd(
 376                     'osd', 'in', '0')
 377
 378         time.sleep(self.EVENT_CREATION_PERIOD)
 379
 380         self.assertTrue(self._no_events_anywhere())
 381
 382         self.mgr_cluster.mon_manager.raw_cluster_cmd("progress", "on")
 383
 384         self._write_some_data(self.WRITE_PERIOD)
 385
 386         self.mgr_cluster.mon_manager.raw_cluster_cmd(
 387                 'osd', 'out', '0')
 388
 389         # Wait for a progress event to pop up
 390         self.wait_until_equal(lambda: self._osd_in_out_events_count('out'), 1,
 391                               timeout=self.EVENT_CREATION_PERIOD*2,
 392                               period=1)
 393
 394         ev1 = self._get_osd_in_out_events('out')[0]
 395
 396         log.info(json.dumps(ev1, indent=1))
 397
 398         self.wait_until_true(lambda: self._is_complete(ev1['id']),
 399                              timeout=self.RECOVERY_PERIOD)
 400         self.assertTrue(self._is_quiet())