ceph/qa/tasks/object_source_down.py

   1 """
   2 Test Object locations going down
   3 """
   4 import logging
   5 import ceph_manager
   6 import time
   7 from teuthology import misc as teuthology
   8 from util.rados import rados
   9
  10 log = logging.getLogger(__name__)
  11
  12 def task(ctx, config):
  13     """
  14     Test handling of object location going down
  15     """
  16     if config is None:
  17         config = {}
  18     assert isinstance(config, dict), \
  19         'lost_unfound task only accepts a dict for configuration'
  20     first_mon = teuthology.get_first_mon(ctx, config)
  21     (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
  22
  23     manager = ceph_manager.CephManager(
  24         mon,
  25         ctx=ctx,
  26         logger=log.getChild('ceph_manager'),
  27         )
  28
  29     while len(manager.get_osd_status()['up']) < 3:
  30         time.sleep(10)
  31     manager.wait_for_clean()
  32
  33     # something that is always there
  34     dummyfile = '/etc/fstab'
  35
  36     # take 0, 1 out
  37     manager.mark_out_osd(0)
  38     manager.mark_out_osd(1)
  39     manager.wait_for_clean()
  40
  41     # delay recovery, and make the pg log very long (to prevent backfill)
  42     manager.raw_cluster_cmd(
  43             'tell', 'osd.0',
  44             'injectargs',
  45             '--osd-recovery-delay-start 10000 --osd-min-pg-log-entries 100000000'
  46             )
  47     # delay recovery, and make the pg log very long (to prevent backfill)
  48     manager.raw_cluster_cmd(
  49             'tell', 'osd.1',
  50             'injectargs',
  51             '--osd-recovery-delay-start 10000 --osd-min-pg-log-entries 100000000'
  52             )
  53     # delay recovery, and make the pg log very long (to prevent backfill)
  54     manager.raw_cluster_cmd(
  55             'tell', 'osd.2',
  56             'injectargs',
  57             '--osd-recovery-delay-start 10000 --osd-min-pg-log-entries 100000000'
  58             )
  59     # delay recovery, and make the pg log very long (to prevent backfill)
  60     manager.raw_cluster_cmd(
  61             'tell', 'osd.3',
  62             'injectargs',
  63             '--osd-recovery-delay-start 10000 --osd-min-pg-log-entries 100000000'
  64             )
  65
  66     # kludge to make sure they get a map
  67     rados(ctx, mon, ['-p', 'data', 'put', 'dummy', dummyfile])
  68
  69     # create old objects
  70     for f in range(1, 10):
  71         rados(ctx, mon, ['-p', 'data', 'put', 'existing_%d' % f, dummyfile])
  72
  73     manager.mark_out_osd(3)
  74     manager.wait_till_active()
  75
  76     manager.mark_in_osd(0)
  77     manager.wait_till_active()
  78
  79     manager.raw_cluster_cmd('tell', 'osd.2', 'flush_pg_stats')
  80     manager.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats')
  81
  82     manager.mark_out_osd(2)
  83     manager.wait_till_active()
  84
  85     # bring up 1
  86     manager.mark_in_osd(1)
  87     manager.wait_till_active()
  88
  89     manager.raw_cluster_cmd('tell', 'osd.1', 'flush_pg_stats')
  90     manager.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats')
  91     log.info("Getting unfound objects")
  92     unfound = manager.get_num_unfound_objects()
  93     assert not unfound
  94
  95     manager.kill_osd(2)
  96     manager.mark_down_osd(2)
  97     manager.kill_osd(3)
  98     manager.mark_down_osd(3)
  99
 100     manager.raw_cluster_cmd('tell', 'osd.1', 'flush_pg_stats')
 101     manager.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats')
 102     log.info("Getting unfound objects")
 103     unfound = manager.get_num_unfound_objects()
 104     assert unfound