ceph/qa/tasks/divergent_priors.py

   1 """
   2 Special case divergence test
   3 """
   4 import logging
   5 import time
   6
   7 from teuthology import misc as teuthology
   8 from util.rados import rados
   9
  10
  11 log = logging.getLogger(__name__)
  12
  13
  14 def task(ctx, config):
  15     """
  16     Test handling of divergent entries with prior_version
  17     prior to log_tail
  18
  19     overrides:
  20       ceph:
  21         conf:
  22           osd:
  23             debug osd: 5
  24
  25     Requires 3 osds on a single test node.
  26     """
  27     if config is None:
  28         config = {}
  29     assert isinstance(config, dict), \
  30         'divergent_priors task only accepts a dict for configuration'
  31
  32     manager = ctx.managers['ceph']
  33
  34     while len(manager.get_osd_status()['up']) < 3:
  35         time.sleep(10)
  36     manager.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats')
  37     manager.raw_cluster_cmd('tell', 'osd.1', 'flush_pg_stats')
  38     manager.raw_cluster_cmd('tell', 'osd.2', 'flush_pg_stats')
  39     manager.raw_cluster_cmd('osd', 'set', 'noout')
  40     manager.raw_cluster_cmd('osd', 'set', 'noin')
  41     manager.raw_cluster_cmd('osd', 'set', 'nodown')
  42     manager.wait_for_clean()
  43
  44     # something that is always there
  45     dummyfile = '/etc/fstab'
  46     dummyfile2 = '/etc/resolv.conf'
  47
  48     # create 1 pg pool
  49     log.info('creating foo')
  50     manager.raw_cluster_cmd('osd', 'pool', 'create', 'foo', '1')
  51
  52     osds = [0, 1, 2]
  53     for i in osds:
  54         manager.set_config(i, osd_min_pg_log_entries=10)
  55         manager.set_config(i, osd_max_pg_log_entries=10)
  56         manager.set_config(i, osd_pg_log_trim_min=5)
  57
  58     # determine primary
  59     divergent = manager.get_pg_primary('foo', 0)
  60     log.info("primary and soon to be divergent is %d", divergent)
  61     non_divergent = list(osds)
  62     non_divergent.remove(divergent)
  63
  64     log.info('writing initial objects')
  65     first_mon = teuthology.get_first_mon(ctx, config)
  66     (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
  67     # write 100 objects
  68     for i in range(100):
  69         rados(ctx, mon, ['-p', 'foo', 'put', 'existing_%d' % i, dummyfile])
  70
  71     manager.wait_for_clean()
  72
  73     # blackhole non_divergent
  74     log.info("blackholing osds %s", str(non_divergent))
  75     for i in non_divergent:
  76         manager.set_config(i, objectstore_blackhole=1)
  77
  78     DIVERGENT_WRITE = 5
  79     DIVERGENT_REMOVE = 5
  80     # Write some soon to be divergent
  81     log.info('writing divergent objects')
  82     for i in range(DIVERGENT_WRITE):
  83         rados(ctx, mon, ['-p', 'foo', 'put', 'existing_%d' % i,
  84                          dummyfile2], wait=False)
  85     # Remove some soon to be divergent
  86     log.info('remove divergent objects')
  87     for i in range(DIVERGENT_REMOVE):
  88         rados(ctx, mon, ['-p', 'foo', 'rm',
  89                          'existing_%d' % (i + DIVERGENT_WRITE)], wait=False)
  90     time.sleep(10)
  91     mon.run(
  92         args=['killall', '-9', 'rados'],
  93         wait=True,
  94         check_status=False)
  95
  96     # kill all the osds but leave divergent in
  97     log.info('killing all the osds')
  98     for i in osds:
  99         manager.kill_osd(i)
 100     for i in osds:
 101         manager.mark_down_osd(i)
 102     for i in non_divergent:
 103         manager.mark_out_osd(i)
 104
 105     # bring up non-divergent
 106     log.info("bringing up non_divergent %s", str(non_divergent))
 107     for i in non_divergent:
 108         manager.revive_osd(i)
 109     for i in non_divergent:
 110         manager.mark_in_osd(i)
 111
 112     # write 1 non-divergent object (ensure that old divergent one is divergent)
 113     objname = "existing_%d" % (DIVERGENT_WRITE + DIVERGENT_REMOVE)
 114     log.info('writing non-divergent object ' + objname)
 115     rados(ctx, mon, ['-p', 'foo', 'put', objname, dummyfile2])
 116
 117     manager.wait_for_recovery()
 118
 119     # ensure no recovery of up osds first
 120     log.info('delay recovery')
 121     for i in non_divergent:
 122         manager.wait_run_admin_socket(
 123             'osd', i, ['set_recovery_delay', '100000'])
 124
 125     # bring in our divergent friend
 126     log.info("revive divergent %d", divergent)
 127     manager.raw_cluster_cmd('osd', 'set', 'noup')
 128     manager.revive_osd(divergent)
 129
 130     log.info('delay recovery divergent')
 131     manager.wait_run_admin_socket(
 132         'osd', divergent, ['set_recovery_delay', '100000'])
 133
 134     manager.raw_cluster_cmd('osd', 'unset', 'noup')
 135     while len(manager.get_osd_status()['up']) < 3:
 136         time.sleep(10)
 137
 138     log.info('wait for peering')
 139     rados(ctx, mon, ['-p', 'foo', 'put', 'foo', dummyfile])
 140
 141     # At this point the divergent_priors should have been detected
 142
 143     log.info("killing divergent %d", divergent)
 144     manager.kill_osd(divergent)
 145     log.info("reviving divergent %d", divergent)
 146     manager.revive_osd(divergent)
 147
 148     time.sleep(20)
 149
 150     log.info('allowing recovery')
 151     # Set osd_recovery_delay_start back to 0 and kick the queue
 152     for i in osds:
 153         manager.raw_cluster_cmd('tell', 'osd.%d' % i, 'debug',
 154                                     'kick_recovery_wq', ' 0')
 155
 156     log.info('reading divergent objects')
 157     for i in range(DIVERGENT_WRITE + DIVERGENT_REMOVE):
 158         exit_status = rados(ctx, mon, ['-p', 'foo', 'get', 'existing_%d' % i,
 159                                        '/tmp/existing'])
 160         assert exit_status is 0
 161
 162     log.info("success")