]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/divergent_priors.py
2 Special case divergence test
7 from teuthology
import misc
as teuthology
8 from util
.rados
import rados
11 log
= logging
.getLogger(__name__
)
14 def task(ctx
, config
):
16 Test handling of divergent entries with prior_version
25 Requires 3 osds on a single test node.
29 assert isinstance(config
, dict), \
30 'divergent_priors task only accepts a dict for configuration'
32 manager
= ctx
.managers
['ceph']
34 while len(manager
.get_osd_status()['up']) < 3:
36 manager
.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats')
37 manager
.raw_cluster_cmd('tell', 'osd.1', 'flush_pg_stats')
38 manager
.raw_cluster_cmd('tell', 'osd.2', 'flush_pg_stats')
39 manager
.raw_cluster_cmd('osd', 'set', 'noout')
40 manager
.raw_cluster_cmd('osd', 'set', 'noin')
41 manager
.raw_cluster_cmd('osd', 'set', 'nodown')
42 manager
.wait_for_clean()
44 # something that is always there
45 dummyfile
= '/etc/fstab'
46 dummyfile2
= '/etc/resolv.conf'
49 log
.info('creating foo')
50 manager
.raw_cluster_cmd('osd', 'pool', 'create', 'foo', '1')
54 manager
.set_config(i
, osd_min_pg_log_entries
=10)
55 manager
.set_config(i
, osd_max_pg_log_entries
=10)
56 manager
.set_config(i
, osd_pg_log_trim_min
=5)
59 divergent
= manager
.get_pg_primary('foo', 0)
60 log
.info("primary and soon to be divergent is %d", divergent
)
61 non_divergent
= list(osds
)
62 non_divergent
.remove(divergent
)
64 log
.info('writing initial objects')
65 first_mon
= teuthology
.get_first_mon(ctx
, config
)
66 (mon
,) = ctx
.cluster
.only(first_mon
).remotes
.iterkeys()
69 rados(ctx
, mon
, ['-p', 'foo', 'put', 'existing_%d' % i
, dummyfile
])
71 manager
.wait_for_clean()
73 # blackhole non_divergent
74 log
.info("blackholing osds %s", str(non_divergent
))
75 for i
in non_divergent
:
76 manager
.set_config(i
, objectstore_blackhole
=1)
80 # Write some soon to be divergent
81 log
.info('writing divergent objects')
82 for i
in range(DIVERGENT_WRITE
):
83 rados(ctx
, mon
, ['-p', 'foo', 'put', 'existing_%d' % i
,
84 dummyfile2
], wait
=False)
85 # Remove some soon to be divergent
86 log
.info('remove divergent objects')
87 for i
in range(DIVERGENT_REMOVE
):
88 rados(ctx
, mon
, ['-p', 'foo', 'rm',
89 'existing_%d' % (i
+ DIVERGENT_WRITE
)], wait
=False)
92 args
=['killall', '-9', 'rados'],
96 # kill all the osds but leave divergent in
97 log
.info('killing all the osds')
101 manager
.mark_down_osd(i
)
102 for i
in non_divergent
:
103 manager
.mark_out_osd(i
)
105 # bring up non-divergent
106 log
.info("bringing up non_divergent %s", str(non_divergent
))
107 for i
in non_divergent
:
108 manager
.revive_osd(i
)
109 for i
in non_divergent
:
110 manager
.mark_in_osd(i
)
112 # write 1 non-divergent object (ensure that old divergent one is divergent)
113 objname
= "existing_%d" % (DIVERGENT_WRITE
+ DIVERGENT_REMOVE
)
114 log
.info('writing non-divergent object ' + objname
)
115 rados(ctx
, mon
, ['-p', 'foo', 'put', objname
, dummyfile2
])
117 manager
.wait_for_recovery()
119 # ensure no recovery of up osds first
120 log
.info('delay recovery')
121 for i
in non_divergent
:
122 manager
.wait_run_admin_socket(
123 'osd', i
, ['set_recovery_delay', '100000'])
125 # bring in our divergent friend
126 log
.info("revive divergent %d", divergent
)
127 manager
.raw_cluster_cmd('osd', 'set', 'noup')
128 manager
.revive_osd(divergent
)
130 log
.info('delay recovery divergent')
131 manager
.wait_run_admin_socket(
132 'osd', divergent
, ['set_recovery_delay', '100000'])
134 manager
.raw_cluster_cmd('osd', 'unset', 'noup')
135 while len(manager
.get_osd_status()['up']) < 3:
138 log
.info('wait for peering')
139 rados(ctx
, mon
, ['-p', 'foo', 'put', 'foo', dummyfile
])
141 # At this point the divergent_priors should have been detected
143 log
.info("killing divergent %d", divergent
)
144 manager
.kill_osd(divergent
)
145 log
.info("reviving divergent %d", divergent
)
146 manager
.revive_osd(divergent
)
150 log
.info('allowing recovery')
151 # Set osd_recovery_delay_start back to 0 and kick the queue
153 manager
.raw_cluster_cmd('tell', 'osd.%d' % i
, 'debug',
154 'kick_recovery_wq', ' 0')
156 log
.info('reading divergent objects')
157 for i
in range(DIVERGENT_WRITE
+ DIVERGENT_REMOVE
):
158 exit_status
= rados(ctx
, mon
, ['-p', 'foo', 'get', 'existing_%d' % i
,
160 assert exit_status
is 0