]>
Commit | Line | Data |
---|---|---|
1911f103 TL |
1 | #!/usr/bin/env bash |
2 | ||
3 | source $CEPH_ROOT/qa/standalone/ceph-helpers.sh | |
4 | ||
5 | ||
6 | function run() { | |
7 | local dir=$1 | |
8 | shift | |
9 | ||
10 | export CEPH_MON="127.0.0.1:7302" # git grep '\<7105\>' : there must be only one | |
11 | export CEPH_ARGS | |
12 | CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " | |
13 | CEPH_ARGS+="--mon-host=$CEPH_MON " | |
14 | ||
15 | local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} | |
16 | for func in $funcs ; do | |
17 | setup $dir || return 1 | |
18 | $func $dir || return 1 | |
19 | teardown $dir || return 1 | |
20 | done | |
21 | } | |
22 | ||
23 | ||
24 | function check_lec_equals_pools() { | |
25 | ||
26 | local pool_id=$1 | |
27 | ||
28 | report=$(ceph report) | |
29 | lec=$(echo $report | \ | |
30 | jq '.osdmap_clean_epochs.min_last_epoch_clean') | |
31 | ||
32 | if [[ -z "$pool_id" ]]; then | |
33 | pools=($(echo $report | \ | |
34 | jq \ | |
35 | ".osdmap_clean_epochs.last_epoch_clean.per_pool[] |" \ | |
36 | " select(.floor == $lec) | .poolid")) | |
37 | ||
38 | [[ ${#pools[*]} -eq 2 ]] || ( echo $report ; return 1 ) | |
39 | else | |
40 | floor=($(echo $report | \ | |
41 | jq \ | |
42 | ".osdmap_clean_epochs.last_epoch_clean.per_pool[] |" \ | |
43 | " select(.poolid == $pool_id) | .floor")) | |
44 | ||
45 | [[ $lec -eq $floor ]] || ( echo $report ; return 1 ) | |
46 | fi | |
47 | return 0 | |
48 | } | |
49 | ||
50 | function check_lec_lower_than_pool() { | |
51 | ||
52 | local pool_id=$1 | |
53 | [[ -z "$pool_id" ]] && ( echo "expected pool_id as parameter" ; exit 1 ) | |
54 | ||
55 | report=$(ceph report) | |
56 | lec=$(echo $report | \ | |
57 | jq '.osdmap_clean_epochs.min_last_epoch_clean') | |
58 | ||
59 | floor=($(echo $report | \ | |
60 | jq \ | |
61 | ".osdmap_clean_epochs.last_epoch_clean.per_pool[] |" \ | |
62 | " select(.poolid == $pool_id) | .floor")) | |
63 | ||
64 | [[ $lec -lt $floor ]] || ( echo $report ; return 1 ) | |
65 | return 0 | |
66 | } | |
67 | ||
68 | function check_floor_pool_greater_than_pool() { | |
69 | ||
70 | local pool_a=$1 | |
71 | local pool_b=$1 | |
72 | [[ -z "$pool_a" ]] && ( echo "expected id as first parameter" ; exit 1 ) | |
73 | [[ -z "$pool_b" ]] && ( echo "expected id as second parameter" ; exit 1 ) | |
74 | ||
75 | report=$(ceph report) | |
76 | ||
77 | floor_a=($(echo $report | \ | |
78 | jq \ | |
79 | ".osdmap_clean_epochs.last_epoch_clean.per_pool[] |" \ | |
80 | " select(.poolid == $pool_a) | .floor")) | |
81 | ||
82 | floor_b=($(echo $report | \ | |
83 | jq \ | |
84 | ".osdmap_clean_epochs.last_epoch_clean.per_pool[] |" \ | |
85 | " select(.poolid == $pool_b) | .floor")) | |
86 | ||
87 | [[ $floor_a -gt $floor_b ]] || ( echo $report ; return 1 ) | |
88 | return 0 | |
89 | } | |
90 | ||
91 | function check_lec_honours_osd() { | |
92 | ||
93 | local osd=$1 | |
94 | ||
95 | report=$(ceph report) | |
96 | lec=$(echo $report | \ | |
97 | jq '.osdmap_clean_epochs.min_last_epoch_clean') | |
98 | ||
99 | if [[ -z "$osd" ]]; then | |
100 | osds=($(echo $report | \ | |
101 | jq \ | |
102 | ".osdmap_clean_epochs.osd_epochs[] |" \ | |
103 | " select(.epoch >= $lec) | .id")) | |
104 | ||
105 | [[ ${#osds[*]} -eq 3 ]] || ( echo $report ; return 1 ) | |
106 | else | |
107 | epoch=($(echo $report | \ | |
108 | jq \ | |
109 | ".osdmap_clean_epochs.osd_epochs[] |" \ | |
110 | " select(.id == $id) | .epoch")) | |
111 | [[ ${#epoch[*]} -eq 1 ]] || ( echo $report ; return 1 ) | |
112 | [[ ${epoch[0]} -ge $lec ]] || ( echo $report ; return 1 ) | |
113 | fi | |
114 | ||
115 | return 0 | |
116 | } | |
117 | ||
118 | function validate_fc() { | |
119 | report=$(ceph report) | |
120 | lec=$(echo $report | \ | |
121 | jq '.osdmap_clean_epochs.min_last_epoch_clean') | |
122 | osdm_fc=$(echo $report | \ | |
123 | jq '.osdmap_first_committed') | |
124 | ||
125 | [[ $lec -eq $osdm_fc ]] || ( echo $report ; return 1 ) | |
126 | return 0 | |
127 | } | |
128 | ||
129 | function get_fc_lc_diff() { | |
130 | report=$(ceph report) | |
131 | osdm_fc=$(echo $report | \ | |
132 | jq '.osdmap_first_committed') | |
133 | osdm_lc=$(echo $report | \ | |
134 | jq '.osdmap_last_committed') | |
135 | ||
136 | echo $((osdm_lc - osdm_fc)) | |
137 | } | |
138 | ||
139 | function get_pool_id() { | |
140 | ||
141 | local pn=$1 | |
142 | [[ -z "$pn" ]] && ( echo "expected pool name as argument" ; exit 1 ) | |
143 | ||
144 | report=$(ceph report) | |
145 | pool_id=$(echo $report | \ | |
146 | jq ".osdmap.pools[] | select(.pool_name == \"$pn\") | .pool") | |
147 | ||
148 | [[ $pool_id -ge 0 ]] || \ | |
149 | ( echo "unexpected pool id for pool \'$pn\': $pool_id" ; return -1 ) | |
150 | ||
151 | echo $pool_id | |
152 | return 0 | |
153 | } | |
154 | ||
155 | function wait_for_total_num_maps() { | |
156 | # rip wait_for_health, becaue it's easier than deduplicating the code | |
157 | local -a delays=($(get_timeout_delays $TIMEOUT .1)) | |
158 | local -i loop=0 | |
159 | local -i v_diff=$1 | |
160 | ||
161 | while [[ $(get_fc_lc_diff) -gt $v_diff ]]; do | |
162 | if (( $loop >= ${#delays[*]} )) ; then | |
163 | echo "maps were not trimmed" | |
164 | return 1 | |
165 | fi | |
166 | sleep ${delays[$loop]} | |
167 | loop+=1 | |
168 | done | |
169 | } | |
170 | ||
171 | function TEST_mon_last_clean_epoch() { | |
172 | ||
173 | local dir=$1 | |
174 | ||
175 | run_mon $dir a || return 1 | |
176 | run_mgr $dir x || return 1 | |
177 | run_osd $dir 0 || return 1 | |
178 | run_osd $dir 1 || return 1 | |
179 | run_osd $dir 2 || return 1 | |
180 | osd_pid=$(cat $dir/osd.2.pid) | |
181 | ||
182 | sleep 5 | |
183 | ||
f6b5b4d7 TL |
184 | ceph tell 'osd.*' injectargs '--osd-beacon-report-interval 10' || exit 1 |
185 | ceph tell 'mon.*' injectargs \ | |
1911f103 TL |
186 | '--mon-min-osdmap-epochs 2 --paxos-service-trim-min 1' || exit 1 |
187 | ||
188 | create_pool foo 32 | |
189 | create_pool bar 32 | |
190 | ||
191 | foo_id=$(get_pool_id "foo") | |
192 | bar_id=$(get_pool_id "bar") | |
193 | ||
194 | [[ $foo_id -lt 0 ]] && ( echo "couldn't find pool 'foo' id" ; exit 1 ) | |
195 | [[ $bar_id -lt 0 ]] && ( echo "couldn't find pool 'bar' id" ; exit 1 ) | |
196 | ||
197 | # no real clue why we are getting these warnings, but let's make them go | |
198 | # away so we can be happy. | |
199 | ||
200 | ceph osd set-full-ratio 0.97 | |
201 | ceph osd set-backfillfull-ratio 0.97 | |
202 | ||
203 | wait_for_health_ok || exit 1 | |
204 | ||
205 | pre_map_diff=$(get_fc_lc_diff) | |
206 | wait_for_total_num_maps 2 | |
207 | post_map_diff=$(get_fc_lc_diff) | |
208 | ||
209 | [[ $post_map_diff -le $pre_map_diff ]] || exit 1 | |
210 | ||
211 | pre_map_diff=$post_map_diff | |
212 | ||
213 | ceph osd pool set foo size 3 | |
214 | ceph osd pool set bar size 3 | |
215 | ||
216 | wait_for_health_ok || exit 1 | |
217 | ||
218 | check_lec_equals_pools || exit 1 | |
219 | check_lec_honours_osd || exit 1 | |
220 | validate_fc || exit 1 | |
221 | ||
222 | # down osd.2; expected result (because all pools' size equals 3): | |
223 | # - number of committed maps increase over 2 | |
224 | # - lec equals fc | |
225 | # - lec equals osd.2's epoch | |
226 | # - all pools have floor equal to lec | |
227 | ||
228 | while kill $osd_pid ; do sleep 1 ; done | |
f91f0fd5 | 229 | ceph osd out 2 |
1911f103 TL |
230 | sleep 5 # seriously, just to make sure things settle; we may not need this. |
231 | ||
232 | # generate some maps | |
233 | for ((i=0; i <= 10; ++i)); do | |
234 | ceph osd set noup | |
235 | sleep 1 | |
236 | ceph osd unset noup | |
237 | sleep 1 | |
238 | done | |
239 | ||
240 | post_map_diff=$(get_fc_lc_diff) | |
241 | [[ $post_map_diff -gt 2 ]] || exit 1 | |
242 | ||
243 | validate_fc || exit 1 | |
244 | check_lec_equals_pools || exit 1 | |
245 | check_lec_honours_osd 2 || exit 1 | |
246 | ||
247 | # adjust pool 'bar' size to 2; expect: | |
248 | # - number of committed maps still over 2 | |
249 | # - lec equals fc | |
250 | # - lec equals pool 'foo' floor | |
251 | # - pool 'bar' floor greater than pool 'foo' | |
252 | ||
253 | ceph osd pool set bar size 2 | |
254 | ||
255 | diff_ver=$(get_fc_lc_diff) | |
256 | [[ $diff_ver -gt 2 ]] || exit 1 | |
257 | ||
258 | validate_fc || exit 1 | |
259 | ||
260 | check_lec_equals_pools $foo_id || exit 1 | |
261 | check_lec_lower_than_pool $bar_id || exit 1 | |
262 | ||
263 | check_floor_pool_greater_than_pool $bar_id $foo_id || exit 1 | |
264 | ||
265 | # set pool 'foo' size to 2; expect: | |
266 | # - health_ok | |
267 | # - lec equals pools | |
268 | # - number of committed maps decreases | |
269 | # - lec equals fc | |
270 | ||
271 | pre_map_diff=$(get_fc_lc_diff) | |
272 | ||
273 | ceph osd pool set foo size 2 || exit 1 | |
274 | wait_for_clean || exit 1 | |
275 | ||
276 | check_lec_equals_pools || exit 1 | |
277 | validate_fc || exit 1 | |
278 | ||
279 | if ! wait_for_total_num_maps 2 ; then | |
280 | post_map_diff=$(get_fc_lc_diff) | |
281 | # number of maps is decreasing though, right? | |
282 | [[ $post_map_diff -lt $pre_map_diff ]] || exit 1 | |
283 | fi | |
284 | ||
285 | # bring back osd.2; expect: | |
286 | # - health_ok | |
287 | # - lec equals fc | |
288 | # - number of committed maps equals 2 | |
289 | # - all pools have floor equal to lec | |
290 | ||
291 | pre_map_diff=$(get_fc_lc_diff) | |
292 | ||
293 | activate_osd $dir 2 || exit 1 | |
294 | wait_for_health_ok || exit 1 | |
295 | validate_fc || exit 1 | |
296 | check_lec_equals_pools || exit 1 | |
297 | ||
298 | if ! wait_for_total_num_maps 2 ; then | |
299 | post_map_diff=$(get_fc_lc_diff) | |
300 | # number of maps is decreasing though, right? | |
301 | [[ $post_map_diff -lt $pre_map_diff ]] || exit 1 | |
302 | fi | |
303 | ||
304 | return 0 | |
305 | } | |
306 | ||
307 | main mon-last-clean-epoch "$@" |