]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | #!/bin/bash -x |
2 | # | |
3 | # Copyright (C) 2014 Red Hat <contact@redhat.com> | |
4 | # | |
5 | # Author: Loic Dachary <loic@dachary.org> | |
6 | # | |
7 | # This program is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU Library Public License as published by | |
9 | # the Free Software Foundation; either version 2, or (at your option) | |
10 | # any later version. | |
11 | # | |
12 | # This program is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU Library Public License for more details. | |
16 | # | |
c07f9fc5 | 17 | source $CEPH_ROOT/qa/standalone/ceph-helpers.sh |
7c673cae | 18 | |
224ce89b | 19 | if [ `uname` = FreeBSD ]; then |
c07f9fc5 | 20 | # erasure coding overwrites are only tested on Bluestore |
224ce89b WB |
21 | # erasure coding on filestore is unsafe |
22 | # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites | |
23 | use_ec_overwrite=false | |
24 | else | |
25 | use_ec_overwrite=true | |
26 | fi | |
27 | ||
7c673cae FG |
28 | # Test development and debugging |
29 | # Set to "yes" in order to ignore diff errors and save results to update test | |
30 | getjson="no" | |
31 | ||
7c673cae | 32 | # Ignore the epoch and filter out the attr '_' value because it has date information and won't match |
b5b8bbf5 FG |
33 | if [ "$(jq --version 2>&1 | awk '{ print $3}')" = "1.3" ]; then # Not sure all versions that apply here |
34 | jqfilter='.inconsistents | (.[].shards[].attrs[] | select(.name == "_") | .value) |= "----Stripped-by-test----"' | |
35 | else | |
36 | jqfilter='.inconsistents | (.[].shards[].attrs[]? | select(.name == "_") | .value) |= "----Stripped-by-test----"' | |
37 | fi | |
7c673cae FG |
38 | sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' |
39 | ||
40 | # Remove items are not consistent across runs, the pg interval and client | |
41 | sedfilter='s/\([ ]*\"\(selected_\)*object_info\":.*head[(]\)[^[:space:]]* [^[:space:]]* \(.*\)/\1\3/' | |
42 | ||
43 | function run() { | |
44 | local dir=$1 | |
45 | shift | |
46 | ||
47 | export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one | |
48 | export CEPH_ARGS | |
49 | CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " | |
50 | CEPH_ARGS+="--mon-host=$CEPH_MON " | |
51 | ||
52 | local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} | |
53 | for func in $funcs ; do | |
54 | $func $dir || return 1 | |
55 | done | |
56 | } | |
57 | ||
58 | function add_something() { | |
59 | local dir=$1 | |
60 | local poolname=$2 | |
61 | local obj=${3:-SOMETHING} | |
62 | local scrub=${4:-noscrub} | |
63 | ||
64 | if [ "$scrub" = "noscrub" ]; | |
65 | then | |
66 | ceph osd set noscrub || return 1 | |
67 | ceph osd set nodeep-scrub || return 1 | |
68 | else | |
69 | ceph osd unset noscrub || return 1 | |
70 | ceph osd unset nodeep-scrub || return 1 | |
71 | fi | |
72 | ||
73 | local payload=ABCDEF | |
74 | echo $payload > $dir/ORIGINAL | |
75 | rados --pool $poolname put $obj $dir/ORIGINAL || return 1 | |
76 | } | |
77 | ||
78 | # | |
79 | # Corrupt one copy of a replicated pool | |
80 | # | |
81 | function TEST_corrupt_and_repair_replicated() { | |
82 | local dir=$1 | |
83 | local poolname=rbd | |
84 | ||
85 | setup $dir || return 1 | |
86 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
87 | run_mgr $dir x || return 1 | |
88 | run_osd $dir 0 || return 1 | |
89 | run_osd $dir 1 || return 1 | |
c07f9fc5 | 90 | create_rbd_pool || return 1 |
7c673cae FG |
91 | wait_for_clean || return 1 |
92 | ||
93 | add_something $dir $poolname || return 1 | |
94 | corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1 | |
95 | # Reproduces http://tracker.ceph.com/issues/8914 | |
96 | corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1 | |
97 | ||
98 | teardown $dir || return 1 | |
99 | } | |
100 | ||
101 | function corrupt_and_repair_two() { | |
102 | local dir=$1 | |
103 | local poolname=$2 | |
104 | local first=$3 | |
105 | local second=$4 | |
106 | ||
107 | # | |
108 | # 1) remove the corresponding file from the OSDs | |
109 | # | |
110 | pids="" | |
111 | run_in_background pids objectstore_tool $dir $first SOMETHING remove | |
112 | run_in_background pids objectstore_tool $dir $second SOMETHING remove | |
113 | wait_background pids | |
114 | return_code=$? | |
115 | if [ $return_code -ne 0 ]; then return $return_code; fi | |
116 | ||
117 | # | |
118 | # 2) repair the PG | |
119 | # | |
120 | local pg=$(get_pg $poolname SOMETHING) | |
121 | repair $pg | |
122 | # | |
123 | # 3) The files must be back | |
124 | # | |
125 | pids="" | |
126 | run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs | |
127 | run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs | |
128 | wait_background pids | |
129 | return_code=$? | |
130 | if [ $return_code -ne 0 ]; then return $return_code; fi | |
131 | ||
132 | rados --pool $poolname get SOMETHING $dir/COPY || return 1 | |
133 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
134 | } | |
135 | ||
136 | # | |
137 | # 1) add an object | |
138 | # 2) remove the corresponding file from a designated OSD | |
139 | # 3) repair the PG | |
140 | # 4) check that the file has been restored in the designated OSD | |
141 | # | |
142 | function corrupt_and_repair_one() { | |
143 | local dir=$1 | |
144 | local poolname=$2 | |
145 | local osd=$3 | |
146 | ||
147 | # | |
148 | # 1) remove the corresponding file from the OSD | |
149 | # | |
150 | objectstore_tool $dir $osd SOMETHING remove || return 1 | |
151 | # | |
152 | # 2) repair the PG | |
153 | # | |
154 | local pg=$(get_pg $poolname SOMETHING) | |
155 | repair $pg | |
156 | # | |
157 | # 3) The file must be back | |
158 | # | |
159 | objectstore_tool $dir $osd SOMETHING list-attrs || return 1 | |
160 | rados --pool $poolname get SOMETHING $dir/COPY || return 1 | |
161 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
162 | } | |
163 | ||
164 | function corrupt_and_repair_erasure_coded() { | |
165 | local dir=$1 | |
166 | local poolname=$2 | |
167 | ||
168 | add_something $dir $poolname || return 1 | |
169 | ||
170 | local primary=$(get_primary $poolname SOMETHING) | |
171 | local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//")) | |
172 | local not_primary_first=${osds[0]} | |
173 | local not_primary_second=${osds[1]} | |
174 | ||
175 | # Reproduces http://tracker.ceph.com/issues/10017 | |
176 | corrupt_and_repair_one $dir $poolname $primary || return 1 | |
177 | # Reproduces http://tracker.ceph.com/issues/10409 | |
178 | corrupt_and_repair_one $dir $poolname $not_primary_first || return 1 | |
179 | corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1 | |
180 | corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1 | |
181 | ||
182 | } | |
183 | ||
184 | function create_ec_pool() { | |
185 | local pool_name=$1 | |
186 | local allow_overwrites=$2 | |
187 | ||
224ce89b | 188 | ceph osd erasure-code-profile set myprofile crush-failure-domain=osd $3 $4 $5 $6 $7 || return 1 |
7c673cae | 189 | |
b5b8bbf5 | 190 | create_pool "$poolname" 1 1 erasure myprofile || return 1 |
7c673cae FG |
191 | |
192 | if [ "$allow_overwrites" = "true" ]; then | |
193 | ceph osd pool set "$poolname" allow_ec_overwrites true || return 1 | |
194 | fi | |
195 | ||
196 | wait_for_clean || return 1 | |
197 | return 0 | |
198 | } | |
199 | ||
200 | function auto_repair_erasure_coded() { | |
201 | local dir=$1 | |
202 | local allow_overwrites=$2 | |
203 | local poolname=ecpool | |
204 | ||
205 | # Launch a cluster with 5 seconds scrub interval | |
206 | setup $dir || return 1 | |
207 | run_mon $dir a || return 1 | |
208 | run_mgr $dir x || return 1 | |
209 | local ceph_osd_args="--osd-scrub-auto-repair=true \ | |
210 | --osd-deep-scrub-interval=5 \ | |
211 | --osd-scrub-max-interval=5 \ | |
212 | --osd-scrub-min-interval=5 \ | |
213 | --osd-scrub-interval-randomize-ratio=0" | |
214 | for id in $(seq 0 2) ; do | |
215 | if [ "$allow_overwrites" = "true" ]; then | |
216 | run_osd_bluestore $dir $id $ceph_osd_args || return 1 | |
217 | else | |
218 | run_osd $dir $id $ceph_osd_args || return 1 | |
219 | fi | |
220 | done | |
c07f9fc5 | 221 | create_rbd_pool || return 1 |
7c673cae FG |
222 | wait_for_clean || return 1 |
223 | ||
224 | # Create an EC pool | |
225 | create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1 | |
226 | ||
227 | # Put an object | |
228 | local payload=ABCDEF | |
229 | echo $payload > $dir/ORIGINAL | |
230 | rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1 | |
231 | ||
232 | # Remove the object from one shard physically | |
233 | # Restarted osd get $ceph_osd_args passed | |
234 | objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1 | |
235 | # Wait for auto repair | |
236 | local pgid=$(get_pg $poolname SOMETHING) | |
237 | wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)" | |
238 | wait_for_clean || return 1 | |
239 | # Verify - the file should be back | |
240 | # Restarted osd get $ceph_osd_args passed | |
241 | objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1 | |
242 | rados --pool $poolname get SOMETHING $dir/COPY || return 1 | |
243 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
244 | ||
245 | # Tear down | |
246 | teardown $dir || return 1 | |
247 | } | |
248 | ||
249 | function TEST_auto_repair_erasure_coded_appends() { | |
250 | auto_repair_erasure_coded $1 false | |
251 | } | |
252 | ||
253 | function TEST_auto_repair_erasure_coded_overwrites() { | |
224ce89b WB |
254 | if [ "$use_ec_overwrite" = "true" ]; then |
255 | auto_repair_erasure_coded $1 true | |
256 | fi | |
7c673cae FG |
257 | } |
258 | ||
259 | function corrupt_and_repair_jerasure() { | |
260 | local dir=$1 | |
261 | local allow_overwrites=$2 | |
262 | local poolname=ecpool | |
263 | ||
264 | setup $dir || return 1 | |
265 | run_mon $dir a || return 1 | |
266 | run_mgr $dir x || return 1 | |
267 | for id in $(seq 0 3) ; do | |
268 | if [ "$allow_overwrites" = "true" ]; then | |
269 | run_osd_bluestore $dir $id || return 1 | |
270 | else | |
271 | run_osd $dir $id || return 1 | |
272 | fi | |
273 | done | |
c07f9fc5 | 274 | create_rbd_pool || return 1 |
7c673cae FG |
275 | wait_for_clean || return 1 |
276 | ||
277 | create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 | |
278 | corrupt_and_repair_erasure_coded $dir $poolname || return 1 | |
279 | ||
280 | teardown $dir || return 1 | |
281 | } | |
282 | ||
283 | function TEST_corrupt_and_repair_jerasure_appends() { | |
284 | corrupt_and_repair_jerasure $1 | |
285 | } | |
286 | ||
287 | function TEST_corrupt_and_repair_jerasure_overwrites() { | |
224ce89b WB |
288 | if [ "$use_ec_overwrite" = "true" ]; then |
289 | corrupt_and_repair_jerasure $1 true | |
290 | fi | |
7c673cae FG |
291 | } |
292 | ||
293 | function corrupt_and_repair_lrc() { | |
294 | local dir=$1 | |
295 | local allow_overwrites=$2 | |
296 | local poolname=ecpool | |
297 | ||
298 | setup $dir || return 1 | |
299 | run_mon $dir a || return 1 | |
300 | run_mgr $dir x || return 1 | |
301 | for id in $(seq 0 9) ; do | |
302 | if [ "$allow_overwrites" = "true" ]; then | |
303 | run_osd_bluestore $dir $id || return 1 | |
304 | else | |
305 | run_osd $dir $id || return 1 | |
306 | fi | |
307 | done | |
c07f9fc5 | 308 | create_rbd_pool || return 1 |
7c673cae FG |
309 | wait_for_clean || return 1 |
310 | ||
311 | create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1 | |
312 | corrupt_and_repair_erasure_coded $dir $poolname || return 1 | |
313 | ||
314 | teardown $dir || return 1 | |
315 | } | |
316 | ||
317 | function TEST_corrupt_and_repair_lrc_appends() { | |
318 | corrupt_and_repair_jerasure $1 | |
319 | } | |
320 | ||
321 | function TEST_corrupt_and_repair_lrc_overwrites() { | |
224ce89b WB |
322 | if [ "$use_ec_overwrite" = "true" ]; then |
323 | corrupt_and_repair_jerasure $1 true | |
324 | fi | |
7c673cae FG |
325 | } |
326 | ||
327 | function unfound_erasure_coded() { | |
328 | local dir=$1 | |
329 | local allow_overwrites=$2 | |
330 | local poolname=ecpool | |
331 | local payload=ABCDEF | |
332 | ||
333 | setup $dir || return 1 | |
334 | run_mon $dir a || return 1 | |
335 | run_mgr $dir x || return 1 | |
336 | for id in $(seq 0 3) ; do | |
337 | if [ "$allow_overwrites" = "true" ]; then | |
338 | run_osd_bluestore $dir $id || return 1 | |
339 | else | |
340 | run_osd $dir $id || return 1 | |
341 | fi | |
342 | done | |
c07f9fc5 | 343 | create_rbd_pool || return 1 |
7c673cae FG |
344 | wait_for_clean || return 1 |
345 | ||
346 | create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 | |
347 | ||
348 | add_something $dir $poolname || return 1 | |
349 | ||
350 | local primary=$(get_primary $poolname SOMETHING) | |
351 | local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//")) | |
352 | local not_primary_first=${osds[0]} | |
353 | local not_primary_second=${osds[1]} | |
354 | local not_primary_third=${osds[2]} | |
355 | ||
356 | # | |
357 | # 1) remove the corresponding file from the OSDs | |
358 | # | |
359 | pids="" | |
360 | run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove | |
361 | run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove | |
362 | run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove | |
363 | wait_background pids | |
364 | return_code=$? | |
365 | if [ $return_code -ne 0 ]; then return $return_code; fi | |
366 | ||
367 | # | |
368 | # 2) repair the PG | |
369 | # | |
370 | local pg=$(get_pg $poolname SOMETHING) | |
371 | repair $pg | |
372 | # | |
373 | # 3) check pg state | |
374 | # | |
375 | # it may take a bit to appear due to mon/mgr asynchrony | |
376 | for f in `seq 1 60`; do | |
b5b8bbf5 | 377 | ceph -s | grep "1/1 objects unfound" && break |
7c673cae FG |
378 | sleep 1 |
379 | done | |
380 | ceph -s|grep "4 osds: 4 up, 4 in" || return 1 | |
b5b8bbf5 | 381 | ceph -s|grep "1/1 objects unfound" || return 1 |
7c673cae FG |
382 | |
383 | teardown $dir || return 1 | |
384 | } | |
385 | ||
386 | function TEST_unfound_erasure_coded_appends() { | |
387 | unfound_erasure_coded $1 | |
388 | } | |
389 | ||
390 | function TEST_unfound_erasure_coded_overwrites() { | |
224ce89b WB |
391 | if [ "$use_ec_overwrite" = "true" ]; then |
392 | unfound_erasure_coded $1 true | |
393 | fi | |
7c673cae FG |
394 | } |
395 | ||
396 | # | |
397 | # list_missing for EC pool | |
398 | # | |
399 | function list_missing_erasure_coded() { | |
400 | local dir=$1 | |
401 | local allow_overwrites=$2 | |
402 | local poolname=ecpool | |
403 | ||
404 | setup $dir || return 1 | |
405 | run_mon $dir a || return 1 | |
406 | run_mgr $dir x || return 1 | |
407 | for id in $(seq 0 2) ; do | |
408 | if [ "$allow_overwrites" = "true" ]; then | |
409 | run_osd_bluestore $dir $id || return 1 | |
410 | else | |
411 | run_osd $dir $id || return 1 | |
412 | fi | |
413 | done | |
c07f9fc5 | 414 | create_rbd_pool || return 1 |
7c673cae FG |
415 | wait_for_clean || return 1 |
416 | ||
417 | create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1 | |
418 | ||
419 | # Put an object and remove the two shards (including primary) | |
420 | add_something $dir $poolname MOBJ0 || return 1 | |
421 | local -a osds0=($(get_osds $poolname MOBJ0)) | |
422 | ||
423 | # Put another object and remove two shards (excluding primary) | |
424 | add_something $dir $poolname MOBJ1 || return 1 | |
425 | local -a osds1=($(get_osds $poolname MOBJ1)) | |
426 | ||
427 | # Stop all osd daemons | |
428 | for id in $(seq 0 2) ; do | |
429 | kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1 | |
430 | done | |
431 | ||
432 | id=${osds0[0]} | |
31f18b77 | 433 | ceph-objectstore-tool --data-path $dir/$id \ |
7c673cae FG |
434 | MOBJ0 remove || return 1 |
435 | id=${osds0[1]} | |
31f18b77 | 436 | ceph-objectstore-tool --data-path $dir/$id \ |
7c673cae FG |
437 | MOBJ0 remove || return 1 |
438 | ||
439 | id=${osds1[1]} | |
31f18b77 | 440 | ceph-objectstore-tool --data-path $dir/$id \ |
7c673cae FG |
441 | MOBJ1 remove || return 1 |
442 | id=${osds1[2]} | |
31f18b77 | 443 | ceph-objectstore-tool --data-path $dir/$id \ |
7c673cae FG |
444 | MOBJ1 remove || return 1 |
445 | ||
446 | for id in $(seq 0 2) ; do | |
447 | activate_osd $dir $id >&2 || return 1 | |
448 | done | |
c07f9fc5 | 449 | create_rbd_pool || return 1 |
7c673cae FG |
450 | wait_for_clean || return 1 |
451 | ||
452 | # Get get - both objects should in the same PG | |
453 | local pg=$(get_pg $poolname MOBJ0) | |
454 | ||
455 | # Repair the PG, which triggers the recovering, | |
456 | # and should mark the object as unfound | |
457 | repair $pg | |
c07f9fc5 | 458 | |
7c673cae FG |
459 | for i in $(seq 0 120) ; do |
460 | [ $i -lt 60 ] || return 1 | |
461 | matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l) | |
462 | [ $matches -eq 2 ] && break | |
463 | done | |
464 | ||
465 | teardown $dir || return 1 | |
466 | } | |
467 | ||
468 | function TEST_list_missing_erasure_coded_appends() { | |
469 | list_missing_erasure_coded $1 false | |
470 | } | |
471 | ||
472 | function TEST_list_missing_erasure_coded_overwrites() { | |
224ce89b WB |
473 | if [ "$use_ec_overwrite" = "true" ]; then |
474 | list_missing_erasure_coded $1 true | |
475 | fi | |
7c673cae FG |
476 | } |
477 | ||
478 | # | |
479 | # Corrupt one copy of a replicated pool | |
480 | # | |
481 | function TEST_corrupt_scrub_replicated() { | |
482 | local dir=$1 | |
483 | local poolname=csr_pool | |
b5b8bbf5 | 484 | local total_objs=16 |
7c673cae FG |
485 | |
486 | setup $dir || return 1 | |
487 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
488 | run_mgr $dir x || return 1 | |
489 | run_osd $dir 0 || return 1 | |
490 | run_osd $dir 1 || return 1 | |
c07f9fc5 | 491 | create_rbd_pool || return 1 |
7c673cae FG |
492 | wait_for_clean || return 1 |
493 | ||
b5b8bbf5 FG |
494 | create_pool foo 1 || return 1 |
495 | create_pool $poolname 1 1 || return 1 | |
7c673cae FG |
496 | wait_for_clean || return 1 |
497 | ||
498 | for i in $(seq 1 $total_objs) ; do | |
499 | objname=ROBJ${i} | |
500 | add_something $dir $poolname $objname || return 1 | |
501 | ||
502 | rados --pool $poolname setomapheader $objname hdr-$objname || return 1 | |
503 | rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1 | |
31f18b77 FG |
504 | done |
505 | ||
506 | local pg=$(get_pg $poolname ROBJ0) | |
507 | ||
508 | # Compute an old omap digest and save oi | |
c07f9fc5 | 509 | CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \ |
31f18b77 | 510 | config set osd_deep_scrub_update_digest_min_age 0 |
c07f9fc5 | 511 | CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \ |
31f18b77 FG |
512 | config set osd_deep_scrub_update_digest_min_age 0 |
513 | pg_deep_scrub $pg | |
514 | ||
515 | for i in $(seq 1 $total_objs) ; do | |
516 | objname=ROBJ${i} | |
7c673cae FG |
517 | |
518 | # Alternate corruption between osd.0 and osd.1 | |
519 | local osd=$(expr $i % 2) | |
520 | ||
521 | case $i in | |
522 | 1) | |
523 | # Size (deep scrub data_digest too) | |
524 | local payload=UVWXYZZZ | |
525 | echo $payload > $dir/CORRUPT | |
526 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
527 | ;; | |
528 | ||
529 | 2) | |
530 | # digest (deep scrub only) | |
531 | local payload=UVWXYZ | |
532 | echo $payload > $dir/CORRUPT | |
533 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
534 | ;; | |
535 | ||
536 | 3) | |
537 | # missing | |
538 | objectstore_tool $dir $osd $objname remove || return 1 | |
539 | ;; | |
540 | ||
541 | 4) | |
542 | # Modify omap value (deep scrub only) | |
543 | objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1 | |
544 | ;; | |
545 | ||
546 | 5) | |
547 | # Delete omap key (deep scrub only) | |
548 | objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1 | |
549 | ;; | |
550 | ||
551 | 6) | |
552 | # Add extra omap key (deep scrub only) | |
553 | echo extra > $dir/extra-val | |
554 | objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1 | |
555 | rm $dir/extra-val | |
556 | ;; | |
557 | ||
558 | 7) | |
559 | # Modify omap header (deep scrub only) | |
560 | echo -n newheader > $dir/hdr | |
561 | objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1 | |
562 | rm $dir/hdr | |
563 | ;; | |
564 | ||
565 | 8) | |
566 | rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1 | |
567 | rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1 | |
568 | ||
569 | # Break xattrs | |
570 | echo -n bad-val > $dir/bad-val | |
571 | objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1 | |
572 | objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1 | |
573 | echo -n val3-$objname > $dir/newval | |
574 | objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1 | |
575 | rm $dir/bad-val $dir/newval | |
576 | ;; | |
577 | ||
578 | 9) | |
579 | objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi | |
580 | echo -n D > $dir/change | |
581 | rados --pool $poolname put $objname $dir/change | |
582 | objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi | |
583 | rm $dir/oi $dir/change | |
584 | ;; | |
585 | ||
586 | # ROBJ10 must be handled after digests are re-computed by a deep scrub below | |
587 | # ROBJ11 must be handled with config change before deep scrub | |
588 | # ROBJ12 must be handled with config change before scrubs | |
589 | # ROBJ13 must be handled before scrubs | |
590 | ||
591 | 14) | |
592 | echo -n bad-val > $dir/bad-val | |
593 | objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1 | |
594 | objectstore_tool $dir 1 $objname rm-attr _ || return 1 | |
595 | rm $dir/bad-val | |
596 | ;; | |
597 | ||
598 | 15) | |
599 | objectstore_tool $dir $osd $objname rm-attr _ || return 1 | |
b5b8bbf5 FG |
600 | ;; |
601 | ||
602 | 16) | |
603 | objectstore_tool $dir 0 $objname rm-attr snapset || return 1 | |
604 | echo -n bad-val > $dir/bad-val | |
605 | objectstore_tool $dir 1 $objname set-attr snapset $dir/bad-val || return 1 | |
7c673cae FG |
606 | |
607 | esac | |
608 | done | |
609 | ||
610 | local pg=$(get_pg $poolname ROBJ0) | |
611 | ||
b5b8bbf5 FG |
612 | inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 |
613 | inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 | |
614 | inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 | |
615 | inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 | |
7c673cae FG |
616 | |
617 | pg_scrub $pg | |
618 | ||
619 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
620 | # Check pg count | |
621 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
622 | # Check pgid | |
623 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
624 | ||
625 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
626 | # Get epoch for repair-get requests | |
627 | epoch=$(jq .epoch $dir/json) | |
628 | ||
629 | jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson | |
630 | { | |
631 | "inconsistents": [ | |
632 | { | |
633 | "shards": [ | |
634 | { | |
635 | "size": 7, | |
636 | "errors": [], | |
b5b8bbf5 FG |
637 | "osd": 0, |
638 | "primary": false | |
7c673cae FG |
639 | }, |
640 | { | |
641 | "size": 9, | |
642 | "errors": [ | |
b5b8bbf5 FG |
643 | "size_mismatch_oi", |
644 | "obj_size_oi_mismatch" | |
7c673cae | 645 | ], |
b5b8bbf5 FG |
646 | "osd": 1, |
647 | "primary": true | |
7c673cae FG |
648 | } |
649 | ], | |
224ce89b | 650 | "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])", |
7c673cae | 651 | "union_shard_errors": [ |
b5b8bbf5 FG |
652 | "size_mismatch_oi", |
653 | "obj_size_oi_mismatch" | |
7c673cae FG |
654 | ], |
655 | "errors": [ | |
656 | "size_mismatch" | |
657 | ], | |
658 | "object": { | |
659 | "version": 3, | |
660 | "snap": "head", | |
661 | "locator": "", | |
662 | "nspace": "", | |
663 | "name": "ROBJ1" | |
664 | } | |
665 | }, | |
666 | { | |
667 | "shards": [ | |
668 | { | |
669 | "errors": [ | |
670 | "stat_error" | |
671 | ], | |
b5b8bbf5 FG |
672 | "osd": 0, |
673 | "primary": false | |
7c673cae FG |
674 | }, |
675 | { | |
676 | "size": 7, | |
677 | "errors": [], | |
b5b8bbf5 FG |
678 | "osd": 1, |
679 | "primary": true | |
7c673cae FG |
680 | } |
681 | ], | |
224ce89b | 682 | "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])", |
7c673cae FG |
683 | "union_shard_errors": [ |
684 | "stat_error" | |
685 | ], | |
686 | "errors": [], | |
687 | "object": { | |
31f18b77 | 688 | "version": 36, |
7c673cae FG |
689 | "snap": "head", |
690 | "locator": "", | |
691 | "nspace": "", | |
692 | "name": "ROBJ12" | |
693 | } | |
694 | }, | |
695 | { | |
696 | "shards": [ | |
697 | { | |
698 | "errors": [ | |
699 | "stat_error" | |
700 | ], | |
b5b8bbf5 FG |
701 | "osd": 0, |
702 | "primary": false | |
7c673cae FG |
703 | }, |
704 | { | |
705 | "size": 7, | |
706 | "errors": [], | |
b5b8bbf5 FG |
707 | "osd": 1, |
708 | "primary": true | |
7c673cae FG |
709 | } |
710 | ], | |
224ce89b | 711 | "selected_object_info": "3:d60617f9:::ROBJ13:head(47'55 osd.0.0:54 dirty|omap|data_digest|omap_digest s 7 uv 39 dd 2ddbf8f5 od 6441854d alloc_hint [0 0 0])", |
7c673cae FG |
712 | "union_shard_errors": [ |
713 | "stat_error" | |
714 | ], | |
715 | "errors": [], | |
716 | "object": { | |
31f18b77 | 717 | "version": 39, |
7c673cae FG |
718 | "snap": "head", |
719 | "locator": "", | |
720 | "nspace": "", | |
721 | "name": "ROBJ13" | |
722 | } | |
723 | }, | |
724 | { | |
725 | "shards": [ | |
726 | { | |
727 | "size": 7, | |
b5b8bbf5 FG |
728 | "attrs": [ |
729 | { | |
730 | "Base64": false, | |
731 | "value": "", | |
732 | "name": "_" | |
733 | }, | |
734 | { | |
735 | "Base64": true, | |
736 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", | |
737 | "name": "snapset" | |
738 | } | |
739 | ], | |
7c673cae FG |
740 | "errors": [ |
741 | "oi_attr_corrupted" | |
742 | ], | |
b5b8bbf5 FG |
743 | "osd": 0, |
744 | "primary": false | |
7c673cae FG |
745 | }, |
746 | { | |
747 | "size": 7, | |
b5b8bbf5 FG |
748 | "attrs": [ |
749 | { | |
750 | "Base64": true, | |
751 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", | |
752 | "name": "snapset" | |
753 | } | |
754 | ], | |
7c673cae FG |
755 | "errors": [ |
756 | "oi_attr_missing" | |
757 | ], | |
b5b8bbf5 FG |
758 | "osd": 1, |
759 | "primary": true | |
7c673cae FG |
760 | } |
761 | ], | |
762 | "union_shard_errors": [ | |
763 | "oi_attr_missing", | |
764 | "oi_attr_corrupted" | |
765 | ], | |
766 | "errors": [], | |
767 | "object": { | |
768 | "version": 0, | |
769 | "snap": "head", | |
770 | "locator": "", | |
771 | "nspace": "", | |
772 | "name": "ROBJ14" | |
773 | } | |
774 | }, | |
775 | { | |
776 | "shards": [ | |
777 | { | |
778 | "attrs": [ | |
779 | { | |
780 | "Base64": true, | |
781 | "value": "", | |
782 | "name": "_" | |
783 | }, | |
784 | { | |
785 | "Base64": true, | |
31f18b77 | 786 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
787 | "name": "snapset" |
788 | } | |
789 | ], | |
790 | "size": 7, | |
791 | "errors": [], | |
b5b8bbf5 FG |
792 | "osd": 0, |
793 | "primary": false | |
7c673cae FG |
794 | }, |
795 | { | |
796 | "attrs": [ | |
797 | { | |
798 | "Base64": true, | |
31f18b77 | 799 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
800 | "name": "snapset" |
801 | } | |
802 | ], | |
803 | "size": 7, | |
804 | "errors": [ | |
805 | "oi_attr_missing" | |
806 | ], | |
b5b8bbf5 FG |
807 | "osd": 1, |
808 | "primary": true | |
7c673cae FG |
809 | } |
810 | ], | |
224ce89b | 811 | "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])", |
7c673cae FG |
812 | "union_shard_errors": [ |
813 | "oi_attr_missing" | |
814 | ], | |
b5b8bbf5 | 815 | "errors": [], |
7c673cae | 816 | "object": { |
31f18b77 | 817 | "version": 45, |
7c673cae FG |
818 | "snap": "head", |
819 | "locator": "", | |
820 | "nspace": "", | |
821 | "name": "ROBJ15" | |
822 | } | |
823 | }, | |
b5b8bbf5 FG |
824 | { |
825 | "errors": [], | |
826 | "object": { | |
827 | "locator": "", | |
828 | "name": "ROBJ16", | |
829 | "nspace": "", | |
830 | "snap": "head", | |
831 | "version": 0 | |
832 | }, | |
833 | "shards": [ | |
834 | { | |
835 | "attrs": [ | |
836 | { | |
837 | "Base64": true, | |
838 | "name": "_", | |
839 | "value": "" | |
840 | } | |
841 | ], | |
842 | "errors": [ | |
843 | "ss_attr_missing" | |
844 | ], | |
845 | "osd": 0, | |
846 | "primary": false, | |
847 | "size": 7 | |
848 | }, | |
849 | { | |
850 | "attrs": [ | |
851 | { | |
852 | "Base64": true, | |
853 | "name": "_", | |
854 | "value": "" | |
855 | }, | |
856 | { | |
857 | "Base64": false, | |
858 | "name": "snapset", | |
859 | "value": "bad-val" | |
860 | } | |
861 | ], | |
862 | "errors": [ | |
863 | "ss_attr_corrupted" | |
864 | ], | |
865 | "osd": 1, | |
866 | "primary": true, | |
867 | "size": 7 | |
868 | } | |
869 | ], | |
870 | "union_shard_errors": [ | |
871 | "ss_attr_missing", | |
872 | "ss_attr_corrupted" | |
873 | ] | |
874 | }, | |
7c673cae FG |
875 | { |
876 | "shards": [ | |
877 | { | |
878 | "size": 7, | |
879 | "errors": [], | |
b5b8bbf5 FG |
880 | "osd": 0, |
881 | "primary": false | |
7c673cae FG |
882 | }, |
883 | { | |
884 | "errors": [ | |
885 | "missing" | |
886 | ], | |
b5b8bbf5 FG |
887 | "osd": 1, |
888 | "primary": true | |
7c673cae FG |
889 | } |
890 | ], | |
224ce89b | 891 | "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])", |
7c673cae FG |
892 | "union_shard_errors": [ |
893 | "missing" | |
894 | ], | |
895 | "errors": [], | |
896 | "object": { | |
897 | "version": 9, | |
898 | "snap": "head", | |
899 | "locator": "", | |
900 | "nspace": "", | |
901 | "name": "ROBJ3" | |
902 | } | |
903 | }, | |
904 | { | |
905 | "shards": [ | |
906 | { | |
907 | "attrs": [ | |
908 | { | |
909 | "Base64": true, | |
910 | "value": "", | |
911 | "name": "_" | |
912 | }, | |
913 | { | |
914 | "Base64": false, | |
915 | "value": "bad-val", | |
916 | "name": "_key1-ROBJ8" | |
917 | }, | |
918 | { | |
919 | "Base64": false, | |
920 | "value": "val3-ROBJ8", | |
921 | "name": "_key3-ROBJ8" | |
922 | }, | |
923 | { | |
924 | "Base64": true, | |
31f18b77 | 925 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
926 | "name": "snapset" |
927 | } | |
928 | ], | |
929 | "size": 7, | |
930 | "errors": [], | |
b5b8bbf5 FG |
931 | "osd": 0, |
932 | "primary": false | |
7c673cae FG |
933 | }, |
934 | { | |
935 | "attrs": [ | |
936 | { | |
937 | "Base64": true, | |
938 | "value": "", | |
939 | "name": "_" | |
940 | }, | |
941 | { | |
942 | "Base64": false, | |
943 | "value": "val1-ROBJ8", | |
944 | "name": "_key1-ROBJ8" | |
945 | }, | |
946 | { | |
947 | "Base64": false, | |
948 | "value": "val2-ROBJ8", | |
949 | "name": "_key2-ROBJ8" | |
950 | }, | |
951 | { | |
952 | "Base64": true, | |
31f18b77 | 953 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
954 | "name": "snapset" |
955 | } | |
956 | ], | |
957 | "size": 7, | |
958 | "errors": [], | |
b5b8bbf5 FG |
959 | "osd": 1, |
960 | "primary": true | |
7c673cae FG |
961 | } |
962 | ], | |
b5b8bbf5 | 963 | "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 66 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])", |
7c673cae FG |
964 | "union_shard_errors": [], |
965 | "errors": [ | |
966 | "attr_value_mismatch", | |
967 | "attr_name_mismatch" | |
968 | ], | |
969 | "object": { | |
b5b8bbf5 | 970 | "version": 66, |
7c673cae FG |
971 | "snap": "head", |
972 | "locator": "", | |
973 | "nspace": "", | |
974 | "name": "ROBJ8" | |
975 | } | |
976 | }, | |
977 | { | |
978 | "shards": [ | |
979 | { | |
b5b8bbf5 | 980 | "object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 67 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])", |
7c673cae FG |
981 | "size": 1, |
982 | "errors": [], | |
b5b8bbf5 FG |
983 | "osd": 0, |
984 | "primary": false | |
7c673cae FG |
985 | }, |
986 | { | |
224ce89b | 987 | "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])", |
7c673cae | 988 | "size": 1, |
b5b8bbf5 FG |
989 | "errors": [ |
990 | "obj_size_oi_mismatch" | |
991 | ], | |
992 | "osd": 1, | |
993 | "primary": true | |
7c673cae FG |
994 | } |
995 | ], | |
b5b8bbf5 FG |
996 | "selected_object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 67 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])", |
997 | "union_shard_errors": [ | |
998 | "obj_size_oi_mismatch" | |
999 | ], | |
7c673cae | 1000 | "errors": [ |
b5b8bbf5 | 1001 | "object_info_inconsistency" |
7c673cae FG |
1002 | ], |
1003 | "object": { | |
b5b8bbf5 | 1004 | "version": 67, |
7c673cae FG |
1005 | "snap": "head", |
1006 | "locator": "", | |
1007 | "nspace": "", | |
1008 | "name": "ROBJ9" | |
1009 | } | |
1010 | } | |
1011 | ], | |
1012 | "epoch": 0 | |
1013 | } | |
1014 | EOF | |
1015 | ||
1016 | jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson | |
31f18b77 | 1017 | diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
7c673cae FG |
1018 | if test $getjson = "yes" |
1019 | then | |
1020 | jq '.' $dir/json > save1.json | |
1021 | fi | |
1022 | ||
1023 | if which jsonschema > /dev/null; | |
1024 | then | |
1025 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
1026 | fi | |
1027 | ||
7c673cae FG |
1028 | objname=ROBJ9 |
1029 | # Change data and size again because digest was recomputed | |
1030 | echo -n ZZZ > $dir/change | |
1031 | rados --pool $poolname put $objname $dir/change | |
1032 | # Set one to an even older value | |
1033 | objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi | |
1034 | rm $dir/oi $dir/change | |
1035 | ||
1036 | objname=ROBJ10 | |
1037 | objectstore_tool $dir 1 $objname get-attr _ > $dir/oi | |
1038 | rados --pool $poolname setomapval $objname key2-$objname val2-$objname | |
1039 | objectstore_tool $dir 0 $objname set-attr _ $dir/oi | |
1040 | objectstore_tool $dir 1 $objname set-attr _ $dir/oi | |
1041 | rm $dir/oi | |
1042 | ||
b5b8bbf5 FG |
1043 | inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 |
1044 | inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 | |
1045 | inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 | |
1046 | inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 | |
7c673cae FG |
1047 | pg_deep_scrub $pg |
1048 | ||
1049 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
1050 | # Check pg count | |
1051 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
1052 | # Check pgid | |
1053 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
1054 | ||
1055 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
1056 | # Get epoch for repair-get requests | |
1057 | epoch=$(jq .epoch $dir/json) | |
1058 | ||
1059 | jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson | |
1060 | { | |
1061 | "inconsistents": [ | |
1062 | { | |
1063 | "shards": [ | |
1064 | { | |
1065 | "data_digest": "0x2ddbf8f5", | |
1066 | "omap_digest": "0xf5fba2c6", | |
1067 | "size": 7, | |
1068 | "errors": [], | |
b5b8bbf5 FG |
1069 | "osd": 0, |
1070 | "primary": false | |
7c673cae FG |
1071 | }, |
1072 | { | |
1073 | "data_digest": "0x2d4a11c2", | |
1074 | "omap_digest": "0xf5fba2c6", | |
1075 | "size": 9, | |
1076 | "errors": [ | |
1077 | "data_digest_mismatch_oi", | |
b5b8bbf5 FG |
1078 | "size_mismatch_oi", |
1079 | "obj_size_oi_mismatch" | |
7c673cae | 1080 | ], |
b5b8bbf5 FG |
1081 | "osd": 1, |
1082 | "primary": true | |
7c673cae FG |
1083 | } |
1084 | ], | |
224ce89b | 1085 | "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])", |
7c673cae FG |
1086 | "union_shard_errors": [ |
1087 | "data_digest_mismatch_oi", | |
b5b8bbf5 FG |
1088 | "size_mismatch_oi", |
1089 | "obj_size_oi_mismatch" | |
7c673cae FG |
1090 | ], |
1091 | "errors": [ | |
1092 | "data_digest_mismatch", | |
1093 | "size_mismatch" | |
1094 | ], | |
1095 | "object": { | |
1096 | "version": 3, | |
1097 | "snap": "head", | |
1098 | "locator": "", | |
1099 | "nspace": "", | |
1100 | "name": "ROBJ1" | |
1101 | } | |
1102 | }, | |
1103 | { | |
1104 | "shards": [ | |
1105 | { | |
1106 | "data_digest": "0x2ddbf8f5", | |
1107 | "omap_digest": "0xa8dd5adc", | |
1108 | "size": 7, | |
1109 | "errors": [ | |
1110 | "omap_digest_mismatch_oi" | |
1111 | ], | |
b5b8bbf5 FG |
1112 | "osd": 0, |
1113 | "primary": false | |
7c673cae FG |
1114 | }, |
1115 | { | |
1116 | "data_digest": "0x2ddbf8f5", | |
1117 | "omap_digest": "0xa8dd5adc", | |
1118 | "size": 7, | |
1119 | "errors": [ | |
1120 | "omap_digest_mismatch_oi" | |
1121 | ], | |
b5b8bbf5 FG |
1122 | "osd": 1, |
1123 | "primary": true | |
7c673cae FG |
1124 | } |
1125 | ], | |
224ce89b | 1126 | "selected_object_info": "3:b1f19cbd:::ROBJ10:head(47'51 osd.0.0:50 dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [0 0 0])", |
7c673cae FG |
1127 | "union_shard_errors": [ |
1128 | "omap_digest_mismatch_oi" | |
1129 | ], | |
1130 | "errors": [], | |
1131 | "object": { | |
31f18b77 | 1132 | "version": 30, |
7c673cae FG |
1133 | "snap": "head", |
1134 | "locator": "", | |
1135 | "nspace": "", | |
1136 | "name": "ROBJ10" | |
1137 | } | |
1138 | }, | |
1139 | { | |
1140 | "shards": [ | |
1141 | { | |
1142 | "data_digest": "0x2ddbf8f5", | |
1143 | "omap_digest": "0xa03cef03", | |
1144 | "size": 7, | |
1145 | "errors": [], | |
b5b8bbf5 FG |
1146 | "osd": 0, |
1147 | "primary": false | |
7c673cae FG |
1148 | }, |
1149 | { | |
1150 | "size": 7, | |
1151 | "errors": [ | |
1152 | "read_error" | |
1153 | ], | |
b5b8bbf5 FG |
1154 | "osd": 1, |
1155 | "primary": true | |
7c673cae FG |
1156 | } |
1157 | ], | |
224ce89b | 1158 | "selected_object_info": "3:87abbf36:::ROBJ11:head(47'48 osd.0.0:47 dirty|omap|data_digest|omap_digest s 7 uv 33 dd 2ddbf8f5 od a03cef03 alloc_hint [0 0 0])", |
7c673cae FG |
1159 | "union_shard_errors": [ |
1160 | "read_error" | |
1161 | ], | |
1162 | "errors": [], | |
1163 | "object": { | |
31f18b77 | 1164 | "version": 33, |
7c673cae FG |
1165 | "snap": "head", |
1166 | "locator": "", | |
1167 | "nspace": "", | |
1168 | "name": "ROBJ11" | |
1169 | } | |
1170 | }, | |
1171 | { | |
1172 | "shards": [ | |
1173 | { | |
1174 | "errors": [ | |
1175 | "stat_error" | |
1176 | ], | |
b5b8bbf5 FG |
1177 | "osd": 0, |
1178 | "primary": false | |
7c673cae FG |
1179 | }, |
1180 | { | |
1181 | "data_digest": "0x2ddbf8f5", | |
1182 | "omap_digest": "0x067f306a", | |
1183 | "size": 7, | |
1184 | "errors": [], | |
b5b8bbf5 FG |
1185 | "osd": 1, |
1186 | "primary": true | |
7c673cae FG |
1187 | } |
1188 | ], | |
224ce89b | 1189 | "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])", |
7c673cae FG |
1190 | "union_shard_errors": [ |
1191 | "stat_error" | |
1192 | ], | |
1193 | "errors": [], | |
1194 | "object": { | |
31f18b77 | 1195 | "version": 36, |
7c673cae FG |
1196 | "snap": "head", |
1197 | "locator": "", | |
1198 | "nspace": "", | |
1199 | "name": "ROBJ12" | |
1200 | } | |
1201 | }, | |
1202 | { | |
1203 | "shards": [ | |
1204 | { | |
1205 | "errors": [ | |
1206 | "stat_error" | |
1207 | ], | |
b5b8bbf5 FG |
1208 | "osd": 0, |
1209 | "primary": false | |
7c673cae FG |
1210 | }, |
1211 | { | |
1212 | "size": 7, | |
1213 | "errors": [ | |
1214 | "read_error" | |
1215 | ], | |
b5b8bbf5 FG |
1216 | "osd": 1, |
1217 | "primary": true | |
7c673cae FG |
1218 | } |
1219 | ], | |
1220 | "union_shard_errors": [ | |
1221 | "stat_error", | |
1222 | "read_error" | |
1223 | ], | |
1224 | "errors": [], | |
1225 | "object": { | |
1226 | "version": 0, | |
1227 | "snap": "head", | |
1228 | "locator": "", | |
1229 | "nspace": "", | |
1230 | "name": "ROBJ13" | |
1231 | } | |
1232 | }, | |
1233 | { | |
1234 | "shards": [ | |
1235 | { | |
b5b8bbf5 FG |
1236 | "attrs": [ |
1237 | { | |
1238 | "Base64": false, | |
1239 | "value": "", | |
1240 | "name": "_" | |
1241 | }, | |
1242 | { | |
1243 | "Base64": true, | |
1244 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", | |
1245 | "name": "snapset" | |
1246 | } | |
1247 | ], | |
7c673cae FG |
1248 | "data_digest": "0x2ddbf8f5", |
1249 | "omap_digest": "0x4f14f849", | |
1250 | "size": 7, | |
1251 | "errors": [ | |
1252 | "oi_attr_corrupted" | |
1253 | ], | |
b5b8bbf5 FG |
1254 | "osd": 0, |
1255 | "primary": false | |
7c673cae FG |
1256 | }, |
1257 | { | |
b5b8bbf5 FG |
1258 | "attrs": [ |
1259 | { | |
1260 | "Base64": true, | |
1261 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", | |
1262 | "name": "snapset" | |
1263 | } | |
1264 | ], | |
7c673cae FG |
1265 | "data_digest": "0x2ddbf8f5", |
1266 | "omap_digest": "0x4f14f849", | |
1267 | "size": 7, | |
1268 | "errors": [ | |
1269 | "oi_attr_missing" | |
1270 | ], | |
b5b8bbf5 FG |
1271 | "osd": 1, |
1272 | "primary": true | |
7c673cae FG |
1273 | } |
1274 | ], | |
1275 | "union_shard_errors": [ | |
1276 | "oi_attr_missing", | |
1277 | "oi_attr_corrupted" | |
1278 | ], | |
1279 | "errors": [], | |
1280 | "object": { | |
1281 | "version": 0, | |
1282 | "snap": "head", | |
1283 | "locator": "", | |
1284 | "nspace": "", | |
1285 | "name": "ROBJ14" | |
1286 | } | |
1287 | }, | |
1288 | { | |
1289 | "shards": [ | |
1290 | { | |
1291 | "attrs": [ | |
1292 | { | |
1293 | "Base64": true, | |
1294 | "value": "", | |
1295 | "name": "_" | |
1296 | }, | |
1297 | { | |
1298 | "Base64": true, | |
31f18b77 | 1299 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
1300 | "name": "snapset" |
1301 | } | |
1302 | ], | |
1303 | "data_digest": "0x2ddbf8f5", | |
1304 | "omap_digest": "0x2d2a4d6e", | |
1305 | "size": 7, | |
1306 | "errors": [], | |
b5b8bbf5 FG |
1307 | "osd": 0, |
1308 | "primary": false | |
7c673cae FG |
1309 | }, |
1310 | { | |
1311 | "attrs": [ | |
1312 | { | |
1313 | "Base64": true, | |
31f18b77 | 1314 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
1315 | "name": "snapset" |
1316 | } | |
1317 | ], | |
1318 | "data_digest": "0x2ddbf8f5", | |
1319 | "omap_digest": "0x2d2a4d6e", | |
1320 | "size": 7, | |
1321 | "errors": [ | |
1322 | "oi_attr_missing" | |
1323 | ], | |
b5b8bbf5 FG |
1324 | "osd": 1, |
1325 | "primary": true | |
7c673cae FG |
1326 | } |
1327 | ], | |
224ce89b | 1328 | "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])", |
7c673cae FG |
1329 | "union_shard_errors": [ |
1330 | "oi_attr_missing" | |
1331 | ], | |
b5b8bbf5 | 1332 | "errors": [], |
7c673cae | 1333 | "object": { |
31f18b77 | 1334 | "version": 45, |
7c673cae FG |
1335 | "snap": "head", |
1336 | "locator": "", | |
1337 | "nspace": "", | |
1338 | "name": "ROBJ15" | |
1339 | } | |
1340 | }, | |
b5b8bbf5 FG |
1341 | { |
1342 | "errors": [], | |
1343 | "object": { | |
1344 | "locator": "", | |
1345 | "name": "ROBJ16", | |
1346 | "nspace": "", | |
1347 | "snap": "head", | |
1348 | "version": 0 | |
1349 | }, | |
1350 | "shards": [ | |
1351 | { | |
1352 | "attrs": [ | |
1353 | { | |
1354 | "Base64": true, | |
1355 | "name": "_", | |
1356 | "value": "" | |
1357 | } | |
1358 | ], | |
1359 | "data_digest": "0x2ddbf8f5", | |
1360 | "errors": [ | |
1361 | "ss_attr_missing" | |
1362 | ], | |
1363 | "omap_digest": "0x8b699207", | |
1364 | "osd": 0, | |
1365 | "primary": false, | |
1366 | "size": 7 | |
1367 | }, | |
1368 | { | |
1369 | "attrs": [ | |
1370 | { | |
1371 | "Base64": true, | |
1372 | "name": "_", | |
1373 | "value": "" | |
1374 | }, | |
1375 | { | |
1376 | "Base64": false, | |
1377 | "name": "snapset", | |
1378 | "value": "bad-val" | |
1379 | } | |
1380 | ], | |
1381 | "data_digest": "0x2ddbf8f5", | |
1382 | "errors": [ | |
1383 | "ss_attr_corrupted" | |
1384 | ], | |
1385 | "omap_digest": "0x8b699207", | |
1386 | "osd": 1, | |
1387 | "primary": true, | |
1388 | "size": 7 | |
1389 | } | |
1390 | ], | |
1391 | "union_shard_errors": [ | |
1392 | "ss_attr_missing", | |
1393 | "ss_attr_corrupted" | |
1394 | ] | |
1395 | }, | |
7c673cae FG |
1396 | { |
1397 | "shards": [ | |
1398 | { | |
1399 | "data_digest": "0x578a4830", | |
1400 | "omap_digest": "0xf8e11918", | |
1401 | "size": 7, | |
1402 | "errors": [ | |
1403 | "data_digest_mismatch_oi" | |
1404 | ], | |
b5b8bbf5 FG |
1405 | "osd": 0, |
1406 | "primary": false | |
7c673cae FG |
1407 | }, |
1408 | { | |
1409 | "data_digest": "0x2ddbf8f5", | |
1410 | "omap_digest": "0xf8e11918", | |
1411 | "size": 7, | |
1412 | "errors": [], | |
b5b8bbf5 FG |
1413 | "osd": 1, |
1414 | "primary": true | |
7c673cae FG |
1415 | } |
1416 | ], | |
224ce89b | 1417 | "selected_object_info": "3:e97ce31e:::ROBJ2:head(47'56 osd.0.0:55 dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [0 0 0])", |
7c673cae FG |
1418 | "union_shard_errors": [ |
1419 | "data_digest_mismatch_oi" | |
1420 | ], | |
1421 | "errors": [ | |
1422 | "data_digest_mismatch" | |
1423 | ], | |
1424 | "object": { | |
1425 | "version": 6, | |
1426 | "snap": "head", | |
1427 | "locator": "", | |
1428 | "nspace": "", | |
1429 | "name": "ROBJ2" | |
1430 | } | |
1431 | }, | |
1432 | { | |
1433 | "shards": [ | |
1434 | { | |
1435 | "data_digest": "0x2ddbf8f5", | |
1436 | "omap_digest": "0x00b35dfd", | |
1437 | "size": 7, | |
1438 | "errors": [], | |
b5b8bbf5 FG |
1439 | "osd": 0, |
1440 | "primary": false | |
7c673cae FG |
1441 | }, |
1442 | { | |
1443 | "errors": [ | |
1444 | "missing" | |
1445 | ], | |
b5b8bbf5 FG |
1446 | "osd": 1, |
1447 | "primary": true | |
7c673cae FG |
1448 | } |
1449 | ], | |
224ce89b | 1450 | "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])", |
7c673cae FG |
1451 | "union_shard_errors": [ |
1452 | "missing" | |
1453 | ], | |
1454 | "errors": [], | |
1455 | "object": { | |
1456 | "version": 9, | |
1457 | "snap": "head", | |
1458 | "locator": "", | |
1459 | "nspace": "", | |
1460 | "name": "ROBJ3" | |
1461 | } | |
1462 | }, | |
1463 | { | |
1464 | "shards": [ | |
1465 | { | |
1466 | "data_digest": "0x2ddbf8f5", | |
1467 | "omap_digest": "0xd7178dfe", | |
1468 | "size": 7, | |
31f18b77 FG |
1469 | "errors": [ |
1470 | "omap_digest_mismatch_oi" | |
1471 | ], | |
b5b8bbf5 FG |
1472 | "osd": 0, |
1473 | "primary": false | |
7c673cae FG |
1474 | }, |
1475 | { | |
1476 | "data_digest": "0x2ddbf8f5", | |
1477 | "omap_digest": "0xe2d46ea4", | |
1478 | "size": 7, | |
31f18b77 | 1479 | "errors": [], |
b5b8bbf5 FG |
1480 | "osd": 1, |
1481 | "primary": true | |
7c673cae FG |
1482 | } |
1483 | ], | |
224ce89b | 1484 | "selected_object_info": "3:f4981d31:::ROBJ4:head(47'58 osd.0.0:57 dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [0 0 0])", |
7c673cae FG |
1485 | "union_shard_errors": [ |
1486 | "omap_digest_mismatch_oi" | |
1487 | ], | |
1488 | "errors": [ | |
1489 | "omap_digest_mismatch" | |
1490 | ], | |
1491 | "object": { | |
1492 | "version": 12, | |
1493 | "snap": "head", | |
1494 | "locator": "", | |
1495 | "nspace": "", | |
1496 | "name": "ROBJ4" | |
1497 | } | |
1498 | }, | |
1499 | { | |
1500 | "shards": [ | |
1501 | { | |
1502 | "data_digest": "0x2ddbf8f5", | |
1503 | "omap_digest": "0x1a862a41", | |
1504 | "size": 7, | |
1505 | "errors": [], | |
b5b8bbf5 FG |
1506 | "osd": 0, |
1507 | "primary": false | |
7c673cae FG |
1508 | }, |
1509 | { | |
1510 | "data_digest": "0x2ddbf8f5", | |
1511 | "omap_digest": "0x06cac8f6", | |
1512 | "size": 7, | |
1513 | "errors": [ | |
1514 | "omap_digest_mismatch_oi" | |
1515 | ], | |
b5b8bbf5 FG |
1516 | "osd": 1, |
1517 | "primary": true | |
7c673cae FG |
1518 | } |
1519 | ], | |
224ce89b | 1520 | "selected_object_info": "3:f4bfd4d1:::ROBJ5:head(47'59 osd.0.0:58 dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [0 0 0])", |
7c673cae FG |
1521 | "union_shard_errors": [ |
1522 | "omap_digest_mismatch_oi" | |
1523 | ], | |
1524 | "errors": [ | |
1525 | "omap_digest_mismatch" | |
1526 | ], | |
1527 | "object": { | |
1528 | "version": 15, | |
1529 | "snap": "head", | |
1530 | "locator": "", | |
1531 | "nspace": "", | |
1532 | "name": "ROBJ5" | |
1533 | } | |
1534 | }, | |
1535 | { | |
1536 | "shards": [ | |
1537 | { | |
1538 | "data_digest": "0x2ddbf8f5", | |
1539 | "omap_digest": "0x689ee887", | |
1540 | "size": 7, | |
31f18b77 FG |
1541 | "errors": [ |
1542 | "omap_digest_mismatch_oi" | |
1543 | ], | |
b5b8bbf5 FG |
1544 | "osd": 0, |
1545 | "primary": false | |
7c673cae FG |
1546 | }, |
1547 | { | |
1548 | "data_digest": "0x2ddbf8f5", | |
1549 | "omap_digest": "0x179c919f", | |
1550 | "size": 7, | |
31f18b77 | 1551 | "errors": [], |
b5b8bbf5 FG |
1552 | "osd": 1, |
1553 | "primary": true | |
7c673cae FG |
1554 | } |
1555 | ], | |
224ce89b | 1556 | "selected_object_info": "3:a53c12e8:::ROBJ6:head(47'50 osd.0.0:49 dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [0 0 0])", |
7c673cae FG |
1557 | "union_shard_errors": [ |
1558 | "omap_digest_mismatch_oi" | |
1559 | ], | |
1560 | "errors": [ | |
1561 | "omap_digest_mismatch" | |
1562 | ], | |
1563 | "object": { | |
1564 | "version": 18, | |
1565 | "snap": "head", | |
1566 | "locator": "", | |
1567 | "nspace": "", | |
1568 | "name": "ROBJ6" | |
1569 | } | |
1570 | }, | |
1571 | { | |
1572 | "shards": [ | |
1573 | { | |
1574 | "data_digest": "0x2ddbf8f5", | |
1575 | "omap_digest": "0xefced57a", | |
1576 | "size": 7, | |
1577 | "errors": [], | |
b5b8bbf5 FG |
1578 | "osd": 0, |
1579 | "primary": false | |
7c673cae FG |
1580 | }, |
1581 | { | |
1582 | "data_digest": "0x2ddbf8f5", | |
1583 | "omap_digest": "0x6a73cc07", | |
1584 | "size": 7, | |
1585 | "errors": [ | |
1586 | "omap_digest_mismatch_oi" | |
1587 | ], | |
b5b8bbf5 FG |
1588 | "osd": 1, |
1589 | "primary": true | |
7c673cae FG |
1590 | } |
1591 | ], | |
224ce89b | 1592 | "selected_object_info": "3:8b55fa4b:::ROBJ7:head(47'49 osd.0.0:48 dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [0 0 0])", |
7c673cae FG |
1593 | "union_shard_errors": [ |
1594 | "omap_digest_mismatch_oi" | |
1595 | ], | |
1596 | "errors": [ | |
1597 | "omap_digest_mismatch" | |
1598 | ], | |
1599 | "object": { | |
1600 | "version": 21, | |
1601 | "snap": "head", | |
1602 | "locator": "", | |
1603 | "nspace": "", | |
1604 | "name": "ROBJ7" | |
1605 | } | |
1606 | }, | |
1607 | { | |
1608 | "shards": [ | |
1609 | { | |
1610 | "attrs": [ | |
1611 | { | |
1612 | "Base64": true, | |
1613 | "value": "", | |
1614 | "name": "_" | |
1615 | }, | |
1616 | { | |
1617 | "Base64": false, | |
1618 | "value": "bad-val", | |
1619 | "name": "_key1-ROBJ8" | |
1620 | }, | |
1621 | { | |
1622 | "Base64": false, | |
1623 | "value": "val3-ROBJ8", | |
1624 | "name": "_key3-ROBJ8" | |
1625 | }, | |
1626 | { | |
1627 | "Base64": true, | |
31f18b77 | 1628 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
1629 | "name": "snapset" |
1630 | } | |
1631 | ], | |
1632 | "data_digest": "0x2ddbf8f5", | |
1633 | "omap_digest": "0xd6be81dc", | |
1634 | "size": 7, | |
1635 | "errors": [], | |
b5b8bbf5 FG |
1636 | "osd": 0, |
1637 | "primary": false | |
7c673cae FG |
1638 | }, |
1639 | { | |
1640 | "attrs": [ | |
1641 | { | |
1642 | "Base64": true, | |
1643 | "value": "", | |
1644 | "name": "_" | |
1645 | }, | |
1646 | { | |
1647 | "Base64": false, | |
1648 | "value": "val1-ROBJ8", | |
1649 | "name": "_key1-ROBJ8" | |
1650 | }, | |
1651 | { | |
1652 | "Base64": false, | |
1653 | "value": "val2-ROBJ8", | |
1654 | "name": "_key2-ROBJ8" | |
1655 | }, | |
1656 | { | |
1657 | "Base64": true, | |
31f18b77 | 1658 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
1659 | "name": "snapset" |
1660 | } | |
1661 | ], | |
1662 | "data_digest": "0x2ddbf8f5", | |
1663 | "omap_digest": "0xd6be81dc", | |
1664 | "size": 7, | |
1665 | "errors": [], | |
b5b8bbf5 FG |
1666 | "osd": 1, |
1667 | "primary": true | |
7c673cae FG |
1668 | } |
1669 | ], | |
b5b8bbf5 | 1670 | "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 66 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])", |
7c673cae FG |
1671 | "union_shard_errors": [], |
1672 | "errors": [ | |
1673 | "attr_value_mismatch", | |
1674 | "attr_name_mismatch" | |
1675 | ], | |
1676 | "object": { | |
b5b8bbf5 | 1677 | "version": 66, |
7c673cae FG |
1678 | "snap": "head", |
1679 | "locator": "", | |
1680 | "nspace": "", | |
1681 | "name": "ROBJ8" | |
1682 | } | |
1683 | }, | |
1684 | { | |
1685 | "shards": [ | |
1686 | { | |
224ce89b | 1687 | "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])", |
7c673cae FG |
1688 | "data_digest": "0x1f26fb26", |
1689 | "omap_digest": "0x2eecc539", | |
1690 | "size": 3, | |
b5b8bbf5 FG |
1691 | "errors": [ |
1692 | "obj_size_oi_mismatch" | |
1693 | ], | |
1694 | "osd": 0, | |
1695 | "primary": false | |
7c673cae FG |
1696 | }, |
1697 | { | |
b5b8bbf5 | 1698 | "object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 68 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])", |
7c673cae FG |
1699 | "data_digest": "0x1f26fb26", |
1700 | "omap_digest": "0x2eecc539", | |
1701 | "size": 3, | |
1702 | "errors": [], | |
b5b8bbf5 FG |
1703 | "osd": 1, |
1704 | "primary": true | |
7c673cae FG |
1705 | } |
1706 | ], | |
b5b8bbf5 FG |
1707 | "selected_object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 68 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])", |
1708 | "union_shard_errors": [ | |
1709 | "obj_size_oi_mismatch" | |
1710 | ], | |
7c673cae | 1711 | "errors": [ |
b5b8bbf5 | 1712 | "object_info_inconsistency" |
7c673cae FG |
1713 | ], |
1714 | "object": { | |
b5b8bbf5 | 1715 | "version": 68, |
7c673cae FG |
1716 | "snap": "head", |
1717 | "locator": "", | |
1718 | "nspace": "", | |
1719 | "name": "ROBJ9" | |
1720 | } | |
1721 | } | |
1722 | ], | |
1723 | "epoch": 0 | |
1724 | } | |
1725 | EOF | |
1726 | ||
1727 | jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson | |
31f18b77 | 1728 | diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
7c673cae FG |
1729 | if test $getjson = "yes" |
1730 | then | |
1731 | jq '.' $dir/json > save2.json | |
1732 | fi | |
1733 | ||
1734 | if which jsonschema > /dev/null; | |
1735 | then | |
1736 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
1737 | fi | |
1738 | ||
1739 | rados rmpool $poolname $poolname --yes-i-really-really-mean-it | |
1740 | teardown $dir || return 1 | |
1741 | } | |
1742 | ||
1743 | ||
1744 | # | |
1745 | # Test scrub errors for an erasure coded pool | |
1746 | # | |
1747 | function corrupt_scrub_erasure() { | |
1748 | local dir=$1 | |
1749 | local allow_overwrites=$2 | |
1750 | local poolname=ecpool | |
1751 | local total_objs=5 | |
1752 | ||
1753 | setup $dir || return 1 | |
1754 | run_mon $dir a || return 1 | |
1755 | run_mgr $dir x || return 1 | |
1756 | for id in $(seq 0 2) ; do | |
1757 | if [ "$allow_overwrites" = "true" ]; then | |
1758 | run_osd_bluestore $dir $id || return 1 | |
1759 | else | |
1760 | run_osd $dir $id || return 1 | |
1761 | fi | |
1762 | done | |
c07f9fc5 | 1763 | create_rbd_pool || return 1 |
b5b8bbf5 | 1764 | create_pool foo 1 |
7c673cae FG |
1765 | |
1766 | create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1 | |
c07f9fc5 | 1767 | wait_for_clean || return 1 |
7c673cae FG |
1768 | |
1769 | for i in $(seq 1 $total_objs) ; do | |
1770 | objname=EOBJ${i} | |
1771 | add_something $dir $poolname $objname || return 1 | |
1772 | ||
1773 | local osd=$(expr $i % 2) | |
1774 | ||
1775 | case $i in | |
1776 | 1) | |
1777 | # Size (deep scrub data_digest too) | |
1778 | local payload=UVWXYZZZ | |
1779 | echo $payload > $dir/CORRUPT | |
1780 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
1781 | ;; | |
1782 | ||
1783 | 2) | |
1784 | # Corrupt EC shard | |
1785 | dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1 | |
1786 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
1787 | ;; | |
1788 | ||
1789 | 3) | |
1790 | # missing | |
1791 | objectstore_tool $dir $osd $objname remove || return 1 | |
1792 | ;; | |
1793 | ||
1794 | 4) | |
1795 | rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1 | |
1796 | rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1 | |
1797 | ||
1798 | # Break xattrs | |
1799 | echo -n bad-val > $dir/bad-val | |
1800 | objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1 | |
1801 | objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1 | |
1802 | echo -n val3-$objname > $dir/newval | |
1803 | objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1 | |
1804 | rm $dir/bad-val $dir/newval | |
1805 | ;; | |
1806 | ||
1807 | 5) | |
1808 | # Corrupt EC shard | |
1809 | dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2 | |
1810 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
1811 | ;; | |
1812 | ||
1813 | esac | |
1814 | done | |
1815 | ||
1816 | local pg=$(get_pg $poolname EOBJ0) | |
1817 | ||
1818 | pg_scrub $pg | |
1819 | ||
1820 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
1821 | # Check pg count | |
1822 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
1823 | # Check pgid | |
1824 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
1825 | ||
1826 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
1827 | # Get epoch for repair-get requests | |
1828 | epoch=$(jq .epoch $dir/json) | |
1829 | ||
1830 | jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson | |
1831 | { | |
1832 | "inconsistents": [ | |
1833 | { | |
1834 | "shards": [ | |
1835 | { | |
1836 | "size": 2048, | |
1837 | "errors": [], | |
1838 | "shard": 2, | |
b5b8bbf5 FG |
1839 | "osd": 0, |
1840 | "primary": false | |
7c673cae FG |
1841 | }, |
1842 | { | |
1843 | "size": 9, | |
224ce89b | 1844 | "shard": 0, |
7c673cae | 1845 | "errors": [ |
b5b8bbf5 FG |
1846 | "size_mismatch_oi", |
1847 | "obj_size_oi_mismatch" | |
7c673cae | 1848 | ], |
b5b8bbf5 FG |
1849 | "osd": 1, |
1850 | "primary": true | |
7c673cae FG |
1851 | }, |
1852 | { | |
1853 | "size": 2048, | |
224ce89b | 1854 | "shard": 1, |
7c673cae | 1855 | "errors": [], |
b5b8bbf5 FG |
1856 | "osd": 2, |
1857 | "primary": false | |
7c673cae FG |
1858 | } |
1859 | ], | |
224ce89b | 1860 | "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
7c673cae | 1861 | "union_shard_errors": [ |
b5b8bbf5 FG |
1862 | "size_mismatch_oi", |
1863 | "obj_size_oi_mismatch" | |
7c673cae FG |
1864 | ], |
1865 | "errors": [ | |
1866 | "size_mismatch" | |
1867 | ], | |
1868 | "object": { | |
1869 | "version": 1, | |
1870 | "snap": "head", | |
1871 | "locator": "", | |
1872 | "nspace": "", | |
1873 | "name": "EOBJ1" | |
1874 | } | |
1875 | }, | |
1876 | { | |
1877 | "shards": [ | |
1878 | { | |
1879 | "size": 2048, | |
1880 | "errors": [], | |
1881 | "shard": 2, | |
b5b8bbf5 FG |
1882 | "osd": 0, |
1883 | "primary": false | |
7c673cae FG |
1884 | }, |
1885 | { | |
224ce89b | 1886 | "shard": 0, |
7c673cae FG |
1887 | "errors": [ |
1888 | "missing" | |
1889 | ], | |
b5b8bbf5 FG |
1890 | "osd": 1, |
1891 | "primary": true | |
7c673cae FG |
1892 | }, |
1893 | { | |
1894 | "size": 2048, | |
224ce89b | 1895 | "shard": 1, |
7c673cae | 1896 | "errors": [], |
b5b8bbf5 FG |
1897 | "osd": 2, |
1898 | "primary": false | |
7c673cae FG |
1899 | } |
1900 | ], | |
224ce89b | 1901 | "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
7c673cae FG |
1902 | "union_shard_errors": [ |
1903 | "missing" | |
1904 | ], | |
1905 | "errors": [], | |
1906 | "object": { | |
1907 | "version": 3, | |
1908 | "snap": "head", | |
1909 | "locator": "", | |
1910 | "nspace": "", | |
1911 | "name": "EOBJ3" | |
1912 | } | |
1913 | }, | |
1914 | { | |
1915 | "shards": [ | |
1916 | { | |
1917 | "attrs": [ | |
1918 | { | |
1919 | "Base64": true, | |
1920 | "value": "", | |
1921 | "name": "_" | |
1922 | }, | |
1923 | { | |
1924 | "Base64": false, | |
1925 | "value": "bad-val", | |
1926 | "name": "_key1-EOBJ4" | |
1927 | }, | |
1928 | { | |
1929 | "Base64": false, | |
1930 | "value": "val3-EOBJ4", | |
1931 | "name": "_key3-EOBJ4" | |
1932 | }, | |
1933 | { | |
1934 | "Base64": true, | |
1935 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
1936 | "name": "hinfo_key" | |
1937 | }, | |
1938 | { | |
1939 | "Base64": true, | |
31f18b77 | 1940 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
1941 | "name": "snapset" |
1942 | } | |
1943 | ], | |
1944 | "size": 2048, | |
1945 | "errors": [], | |
1946 | "shard": 2, | |
b5b8bbf5 FG |
1947 | "osd": 0, |
1948 | "primary": false | |
7c673cae FG |
1949 | }, |
1950 | { | |
224ce89b | 1951 | "osd": 1, |
b5b8bbf5 | 1952 | "primary": true, |
224ce89b WB |
1953 | "shard": 0, |
1954 | "errors": [], | |
1955 | "size": 2048, | |
7c673cae FG |
1956 | "attrs": [ |
1957 | { | |
1958 | "Base64": true, | |
1959 | "value": "", | |
1960 | "name": "_" | |
1961 | }, | |
1962 | { | |
1963 | "Base64": false, | |
1964 | "value": "val1-EOBJ4", | |
1965 | "name": "_key1-EOBJ4" | |
1966 | }, | |
1967 | { | |
1968 | "Base64": false, | |
1969 | "value": "val2-EOBJ4", | |
1970 | "name": "_key2-EOBJ4" | |
1971 | }, | |
1972 | { | |
1973 | "Base64": true, | |
1974 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
1975 | "name": "hinfo_key" | |
1976 | }, | |
1977 | { | |
1978 | "Base64": true, | |
31f18b77 | 1979 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
1980 | "name": "snapset" |
1981 | } | |
224ce89b | 1982 | ] |
7c673cae FG |
1983 | }, |
1984 | { | |
224ce89b | 1985 | "osd": 2, |
b5b8bbf5 | 1986 | "primary": false, |
224ce89b WB |
1987 | "shard": 1, |
1988 | "errors": [], | |
1989 | "size": 2048, | |
7c673cae FG |
1990 | "attrs": [ |
1991 | { | |
1992 | "Base64": true, | |
1993 | "value": "", | |
1994 | "name": "_" | |
1995 | }, | |
1996 | { | |
1997 | "Base64": false, | |
1998 | "value": "val1-EOBJ4", | |
1999 | "name": "_key1-EOBJ4" | |
2000 | }, | |
2001 | { | |
2002 | "Base64": false, | |
2003 | "value": "val2-EOBJ4", | |
2004 | "name": "_key2-EOBJ4" | |
2005 | }, | |
2006 | { | |
2007 | "Base64": true, | |
2008 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
2009 | "name": "hinfo_key" | |
2010 | }, | |
2011 | { | |
2012 | "Base64": true, | |
31f18b77 | 2013 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
2014 | "name": "snapset" |
2015 | } | |
224ce89b | 2016 | ] |
7c673cae FG |
2017 | } |
2018 | ], | |
224ce89b | 2019 | "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
7c673cae FG |
2020 | "union_shard_errors": [], |
2021 | "errors": [ | |
2022 | "attr_value_mismatch", | |
2023 | "attr_name_mismatch" | |
2024 | ], | |
2025 | "object": { | |
2026 | "version": 6, | |
2027 | "snap": "head", | |
2028 | "locator": "", | |
2029 | "nspace": "", | |
2030 | "name": "EOBJ4" | |
2031 | } | |
2032 | }, | |
2033 | { | |
2034 | "shards": [ | |
2035 | { | |
2036 | "size": 2048, | |
2037 | "errors": [], | |
2038 | "shard": 2, | |
b5b8bbf5 FG |
2039 | "osd": 0, |
2040 | "primary": false | |
7c673cae FG |
2041 | }, |
2042 | { | |
2043 | "size": 4096, | |
224ce89b | 2044 | "shard": 0, |
7c673cae | 2045 | "errors": [ |
b5b8bbf5 FG |
2046 | "size_mismatch_oi", |
2047 | "obj_size_oi_mismatch" | |
7c673cae | 2048 | ], |
b5b8bbf5 FG |
2049 | "osd": 1, |
2050 | "primary": true | |
7c673cae FG |
2051 | }, |
2052 | { | |
2053 | "size": 2048, | |
224ce89b | 2054 | "shard": 1, |
7c673cae | 2055 | "errors": [], |
b5b8bbf5 FG |
2056 | "osd": 2, |
2057 | "primary": false | |
7c673cae FG |
2058 | } |
2059 | ], | |
224ce89b | 2060 | "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
7c673cae | 2061 | "union_shard_errors": [ |
b5b8bbf5 FG |
2062 | "size_mismatch_oi", |
2063 | "obj_size_oi_mismatch" | |
7c673cae FG |
2064 | ], |
2065 | "errors": [ | |
2066 | "size_mismatch" | |
2067 | ], | |
2068 | "object": { | |
2069 | "version": 7, | |
2070 | "snap": "head", | |
2071 | "locator": "", | |
2072 | "nspace": "", | |
2073 | "name": "EOBJ5" | |
2074 | } | |
2075 | } | |
2076 | ], | |
2077 | "epoch": 0 | |
2078 | } | |
2079 | EOF | |
2080 | ||
2081 | jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson | |
224ce89b | 2082 | diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
7c673cae FG |
2083 | if test $getjson = "yes" |
2084 | then | |
2085 | jq '.' $dir/json > save3.json | |
2086 | fi | |
2087 | ||
2088 | if which jsonschema > /dev/null; | |
2089 | then | |
2090 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
2091 | fi | |
2092 | ||
2093 | pg_deep_scrub $pg | |
2094 | ||
2095 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
2096 | # Check pg count | |
2097 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
2098 | # Check pgid | |
2099 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
2100 | ||
2101 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
2102 | # Get epoch for repair-get requests | |
2103 | epoch=$(jq .epoch $dir/json) | |
2104 | ||
31f18b77 FG |
2105 | if [ "$allow_overwrites" = "true" ] |
2106 | then | |
2107 | jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson | |
2108 | { | |
2109 | "inconsistents": [ | |
2110 | { | |
2111 | "shards": [ | |
2112 | { | |
2113 | "data_digest": "0x00000000", | |
2114 | "omap_digest": "0xffffffff", | |
2115 | "size": 2048, | |
2116 | "errors": [], | |
2117 | "shard": 2, | |
b5b8bbf5 FG |
2118 | "osd": 0, |
2119 | "primary": false | |
31f18b77 FG |
2120 | }, |
2121 | { | |
2122 | "size": 9, | |
224ce89b | 2123 | "shard": 0, |
31f18b77 FG |
2124 | "errors": [ |
2125 | "read_error", | |
b5b8bbf5 FG |
2126 | "size_mismatch_oi", |
2127 | "obj_size_oi_mismatch" | |
31f18b77 | 2128 | ], |
b5b8bbf5 FG |
2129 | "osd": 1, |
2130 | "primary": true | |
31f18b77 FG |
2131 | }, |
2132 | { | |
2133 | "data_digest": "0x00000000", | |
2134 | "omap_digest": "0xffffffff", | |
2135 | "size": 2048, | |
224ce89b | 2136 | "shard": 1, |
31f18b77 | 2137 | "errors": [], |
b5b8bbf5 FG |
2138 | "osd": 2, |
2139 | "primary": false | |
31f18b77 FG |
2140 | } |
2141 | ], | |
224ce89b | 2142 | "selected_object_info": "3:9175b684:::EOBJ1:head(27'1 client.4155.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
31f18b77 FG |
2143 | "union_shard_errors": [ |
2144 | "read_error", | |
b5b8bbf5 FG |
2145 | "size_mismatch_oi", |
2146 | "obj_size_oi_mismatch" | |
31f18b77 FG |
2147 | ], |
2148 | "errors": [ | |
2149 | "size_mismatch" | |
2150 | ], | |
2151 | "object": { | |
2152 | "version": 1, | |
2153 | "snap": "head", | |
2154 | "locator": "", | |
2155 | "nspace": "", | |
2156 | "name": "EOBJ1" | |
2157 | } | |
2158 | }, | |
2159 | { | |
2160 | "shards": [ | |
2161 | { | |
2162 | "data_digest": "0x00000000", | |
2163 | "omap_digest": "0xffffffff", | |
2164 | "size": 2048, | |
2165 | "errors": [], | |
2166 | "shard": 2, | |
b5b8bbf5 FG |
2167 | "osd": 0, |
2168 | "primary": false | |
31f18b77 FG |
2169 | }, |
2170 | { | |
224ce89b | 2171 | "shard": 0, |
31f18b77 FG |
2172 | "errors": [ |
2173 | "missing" | |
2174 | ], | |
b5b8bbf5 FG |
2175 | "osd": 1, |
2176 | "primary": true | |
31f18b77 FG |
2177 | }, |
2178 | { | |
2179 | "data_digest": "0x00000000", | |
2180 | "omap_digest": "0xffffffff", | |
2181 | "size": 2048, | |
224ce89b | 2182 | "shard": 1, |
31f18b77 | 2183 | "errors": [], |
b5b8bbf5 FG |
2184 | "osd": 2, |
2185 | "primary": false | |
31f18b77 FG |
2186 | } |
2187 | ], | |
224ce89b | 2188 | "selected_object_info": "3:b197b25d:::EOBJ3:head(41'3 client.4199.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
31f18b77 FG |
2189 | "union_shard_errors": [ |
2190 | "missing" | |
2191 | ], | |
2192 | "errors": [], | |
2193 | "object": { | |
2194 | "version": 3, | |
2195 | "snap": "head", | |
2196 | "locator": "", | |
2197 | "nspace": "", | |
2198 | "name": "EOBJ3" | |
2199 | } | |
2200 | }, | |
2201 | { | |
2202 | "shards": [ | |
2203 | { | |
2204 | "attrs": [ | |
2205 | { | |
2206 | "Base64": true, | |
2207 | "value": "", | |
2208 | "name": "_" | |
2209 | }, | |
2210 | { | |
2211 | "Base64": false, | |
2212 | "value": "bad-val", | |
2213 | "name": "_key1-EOBJ4" | |
2214 | }, | |
2215 | { | |
2216 | "Base64": false, | |
2217 | "value": "val3-EOBJ4", | |
2218 | "name": "_key3-EOBJ4" | |
2219 | }, | |
2220 | { | |
2221 | "Base64": true, | |
2222 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
2223 | "name": "hinfo_key" | |
2224 | }, | |
2225 | { | |
2226 | "Base64": true, | |
2227 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", | |
2228 | "name": "snapset" | |
2229 | } | |
2230 | ], | |
2231 | "data_digest": "0x00000000", | |
2232 | "omap_digest": "0xffffffff", | |
2233 | "size": 2048, | |
2234 | "errors": [], | |
2235 | "shard": 2, | |
b5b8bbf5 FG |
2236 | "osd": 0, |
2237 | "primary": false | |
31f18b77 FG |
2238 | }, |
2239 | { | |
2240 | "attrs": [ | |
2241 | { | |
2242 | "Base64": true, | |
2243 | "value": "", | |
2244 | "name": "_" | |
2245 | }, | |
2246 | { | |
2247 | "Base64": false, | |
2248 | "value": "val1-EOBJ4", | |
2249 | "name": "_key1-EOBJ4" | |
2250 | }, | |
2251 | { | |
2252 | "Base64": false, | |
2253 | "value": "val2-EOBJ4", | |
2254 | "name": "_key2-EOBJ4" | |
2255 | }, | |
2256 | { | |
2257 | "Base64": true, | |
2258 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
2259 | "name": "hinfo_key" | |
2260 | }, | |
2261 | { | |
2262 | "Base64": true, | |
2263 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", | |
2264 | "name": "snapset" | |
2265 | } | |
2266 | ], | |
2267 | "data_digest": "0x00000000", | |
2268 | "omap_digest": "0xffffffff", | |
2269 | "size": 2048, | |
2270 | "errors": [], | |
224ce89b | 2271 | "shard": 0, |
b5b8bbf5 FG |
2272 | "osd": 1, |
2273 | "primary": true | |
31f18b77 FG |
2274 | }, |
2275 | { | |
2276 | "attrs": [ | |
2277 | { | |
2278 | "Base64": true, | |
2279 | "value": "", | |
2280 | "name": "_" | |
2281 | }, | |
2282 | { | |
2283 | "Base64": false, | |
2284 | "value": "val1-EOBJ4", | |
2285 | "name": "_key1-EOBJ4" | |
2286 | }, | |
2287 | { | |
2288 | "Base64": false, | |
2289 | "value": "val2-EOBJ4", | |
2290 | "name": "_key2-EOBJ4" | |
2291 | }, | |
2292 | { | |
2293 | "Base64": true, | |
2294 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
2295 | "name": "hinfo_key" | |
2296 | }, | |
2297 | { | |
2298 | "Base64": true, | |
2299 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", | |
2300 | "name": "snapset" | |
2301 | } | |
2302 | ], | |
2303 | "data_digest": "0x00000000", | |
2304 | "omap_digest": "0xffffffff", | |
2305 | "size": 2048, | |
2306 | "errors": [], | |
224ce89b | 2307 | "shard": 1, |
b5b8bbf5 FG |
2308 | "osd": 2, |
2309 | "primary": false | |
31f18b77 FG |
2310 | } |
2311 | ], | |
224ce89b | 2312 | "selected_object_info": "3:5e723e06:::EOBJ4:head(48'6 client.4223.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
31f18b77 FG |
2313 | "union_shard_errors": [], |
2314 | "errors": [ | |
2315 | "attr_value_mismatch", | |
2316 | "attr_name_mismatch" | |
2317 | ], | |
2318 | "object": { | |
2319 | "version": 6, | |
2320 | "snap": "head", | |
2321 | "locator": "", | |
2322 | "nspace": "", | |
2323 | "name": "EOBJ4" | |
2324 | } | |
2325 | }, | |
2326 | { | |
2327 | "shards": [ | |
2328 | { | |
2329 | "data_digest": "0x00000000", | |
2330 | "omap_digest": "0xffffffff", | |
2331 | "size": 2048, | |
2332 | "errors": [], | |
2333 | "shard": 2, | |
b5b8bbf5 FG |
2334 | "osd": 0, |
2335 | "primary": false | |
31f18b77 FG |
2336 | }, |
2337 | { | |
2338 | "data_digest": "0x00000000", | |
2339 | "omap_digest": "0xffffffff", | |
2340 | "size": 4096, | |
2341 | "errors": [ | |
b5b8bbf5 FG |
2342 | "size_mismatch_oi", |
2343 | "obj_size_oi_mismatch" | |
31f18b77 | 2344 | ], |
224ce89b | 2345 | "shard": 0, |
b5b8bbf5 FG |
2346 | "osd": 1, |
2347 | "primary": true | |
31f18b77 FG |
2348 | }, |
2349 | { | |
2350 | "data_digest": "0x00000000", | |
2351 | "omap_digest": "0xffffffff", | |
2352 | "size": 2048, | |
2353 | "errors": [], | |
224ce89b | 2354 | "shard": 1, |
b5b8bbf5 FG |
2355 | "osd": 2, |
2356 | "primary": false | |
31f18b77 FG |
2357 | } |
2358 | ], | |
224ce89b | 2359 | "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4288.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
31f18b77 | 2360 | "union_shard_errors": [ |
b5b8bbf5 FG |
2361 | "size_mismatch_oi", |
2362 | "obj_size_oi_mismatch" | |
31f18b77 FG |
2363 | ], |
2364 | "errors": [ | |
2365 | "size_mismatch" | |
2366 | ], | |
2367 | "object": { | |
2368 | "version": 7, | |
2369 | "snap": "head", | |
2370 | "locator": "", | |
2371 | "nspace": "", | |
2372 | "name": "EOBJ5" | |
2373 | } | |
2374 | } | |
2375 | ], | |
2376 | "epoch": 0 | |
2377 | } | |
2378 | EOF | |
2379 | ||
2380 | else | |
2381 | ||
2382 | jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson | |
7c673cae FG |
2383 | { |
2384 | "inconsistents": [ | |
2385 | { | |
2386 | "shards": [ | |
2387 | { | |
2388 | "data_digest": "0x04cfa72f", | |
2389 | "omap_digest": "0xffffffff", | |
2390 | "size": 2048, | |
2391 | "errors": [], | |
2392 | "shard": 2, | |
b5b8bbf5 FG |
2393 | "osd": 0, |
2394 | "primary": false | |
7c673cae FG |
2395 | }, |
2396 | { | |
2397 | "size": 9, | |
224ce89b | 2398 | "shard": 0, |
7c673cae FG |
2399 | "errors": [ |
2400 | "read_error", | |
b5b8bbf5 FG |
2401 | "size_mismatch_oi", |
2402 | "obj_size_oi_mismatch" | |
7c673cae | 2403 | ], |
b5b8bbf5 FG |
2404 | "osd": 1, |
2405 | "primary": true | |
7c673cae FG |
2406 | }, |
2407 | { | |
2408 | "data_digest": "0x04cfa72f", | |
2409 | "omap_digest": "0xffffffff", | |
2410 | "size": 2048, | |
224ce89b | 2411 | "shard": 1, |
7c673cae | 2412 | "errors": [], |
b5b8bbf5 FG |
2413 | "osd": 2, |
2414 | "primary": false | |
7c673cae FG |
2415 | } |
2416 | ], | |
224ce89b | 2417 | "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
7c673cae FG |
2418 | "union_shard_errors": [ |
2419 | "read_error", | |
b5b8bbf5 FG |
2420 | "size_mismatch_oi", |
2421 | "obj_size_oi_mismatch" | |
7c673cae FG |
2422 | ], |
2423 | "errors": [ | |
2424 | "size_mismatch" | |
2425 | ], | |
2426 | "object": { | |
2427 | "version": 1, | |
2428 | "snap": "head", | |
2429 | "locator": "", | |
2430 | "nspace": "", | |
2431 | "name": "EOBJ1" | |
2432 | } | |
2433 | }, | |
2434 | { | |
2435 | "shards": [ | |
2436 | { | |
2437 | "size": 2048, | |
2438 | "errors": [ | |
2439 | "ec_hash_error" | |
2440 | ], | |
2441 | "shard": 2, | |
b5b8bbf5 FG |
2442 | "osd": 0, |
2443 | "primary": false | |
7c673cae FG |
2444 | }, |
2445 | { | |
2446 | "data_digest": "0x04cfa72f", | |
2447 | "omap_digest": "0xffffffff", | |
2448 | "size": 2048, | |
2449 | "errors": [], | |
224ce89b | 2450 | "shard": 0, |
b5b8bbf5 FG |
2451 | "osd": 1, |
2452 | "primary": true | |
7c673cae FG |
2453 | }, |
2454 | { | |
2455 | "data_digest": "0x04cfa72f", | |
2456 | "omap_digest": "0xffffffff", | |
2457 | "size": 2048, | |
2458 | "errors": [], | |
224ce89b | 2459 | "shard": 1, |
b5b8bbf5 FG |
2460 | "osd": 2, |
2461 | "primary": false | |
7c673cae FG |
2462 | } |
2463 | ], | |
224ce89b | 2464 | "selected_object_info": "3:9babd184:::EOBJ2:head(29'2 client.4217.0:1 dirty|data_digest|omap_digest s 7 uv 2 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
7c673cae FG |
2465 | "union_shard_errors": [ |
2466 | "ec_hash_error" | |
2467 | ], | |
2468 | "errors": [], | |
2469 | "object": { | |
2470 | "version": 2, | |
2471 | "snap": "head", | |
2472 | "locator": "", | |
2473 | "nspace": "", | |
2474 | "name": "EOBJ2" | |
2475 | } | |
2476 | }, | |
2477 | { | |
2478 | "shards": [ | |
2479 | { | |
2480 | "data_digest": "0x04cfa72f", | |
2481 | "omap_digest": "0xffffffff", | |
2482 | "size": 2048, | |
2483 | "errors": [], | |
2484 | "shard": 2, | |
b5b8bbf5 FG |
2485 | "osd": 0, |
2486 | "primary": false | |
7c673cae FG |
2487 | }, |
2488 | { | |
224ce89b | 2489 | "osd": 1, |
b5b8bbf5 | 2490 | "primary": true, |
224ce89b | 2491 | "shard": 0, |
7c673cae FG |
2492 | "errors": [ |
2493 | "missing" | |
224ce89b | 2494 | ] |
7c673cae FG |
2495 | }, |
2496 | { | |
2497 | "data_digest": "0x04cfa72f", | |
2498 | "omap_digest": "0xffffffff", | |
2499 | "size": 2048, | |
224ce89b | 2500 | "shard": 1, |
7c673cae | 2501 | "errors": [], |
b5b8bbf5 FG |
2502 | "osd": 2, |
2503 | "primary": false | |
7c673cae FG |
2504 | } |
2505 | ], | |
224ce89b | 2506 | "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
7c673cae FG |
2507 | "union_shard_errors": [ |
2508 | "missing" | |
2509 | ], | |
2510 | "errors": [], | |
2511 | "object": { | |
2512 | "version": 3, | |
2513 | "snap": "head", | |
2514 | "locator": "", | |
2515 | "nspace": "", | |
2516 | "name": "EOBJ3" | |
2517 | } | |
2518 | }, | |
2519 | { | |
2520 | "shards": [ | |
2521 | { | |
2522 | "attrs": [ | |
2523 | { | |
2524 | "Base64": true, | |
2525 | "value": "", | |
2526 | "name": "_" | |
2527 | }, | |
2528 | { | |
2529 | "Base64": false, | |
2530 | "value": "bad-val", | |
2531 | "name": "_key1-EOBJ4" | |
2532 | }, | |
2533 | { | |
2534 | "Base64": false, | |
2535 | "value": "val3-EOBJ4", | |
2536 | "name": "_key3-EOBJ4" | |
2537 | }, | |
2538 | { | |
2539 | "Base64": true, | |
2540 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
2541 | "name": "hinfo_key" | |
2542 | }, | |
2543 | { | |
2544 | "Base64": true, | |
31f18b77 | 2545 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
2546 | "name": "snapset" |
2547 | } | |
2548 | ], | |
2549 | "data_digest": "0x04cfa72f", | |
2550 | "omap_digest": "0xffffffff", | |
2551 | "size": 2048, | |
2552 | "errors": [], | |
2553 | "shard": 2, | |
b5b8bbf5 FG |
2554 | "osd": 0, |
2555 | "primary": false | |
7c673cae FG |
2556 | }, |
2557 | { | |
224ce89b | 2558 | "osd": 1, |
b5b8bbf5 | 2559 | "primary": true, |
224ce89b WB |
2560 | "shard": 0, |
2561 | "errors": [], | |
2562 | "size": 2048, | |
2563 | "omap_digest": "0xffffffff", | |
2564 | "data_digest": "0x04cfa72f", | |
7c673cae FG |
2565 | "attrs": [ |
2566 | { | |
2567 | "Base64": true, | |
2568 | "value": "", | |
2569 | "name": "_" | |
2570 | }, | |
2571 | { | |
2572 | "Base64": false, | |
2573 | "value": "val1-EOBJ4", | |
2574 | "name": "_key1-EOBJ4" | |
2575 | }, | |
2576 | { | |
2577 | "Base64": false, | |
2578 | "value": "val2-EOBJ4", | |
2579 | "name": "_key2-EOBJ4" | |
2580 | }, | |
2581 | { | |
2582 | "Base64": true, | |
2583 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
2584 | "name": "hinfo_key" | |
2585 | }, | |
2586 | { | |
2587 | "Base64": true, | |
31f18b77 | 2588 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
2589 | "name": "snapset" |
2590 | } | |
224ce89b | 2591 | ] |
7c673cae FG |
2592 | }, |
2593 | { | |
224ce89b | 2594 | "osd": 2, |
b5b8bbf5 | 2595 | "primary": false, |
224ce89b WB |
2596 | "shard": 1, |
2597 | "errors": [], | |
2598 | "size": 2048, | |
2599 | "omap_digest": "0xffffffff", | |
2600 | "data_digest": "0x04cfa72f", | |
7c673cae FG |
2601 | "attrs": [ |
2602 | { | |
2603 | "Base64": true, | |
2604 | "value": "", | |
2605 | "name": "_" | |
2606 | }, | |
2607 | { | |
2608 | "Base64": false, | |
2609 | "value": "val1-EOBJ4", | |
2610 | "name": "_key1-EOBJ4" | |
2611 | }, | |
2612 | { | |
2613 | "Base64": false, | |
2614 | "value": "val2-EOBJ4", | |
2615 | "name": "_key2-EOBJ4" | |
2616 | }, | |
2617 | { | |
2618 | "Base64": true, | |
2619 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
2620 | "name": "hinfo_key" | |
2621 | }, | |
2622 | { | |
2623 | "Base64": true, | |
31f18b77 | 2624 | "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", |
7c673cae FG |
2625 | "name": "snapset" |
2626 | } | |
224ce89b | 2627 | ] |
7c673cae FG |
2628 | } |
2629 | ], | |
224ce89b | 2630 | "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
7c673cae FG |
2631 | "union_shard_errors": [], |
2632 | "errors": [ | |
2633 | "attr_value_mismatch", | |
2634 | "attr_name_mismatch" | |
2635 | ], | |
2636 | "object": { | |
2637 | "version": 6, | |
2638 | "snap": "head", | |
2639 | "locator": "", | |
2640 | "nspace": "", | |
2641 | "name": "EOBJ4" | |
2642 | } | |
2643 | }, | |
2644 | { | |
2645 | "shards": [ | |
2646 | { | |
2647 | "data_digest": "0x04cfa72f", | |
2648 | "omap_digest": "0xffffffff", | |
2649 | "size": 2048, | |
2650 | "errors": [], | |
2651 | "shard": 2, | |
b5b8bbf5 FG |
2652 | "osd": 0, |
2653 | "primary": false | |
7c673cae FG |
2654 | }, |
2655 | { | |
2656 | "size": 4096, | |
224ce89b | 2657 | "shard": 0, |
7c673cae FG |
2658 | "errors": [ |
2659 | "size_mismatch_oi", | |
b5b8bbf5 FG |
2660 | "ec_size_error", |
2661 | "obj_size_oi_mismatch" | |
7c673cae | 2662 | ], |
b5b8bbf5 FG |
2663 | "osd": 1, |
2664 | "primary": true | |
7c673cae FG |
2665 | }, |
2666 | { | |
2667 | "data_digest": "0x04cfa72f", | |
2668 | "omap_digest": "0xffffffff", | |
2669 | "size": 2048, | |
224ce89b | 2670 | "shard": 1, |
7c673cae | 2671 | "errors": [], |
b5b8bbf5 FG |
2672 | "osd": 2, |
2673 | "primary": false | |
7c673cae FG |
2674 | } |
2675 | ], | |
224ce89b | 2676 | "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", |
7c673cae FG |
2677 | "union_shard_errors": [ |
2678 | "size_mismatch_oi", | |
b5b8bbf5 FG |
2679 | "ec_size_error", |
2680 | "obj_size_oi_mismatch" | |
7c673cae FG |
2681 | ], |
2682 | "errors": [ | |
2683 | "size_mismatch" | |
2684 | ], | |
2685 | "object": { | |
2686 | "version": 7, | |
2687 | "snap": "head", | |
2688 | "locator": "", | |
2689 | "nspace": "", | |
2690 | "name": "EOBJ5" | |
2691 | } | |
2692 | } | |
2693 | ], | |
2694 | "epoch": 0 | |
2695 | } | |
2696 | EOF | |
2697 | ||
7c673cae | 2698 | fi |
31f18b77 FG |
2699 | |
2700 | jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson | |
224ce89b | 2701 | diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
7c673cae FG |
2702 | if test $getjson = "yes" |
2703 | then | |
31f18b77 FG |
2704 | if [ "$allow_overwrites" = "true" ] |
2705 | then | |
2706 | num=4 | |
2707 | else | |
2708 | num=5 | |
2709 | fi | |
2710 | jq '.' $dir/json > save${num}.json | |
7c673cae FG |
2711 | fi |
2712 | ||
2713 | if which jsonschema > /dev/null; | |
2714 | then | |
2715 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
2716 | fi | |
2717 | ||
2718 | rados rmpool $poolname $poolname --yes-i-really-really-mean-it | |
2719 | teardown $dir || return 1 | |
2720 | } | |
2721 | ||
2722 | function TEST_corrupt_scrub_erasure_appends() { | |
2723 | corrupt_scrub_erasure $1 false | |
2724 | } | |
2725 | ||
2726 | function TEST_corrupt_scrub_erasure_overwrites() { | |
224ce89b WB |
2727 | if [ "$use_ec_overwrite" = "true" ]; then |
2728 | corrupt_scrub_erasure $1 true | |
2729 | fi | |
7c673cae FG |
2730 | } |
2731 | ||
2732 | # | |
2733 | # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost | |
2734 | # | |
2735 | function TEST_periodic_scrub_replicated() { | |
2736 | local dir=$1 | |
2737 | local poolname=psr_pool | |
2738 | local objname=POBJ | |
2739 | ||
2740 | setup $dir || return 1 | |
2741 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
2742 | run_mgr $dir x || return 1 | |
b5b8bbf5 FG |
2743 | local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 " |
2744 | ceph_osd_args+="--osd_scrub_backoff_ratio=0" | |
7c673cae FG |
2745 | run_osd $dir 0 $ceph_osd_args || return 1 |
2746 | run_osd $dir 1 $ceph_osd_args || return 1 | |
c07f9fc5 | 2747 | create_rbd_pool || return 1 |
7c673cae FG |
2748 | wait_for_clean || return 1 |
2749 | ||
b5b8bbf5 | 2750 | create_pool $poolname 1 1 || return 1 |
7c673cae FG |
2751 | wait_for_clean || return 1 |
2752 | ||
2753 | local osd=0 | |
2754 | add_something $dir $poolname $objname scrub || return 1 | |
2755 | local primary=$(get_primary $poolname $objname) | |
2756 | local pg=$(get_pg $poolname $objname) | |
2757 | ||
2758 | # Add deep-scrub only error | |
2759 | local payload=UVWXYZ | |
2760 | echo $payload > $dir/CORRUPT | |
2761 | # Uses $ceph_osd_args for osd restart | |
2762 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
2763 | ||
2764 | # No scrub information available, so expect failure | |
2765 | set -o pipefail | |
2766 | ! rados list-inconsistent-obj $pg | jq '.' || return 1 | |
2767 | set +o pipefail | |
2768 | ||
2769 | pg_deep_scrub $pg || return 1 | |
2770 | ||
2771 | # Make sure bad object found | |
2772 | rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 | |
2773 | ||
b5b8bbf5 | 2774 | flush_pg_stats |
7c673cae FG |
2775 | local last_scrub=$(get_last_scrub_stamp $pg) |
2776 | # Fake a schedule scrub | |
c07f9fc5 | 2777 | CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \ |
7c673cae FG |
2778 | trigger_scrub $pg || return 1 |
2779 | # Wait for schedule regular scrub | |
2780 | wait_for_scrub $pg "$last_scrub" | |
2781 | ||
2782 | # It needed to be upgraded | |
2783 | grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1 | |
2784 | ||
2785 | # Bad object still known | |
2786 | rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 | |
2787 | ||
2788 | # Can't upgrade with this set | |
2789 | ceph osd set nodeep-scrub | |
224ce89b | 2790 | # Let map change propagate to OSDs |
b5b8bbf5 FG |
2791 | flush pg_stats |
2792 | sleep 5 | |
7c673cae FG |
2793 | |
2794 | # Fake a schedule scrub | |
c07f9fc5 | 2795 | CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \ |
7c673cae FG |
2796 | trigger_scrub $pg || return 1 |
2797 | # Wait for schedule regular scrub | |
2798 | # to notice scrub and skip it | |
2799 | local found=false | |
2800 | for i in $(seq 14 -1 0) | |
2801 | do | |
2802 | sleep 1 | |
2803 | ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; } | |
2804 | echo Time left: $i seconds | |
2805 | done | |
2806 | test $found = "true" || return 1 | |
2807 | ||
2808 | # Bad object still known | |
2809 | rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 | |
2810 | ||
b5b8bbf5 | 2811 | flush_pg_stats |
7c673cae | 2812 | # Request a regular scrub and it will be done |
7c673cae | 2813 | pg_scrub $pg |
c07f9fc5 | 2814 | grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1 |
7c673cae FG |
2815 | |
2816 | # deep-scrub error is no longer present | |
2817 | rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1 | |
2818 | } | |
2819 | ||
2820 | ||
2821 | main osd-scrub-repair "$@" | |
2822 | ||
2823 | # Local Variables: | |
2824 | # compile-command: "cd ../.. ; make -j4 && \ | |
2825 | # test/osd/osd-scrub-repair.sh # TEST_corrupt_and_repair_replicated" | |
2826 | # End: |