]> git.proxmox.com Git - ceph.git/blob - ceph/qa/standalone/scrub/osd-scrub-repair.sh
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
update sources to ceph Nautilus 14.2.1
[ceph.git] / ceph / qa / standalone / scrub / osd-scrub-repair.sh
1 #!/usr/bin/env bash
2 #
3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
4 #
5 # Author: Loic Dachary <loic@dachary.org>
6 #
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU Library Public License as published by
9 # the Free Software Foundation; either version 2, or (at your option)
10 # any later version.
11 #
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Library Public License for more details.
16 #
17 set -x
18 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
19
20 if [ `uname` = FreeBSD ]; then
21 # erasure coding overwrites are only tested on Bluestore
22 # erasure coding on filestore is unsafe
23 # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites
24 use_ec_overwrite=false
25 else
26 use_ec_overwrite=true
27 fi
28
29 # Test development and debugging
30 # Set to "yes" in order to ignore diff errors and save results to update test
31 getjson="no"
32
33 # Filter out mtime and local_mtime dates, version, prior_version and last_reqid (client) from any object_info.
34 jqfilter='def walk(f):
35 . as $in
36 | if type == "object" then
37 reduce keys[] as $key
38 ( {}; . + { ($key): ($in[$key] | walk(f)) } ) | f
39 elif type == "array" then map( walk(f) ) | f
40 else f
41 end;
42 walk(if type == "object" then del(.mtime) else . end)
43 | walk(if type == "object" then del(.local_mtime) else . end)
44 | walk(if type == "object" then del(.last_reqid) else . end)
45 | walk(if type == "object" then del(.version) else . end)
46 | walk(if type == "object" then del(.prior_version) else . end)'
47
48 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
49
50 function run() {
51 local dir=$1
52 shift
53
54 export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
55 export CEPH_ARGS
56 CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
57 CEPH_ARGS+="--mon-host=$CEPH_MON "
58 CEPH_ARGS+="--osd-skip-data-digest=false "
59 CEPH_ARGS+="--osd-objectstore=filestore "
60
61 export -n CEPH_CLI_TEST_DUP_COMMAND
62 local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
63 for func in $funcs ; do
64 $func $dir || return 1
65 done
66 }
67
68 function add_something() {
69 local dir=$1
70 local poolname=$2
71 local obj=${3:-SOMETHING}
72 local scrub=${4:-noscrub}
73
74 if [ "$scrub" = "noscrub" ];
75 then
76 ceph osd set noscrub || return 1
77 ceph osd set nodeep-scrub || return 1
78 else
79 ceph osd unset noscrub || return 1
80 ceph osd unset nodeep-scrub || return 1
81 fi
82
83 local payload=ABCDEF
84 echo $payload > $dir/ORIGINAL
85 rados --pool $poolname put $obj $dir/ORIGINAL || return 1
86 }
87
88 #
89 # Corrupt one copy of a replicated pool
90 #
91 function TEST_corrupt_and_repair_replicated() {
92 local dir=$1
93 local poolname=rbd
94
95 setup $dir || return 1
96 run_mon $dir a --osd_pool_default_size=2 || return 1
97 run_mgr $dir x || return 1
98 run_osd $dir 0 || return 1
99 run_osd $dir 1 || return 1
100 create_rbd_pool || return 1
101 wait_for_clean || return 1
102
103 add_something $dir $poolname || return 1
104 corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
105 # Reproduces http://tracker.ceph.com/issues/8914
106 corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1
107
108 teardown $dir || return 1
109 }
110
111 function corrupt_and_repair_two() {
112 local dir=$1
113 local poolname=$2
114 local first=$3
115 local second=$4
116
117 #
118 # 1) remove the corresponding file from the OSDs
119 #
120 pids=""
121 run_in_background pids objectstore_tool $dir $first SOMETHING remove
122 run_in_background pids objectstore_tool $dir $second SOMETHING remove
123 wait_background pids
124 return_code=$?
125 if [ $return_code -ne 0 ]; then return $return_code; fi
126
127 #
128 # 2) repair the PG
129 #
130 local pg=$(get_pg $poolname SOMETHING)
131 repair $pg
132 #
133 # 3) The files must be back
134 #
135 pids=""
136 run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs
137 run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs
138 wait_background pids
139 return_code=$?
140 if [ $return_code -ne 0 ]; then return $return_code; fi
141
142 rados --pool $poolname get SOMETHING $dir/COPY || return 1
143 diff $dir/ORIGINAL $dir/COPY || return 1
144 }
145
146 #
147 # 1) add an object
148 # 2) remove the corresponding file from a designated OSD
149 # 3) repair the PG
150 # 4) check that the file has been restored in the designated OSD
151 #
152 function corrupt_and_repair_one() {
153 local dir=$1
154 local poolname=$2
155 local osd=$3
156
157 #
158 # 1) remove the corresponding file from the OSD
159 #
160 objectstore_tool $dir $osd SOMETHING remove || return 1
161 #
162 # 2) repair the PG
163 #
164 local pg=$(get_pg $poolname SOMETHING)
165 repair $pg
166 #
167 # 3) The file must be back
168 #
169 objectstore_tool $dir $osd SOMETHING list-attrs || return 1
170 rados --pool $poolname get SOMETHING $dir/COPY || return 1
171 diff $dir/ORIGINAL $dir/COPY || return 1
172 }
173
174 function corrupt_and_repair_erasure_coded() {
175 local dir=$1
176 local poolname=$2
177
178 add_something $dir $poolname || return 1
179
180 local primary=$(get_primary $poolname SOMETHING)
181 local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
182 local not_primary_first=${osds[0]}
183 local not_primary_second=${osds[1]}
184
185 # Reproduces http://tracker.ceph.com/issues/10017
186 corrupt_and_repair_one $dir $poolname $primary || return 1
187 # Reproduces http://tracker.ceph.com/issues/10409
188 corrupt_and_repair_one $dir $poolname $not_primary_first || return 1
189 corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1
190 corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1
191
192 }
193
194 function create_ec_pool() {
195 local pool_name=$1
196 shift
197 local allow_overwrites=$1
198 shift
199
200 ceph osd erasure-code-profile set myprofile crush-failure-domain=osd "$@" || return 1
201
202 create_pool "$poolname" 1 1 erasure myprofile || return 1
203
204 if [ "$allow_overwrites" = "true" ]; then
205 ceph osd pool set "$poolname" allow_ec_overwrites true || return 1
206 fi
207
208 wait_for_clean || return 1
209 return 0
210 }
211
212 function auto_repair_erasure_coded() {
213 local dir=$1
214 local allow_overwrites=$2
215 local poolname=ecpool
216
217 # Launch a cluster with 5 seconds scrub interval
218 setup $dir || return 1
219 run_mon $dir a || return 1
220 run_mgr $dir x || return 1
221 local ceph_osd_args="--osd-scrub-auto-repair=true \
222 --osd-deep-scrub-interval=5 \
223 --osd-scrub-max-interval=5 \
224 --osd-scrub-min-interval=5 \
225 --osd-scrub-interval-randomize-ratio=0"
226 for id in $(seq 0 2) ; do
227 if [ "$allow_overwrites" = "true" ]; then
228 run_osd_bluestore $dir $id $ceph_osd_args || return 1
229 else
230 run_osd $dir $id $ceph_osd_args || return 1
231 fi
232 done
233 create_rbd_pool || return 1
234 wait_for_clean || return 1
235
236 # Create an EC pool
237 create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
238
239 # Put an object
240 local payload=ABCDEF
241 echo $payload > $dir/ORIGINAL
242 rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
243
244 # Remove the object from one shard physically
245 # Restarted osd get $ceph_osd_args passed
246 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
247 # Wait for auto repair
248 local pgid=$(get_pg $poolname SOMETHING)
249 wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)"
250 wait_for_clean || return 1
251 # Verify - the file should be back
252 # Restarted osd get $ceph_osd_args passed
253 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
254 rados --pool $poolname get SOMETHING $dir/COPY || return 1
255 diff $dir/ORIGINAL $dir/COPY || return 1
256
257 # Tear down
258 teardown $dir || return 1
259 }
260
261 function TEST_auto_repair_erasure_coded_appends() {
262 auto_repair_erasure_coded $1 false
263 }
264
265 function TEST_auto_repair_erasure_coded_overwrites() {
266 if [ "$use_ec_overwrite" = "true" ]; then
267 auto_repair_erasure_coded $1 true
268 fi
269 }
270
271 function TEST_auto_repair_bluestore_basic() {
272 local dir=$1
273 local poolname=testpool
274
275 # Launch a cluster with 5 seconds scrub interval
276 setup $dir || return 1
277 run_mon $dir a || return 1
278 run_mgr $dir x || return 1
279 local ceph_osd_args="--osd-scrub-auto-repair=true \
280 --osd_deep_scrub_randomize_ratio=0 \
281 --osd-scrub-interval-randomize-ratio=0"
282 for id in $(seq 0 2) ; do
283 run_osd_bluestore $dir $id $ceph_osd_args || return 1
284 done
285
286 create_pool $poolname 1 1 || return 1
287 ceph osd pool set $poolname size 2
288 wait_for_clean || return 1
289
290 # Put an object
291 local payload=ABCDEF
292 echo $payload > $dir/ORIGINAL
293 rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
294
295 # Remove the object from one shard physically
296 # Restarted osd get $ceph_osd_args passed
297 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
298
299 local pgid=$(get_pg $poolname SOMETHING)
300 local primary=$(get_primary $poolname SOMETHING)
301 local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
302 CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_deep_scrub $pgid
303 CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_scrub $pgid
304
305 # Wait for auto repair
306 wait_for_scrub $pgid "$last_scrub_stamp" || return 1
307 wait_for_clean || return 1
308 ceph pg dump pgs
309 # Verify - the file should be back
310 # Restarted osd get $ceph_osd_args passed
311 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
312 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING get-bytes $dir/COPY || return 1
313 diff $dir/ORIGINAL $dir/COPY || return 1
314 grep scrub_finish $dir/osd.${primary}.log
315
316 # Tear down
317 teardown $dir || return 1
318 }
319
320 function TEST_auto_repair_bluestore_scrub() {
321 local dir=$1
322 local poolname=testpool
323
324 # Launch a cluster with 5 seconds scrub interval
325 setup $dir || return 1
326 run_mon $dir a || return 1
327 run_mgr $dir x || return 1
328 local ceph_osd_args="--osd-scrub-auto-repair=true \
329 --osd_deep_scrub_randomize_ratio=0 \
330 --osd-scrub-interval-randomize-ratio=0"
331 for id in $(seq 0 2) ; do
332 run_osd_bluestore $dir $id $ceph_osd_args || return 1
333 done
334
335 create_pool $poolname 1 1 || return 1
336 ceph osd pool set $poolname size 2
337 wait_for_clean || return 1
338
339 # Put an object
340 local payload=ABCDEF
341 echo $payload > $dir/ORIGINAL
342 rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
343
344 # Remove the object from one shard physically
345 # Restarted osd get $ceph_osd_args passed
346 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
347
348 local pgid=$(get_pg $poolname SOMETHING)
349 local primary=$(get_primary $poolname SOMETHING)
350 local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
351 CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_scrub $pgid
352
353 # Wait for scrub -> auto repair
354 wait_for_scrub $pgid "$last_scrub_stamp" || return 1
355 ceph pg dump pgs
356 # Actually this causes 2 scrubs, so we better wait a little longer
357 sleep 5
358 wait_for_clean || return 1
359 ceph pg dump pgs
360 # Verify - the file should be back
361 # Restarted osd get $ceph_osd_args passed
362 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
363 rados --pool $poolname get SOMETHING $dir/COPY || return 1
364 diff $dir/ORIGINAL $dir/COPY || return 1
365 grep scrub_finish $dir/osd.${primary}.log
366
367 # This should have caused 1 object to be repaired
368 COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
369 test "$COUNT" = "1" || return 1
370
371 # Tear down
372 teardown $dir || return 1
373 }
374
375 function TEST_auto_repair_bluestore_failed() {
376 local dir=$1
377 local poolname=testpool
378
379 # Launch a cluster with 5 seconds scrub interval
380 setup $dir || return 1
381 run_mon $dir a || return 1
382 run_mgr $dir x || return 1
383 local ceph_osd_args="--osd-scrub-auto-repair=true \
384 --osd_deep_scrub_randomize_ratio=0 \
385 --osd-scrub-interval-randomize-ratio=0"
386 for id in $(seq 0 2) ; do
387 run_osd_bluestore $dir $id $ceph_osd_args || return 1
388 done
389
390 create_pool $poolname 1 1 || return 1
391 ceph osd pool set $poolname size 2
392 wait_for_clean || return 1
393
394 # Put an object
395 local payload=ABCDEF
396 echo $payload > $dir/ORIGINAL
397 for i in $(seq 1 10)
398 do
399 rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
400 done
401
402 # Remove the object from one shard physically
403 # Restarted osd get $ceph_osd_args passed
404 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj1 remove || return 1
405 # obj2 can't be repaired
406 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj2 remove || return 1
407 objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 rm-attr _ || return 1
408
409 local pgid=$(get_pg $poolname obj1)
410 local primary=$(get_primary $poolname obj1)
411 local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
412 CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_deep_scrub $pgid
413 CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_scrub $pgid
414
415 # Wait for auto repair
416 wait_for_scrub $pgid "$last_scrub_stamp" || return 1
417 wait_for_clean || return 1
418 flush_pg_stats
419 grep scrub_finish $dir/osd.${primary}.log
420 grep -q "scrub_finish.*still present after re-scrub" $dir/osd.${primary}.log || return 1
421 ceph pg dump pgs
422 ceph pg dump pgs | grep -q "^$(pgid).*+failed_repair" || return 1
423
424 # Verify - obj1 should be back
425 # Restarted osd get $ceph_osd_args passed
426 objectstore_tool $dir $(get_not_primary $poolname obj1) obj1 list-attrs || return 1
427 rados --pool $poolname get obj1 $dir/COPY || return 1
428 diff $dir/ORIGINAL $dir/COPY || return 1
429 grep scrub_finish $dir/osd.${primary}.log
430
431 # Make it repairable
432 objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 remove || return 1
433 repair $pgid
434 sleep 2
435
436 ceph pg dump pgs
437 ceph pg dump pgs | grep -q "^$(pgid).* active+clean " || return 1
438 grep scrub_finish $dir/osd.${primary}.log
439
440 # Tear down
441 teardown $dir || return 1
442 }
443
444 function TEST_auto_repair_bluestore_failed_norecov() {
445 local dir=$1
446 local poolname=testpool
447
448 # Launch a cluster with 5 seconds scrub interval
449 setup $dir || return 1
450 run_mon $dir a || return 1
451 run_mgr $dir x || return 1
452 local ceph_osd_args="--osd-scrub-auto-repair=true \
453 --osd_deep_scrub_randomize_ratio=0 \
454 --osd-scrub-interval-randomize-ratio=0"
455 for id in $(seq 0 2) ; do
456 run_osd_bluestore $dir $id $ceph_osd_args || return 1
457 done
458
459 create_pool $poolname 1 1 || return 1
460 ceph osd pool set $poolname size 2
461 wait_for_clean || return 1
462
463 # Put an object
464 local payload=ABCDEF
465 echo $payload > $dir/ORIGINAL
466 for i in $(seq 1 10)
467 do
468 rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
469 done
470
471 # Remove the object from one shard physically
472 # Restarted osd get $ceph_osd_args passed
473 # obj1 can't be repaired
474 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj1 remove || return 1
475 objectstore_tool $dir $(get_primary $poolname SOMETHING) obj1 rm-attr _ || return 1
476 # obj2 can't be repaired
477 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj2 remove || return 1
478 objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 rm-attr _ || return 1
479
480 local pgid=$(get_pg $poolname obj1)
481 local primary=$(get_primary $poolname obj1)
482 local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
483 CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_deep_scrub $pgid
484 CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_scrub $pgid
485
486 # Wait for auto repair
487 wait_for_scrub $pgid "$last_scrub_stamp" || return 1
488 wait_for_clean || return 1
489 flush_pg_stats
490 grep -q "scrub_finish.*present with no repair possible" $dir/osd.${primary}.log || return 1
491 ceph pg dump pgs
492 ceph pg dump pgs | grep -q "^$(pgid).*+failed_repair" || return 1
493
494 # Tear down
495 teardown $dir || return 1
496 }
497
498 function TEST_repair_stats() {
499 local dir=$1
500 local poolname=testpool
501 local OSDS=2
502 local OBJS=30
503 # This need to be an even number
504 local REPAIRS=20
505
506 # Launch a cluster with 5 seconds scrub interval
507 setup $dir || return 1
508 run_mon $dir a || return 1
509 run_mgr $dir x || return 1
510 local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \
511 --osd-scrub-interval-randomize-ratio=0"
512 for id in $(seq 0 $(expr $OSDS - 1)) ; do
513 run_osd_bluestore $dir $id $ceph_osd_args || return 1
514 done
515
516 create_pool $poolname 1 1 || return 1
517 ceph osd pool set $poolname size 2
518 wait_for_clean || return 1
519
520 # Put an object
521 local payload=ABCDEF
522 echo $payload > $dir/ORIGINAL
523 for i in $(seq 1 $OBJS)
524 do
525 rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
526 done
527
528 # Remove the object from one shard physically
529 # Restarted osd get $ceph_osd_args passed
530 local other=$(get_not_primary $poolname obj1)
531 local pgid=$(get_pg $poolname obj1)
532 local primary=$(get_primary $poolname obj1)
533
534 kill_daemons $dir TERM osd.$other >&2 < /dev/null || return 1
535 kill_daemons $dir TERM osd.$primary >&2 < /dev/null || return 1
536 for i in $(seq 1 $REPAIRS)
537 do
538 # Remove from both osd.0 and osd.1
539 OSD=$(expr $i % 2)
540 _objectstore_tool_nodown $dir $OSD obj$i remove || return 1
541 done
542 run_osd_bluestore $dir $primary $ceph_osd_args || return 1
543 run_osd_bluestore $dir $other $ceph_osd_args || return 1
544 wait_for_clean || return 1
545
546 repair $pgid
547 wait_for_clean || return 1
548 ceph pg dump pgs
549
550 # This should have caused 1 object to be repaired
551 ceph pg $pgid query | jq '.info.stats.stat_sum'
552 COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
553 test "$COUNT" = "$REPAIRS" || return 1
554
555 ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $primary )"
556 COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $primary ).num_shards_repaired")
557 test "$COUNT" = "$(expr $REPAIRS / 2)" || return 1
558
559 ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $other )"
560 COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $other ).num_shards_repaired")
561 test "$COUNT" = "$(expr $REPAIRS / 2)" || return 1
562
563 ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum"
564 COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
565 test "$COUNT" = "$REPAIRS" || return 1
566
567 # Tear down
568 teardown $dir || return 1
569 }
570
571 function TEST_repair_stats_ec() {
572 local dir=$1
573 local poolname=testpool
574 local OSDS=3
575 local OBJS=30
576 # This need to be an even number
577 local REPAIRS=26
578 local allow_overwrites=false
579
580 # Launch a cluster with 5 seconds scrub interval
581 setup $dir || return 1
582 run_mon $dir a || return 1
583 run_mgr $dir x || return 1
584 local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \
585 --osd-scrub-interval-randomize-ratio=0"
586 for id in $(seq 0 $(expr $OSDS - 1)) ; do
587 run_osd_bluestore $dir $id $ceph_osd_args || return 1
588 done
589
590 # Create an EC pool
591 create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
592
593 # Put an object
594 local payload=ABCDEF
595 echo $payload > $dir/ORIGINAL
596 for i in $(seq 1 $OBJS)
597 do
598 rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
599 done
600
601 # Remove the object from one shard physically
602 # Restarted osd get $ceph_osd_args passed
603 local other=$(get_not_primary $poolname obj1)
604 local pgid=$(get_pg $poolname obj1)
605 local primary=$(get_primary $poolname obj1)
606
607 kill_daemons $dir TERM osd.$other >&2 < /dev/null || return 1
608 kill_daemons $dir TERM osd.$primary >&2 < /dev/null || return 1
609 for i in $(seq 1 $REPAIRS)
610 do
611 # Remove from both osd.0 and osd.1
612 OSD=$(expr $i % 2)
613 _objectstore_tool_nodown $dir $OSD obj$i remove || return 1
614 done
615 run_osd_bluestore $dir $primary $ceph_osd_args || return 1
616 run_osd_bluestore $dir $other $ceph_osd_args || return 1
617 wait_for_clean || return 1
618
619 repair $pgid
620 wait_for_clean || return 1
621 ceph pg dump pgs
622
623 # This should have caused 1 object to be repaired
624 ceph pg $pgid query | jq '.info.stats.stat_sum'
625 COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
626 test "$COUNT" = "$REPAIRS" || return 1
627
628 for osd in $(seq 0 $(expr $OSDS - 1)) ; do
629 if [ $osd = $other -o $osd = $primary ]; then
630 repair=$(expr $REPAIRS / 2)
631 else
632 repair="0"
633 fi
634
635 ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $osd )"
636 COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $osd ).num_shards_repaired")
637 test "$COUNT" = "$repair" || return 1
638 done
639
640 ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum"
641 COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
642 test "$COUNT" = "$REPAIRS" || return 1
643
644 # Tear down
645 teardown $dir || return 1
646 }
647
648 function corrupt_and_repair_jerasure() {
649 local dir=$1
650 local allow_overwrites=$2
651 local poolname=ecpool
652
653 setup $dir || return 1
654 run_mon $dir a || return 1
655 run_mgr $dir x || return 1
656 for id in $(seq 0 3) ; do
657 if [ "$allow_overwrites" = "true" ]; then
658 run_osd_bluestore $dir $id || return 1
659 else
660 run_osd $dir $id || return 1
661 fi
662 done
663 create_rbd_pool || return 1
664 wait_for_clean || return 1
665
666 create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
667 corrupt_and_repair_erasure_coded $dir $poolname || return 1
668
669 teardown $dir || return 1
670 }
671
672 function TEST_corrupt_and_repair_jerasure_appends() {
673 corrupt_and_repair_jerasure $1 false
674 }
675
676 function TEST_corrupt_and_repair_jerasure_overwrites() {
677 if [ "$use_ec_overwrite" = "true" ]; then
678 corrupt_and_repair_jerasure $1 true
679 fi
680 }
681
682 function corrupt_and_repair_lrc() {
683 local dir=$1
684 local allow_overwrites=$2
685 local poolname=ecpool
686
687 setup $dir || return 1
688 run_mon $dir a || return 1
689 run_mgr $dir x || return 1
690 for id in $(seq 0 9) ; do
691 if [ "$allow_overwrites" = "true" ]; then
692 run_osd_bluestore $dir $id || return 1
693 else
694 run_osd $dir $id || return 1
695 fi
696 done
697 create_rbd_pool || return 1
698 wait_for_clean || return 1
699
700 create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1
701 corrupt_and_repair_erasure_coded $dir $poolname || return 1
702
703 teardown $dir || return 1
704 }
705
706 function TEST_corrupt_and_repair_lrc_appends() {
707 corrupt_and_repair_lrc $1 false
708 }
709
710 function TEST_corrupt_and_repair_lrc_overwrites() {
711 if [ "$use_ec_overwrite" = "true" ]; then
712 corrupt_and_repair_lrc $1 true
713 fi
714 }
715
716 function unfound_erasure_coded() {
717 local dir=$1
718 local allow_overwrites=$2
719 local poolname=ecpool
720 local payload=ABCDEF
721
722 setup $dir || return 1
723 run_mon $dir a || return 1
724 run_mgr $dir x || return 1
725 for id in $(seq 0 3) ; do
726 if [ "$allow_overwrites" = "true" ]; then
727 run_osd_bluestore $dir $id || return 1
728 else
729 run_osd $dir $id || return 1
730 fi
731 done
732
733 create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
734
735 add_something $dir $poolname || return 1
736
737 local primary=$(get_primary $poolname SOMETHING)
738 local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
739 local not_primary_first=${osds[0]}
740 local not_primary_second=${osds[1]}
741 local not_primary_third=${osds[2]}
742
743 #
744 # 1) remove the corresponding file from the OSDs
745 #
746 pids=""
747 run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove
748 run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove
749 run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove
750 wait_background pids
751 return_code=$?
752 if [ $return_code -ne 0 ]; then return $return_code; fi
753
754 #
755 # 2) repair the PG
756 #
757 local pg=$(get_pg $poolname SOMETHING)
758 repair $pg
759 #
760 # 3) check pg state
761 #
762 # it may take a bit to appear due to mon/mgr asynchrony
763 for f in `seq 1 60`; do
764 ceph -s | grep "1/1 objects unfound" && break
765 sleep 1
766 done
767 ceph -s|grep "4 up" || return 1
768 ceph -s|grep "4 in" || return 1
769 ceph -s|grep "1/1 objects unfound" || return 1
770
771 teardown $dir || return 1
772 }
773
774 function TEST_unfound_erasure_coded_appends() {
775 unfound_erasure_coded $1 false
776 }
777
778 function TEST_unfound_erasure_coded_overwrites() {
779 if [ "$use_ec_overwrite" = "true" ]; then
780 unfound_erasure_coded $1 true
781 fi
782 }
783
784 #
785 # list_missing for EC pool
786 #
787 function list_missing_erasure_coded() {
788 local dir=$1
789 local allow_overwrites=$2
790 local poolname=ecpool
791
792 setup $dir || return 1
793 run_mon $dir a || return 1
794 run_mgr $dir x || return 1
795 for id in $(seq 0 2) ; do
796 if [ "$allow_overwrites" = "true" ]; then
797 run_osd_bluestore $dir $id || return 1
798 else
799 run_osd $dir $id || return 1
800 fi
801 done
802 create_rbd_pool || return 1
803 wait_for_clean || return 1
804
805 create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
806
807 # Put an object and remove the two shards (including primary)
808 add_something $dir $poolname MOBJ0 || return 1
809 local -a osds0=($(get_osds $poolname MOBJ0))
810
811 # Put another object and remove two shards (excluding primary)
812 add_something $dir $poolname MOBJ1 || return 1
813 local -a osds1=($(get_osds $poolname MOBJ1))
814
815 # Stop all osd daemons
816 for id in $(seq 0 2) ; do
817 kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1
818 done
819
820 id=${osds0[0]}
821 ceph-objectstore-tool --data-path $dir/$id \
822 MOBJ0 remove || return 1
823 id=${osds0[1]}
824 ceph-objectstore-tool --data-path $dir/$id \
825 MOBJ0 remove || return 1
826
827 id=${osds1[1]}
828 ceph-objectstore-tool --data-path $dir/$id \
829 MOBJ1 remove || return 1
830 id=${osds1[2]}
831 ceph-objectstore-tool --data-path $dir/$id \
832 MOBJ1 remove || return 1
833
834 for id in $(seq 0 2) ; do
835 activate_osd $dir $id >&2 || return 1
836 done
837 create_rbd_pool || return 1
838 wait_for_clean || return 1
839
840 # Get get - both objects should in the same PG
841 local pg=$(get_pg $poolname MOBJ0)
842
843 # Repair the PG, which triggers the recovering,
844 # and should mark the object as unfound
845 repair $pg
846
847 for i in $(seq 0 120) ; do
848 [ $i -lt 60 ] || return 1
849 matches=$(ceph pg $pg list_unfound | egrep "MOBJ0|MOBJ1" | wc -l)
850 [ $matches -eq 2 ] && break
851 done
852
853 teardown $dir || return 1
854 }
855
856 function TEST_list_missing_erasure_coded_appends() {
857 list_missing_erasure_coded $1 false
858 }
859
860 function TEST_list_missing_erasure_coded_overwrites() {
861 if [ "$use_ec_overwrite" = "true" ]; then
862 list_missing_erasure_coded $1 true
863 fi
864 }
865
866 #
867 # Corrupt one copy of a replicated pool
868 #
869 function TEST_corrupt_scrub_replicated() {
870 local dir=$1
871 local poolname=csr_pool
872 local total_objs=18
873
874 setup $dir || return 1
875 run_mon $dir a --osd_pool_default_size=2 || return 1
876 run_mgr $dir x || return 1
877 run_osd $dir 0 || return 1
878 run_osd $dir 1 || return 1
879 create_rbd_pool || return 1
880 wait_for_clean || return 1
881
882 create_pool foo 1 || return 1
883 create_pool $poolname 1 1 || return 1
884 wait_for_clean || return 1
885
886 for i in $(seq 1 $total_objs) ; do
887 objname=ROBJ${i}
888 add_something $dir $poolname $objname || return 1
889
890 rados --pool $poolname setomapheader $objname hdr-$objname || return 1
891 rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
892 done
893
894 local pg=$(get_pg $poolname ROBJ0)
895 local primary=$(get_primary $poolname ROBJ0)
896
897 # Compute an old omap digest and save oi
898 CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \
899 config set osd_deep_scrub_update_digest_min_age 0
900 CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \
901 config set osd_deep_scrub_update_digest_min_age 0
902 pg_deep_scrub $pg
903
904 for i in $(seq 1 $total_objs) ; do
905 objname=ROBJ${i}
906
907 # Alternate corruption between osd.0 and osd.1
908 local osd=$(expr $i % 2)
909
910 case $i in
911 1)
912 # Size (deep scrub data_digest too)
913 local payload=UVWXYZZZ
914 echo $payload > $dir/CORRUPT
915 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
916 ;;
917
918 2)
919 # digest (deep scrub only)
920 local payload=UVWXYZ
921 echo $payload > $dir/CORRUPT
922 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
923 ;;
924
925 3)
926 # missing
927 objectstore_tool $dir $osd $objname remove || return 1
928 ;;
929
930 4)
931 # Modify omap value (deep scrub only)
932 objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1
933 ;;
934
935 5)
936 # Delete omap key (deep scrub only)
937 objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1
938 ;;
939
940 6)
941 # Add extra omap key (deep scrub only)
942 echo extra > $dir/extra-val
943 objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1
944 rm $dir/extra-val
945 ;;
946
947 7)
948 # Modify omap header (deep scrub only)
949 echo -n newheader > $dir/hdr
950 objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1
951 rm $dir/hdr
952 ;;
953
954 8)
955 rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
956 rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
957
958 # Break xattrs
959 echo -n bad-val > $dir/bad-val
960 objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
961 objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
962 echo -n val3-$objname > $dir/newval
963 objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
964 rm $dir/bad-val $dir/newval
965 ;;
966
967 9)
968 objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi
969 echo -n D > $dir/change
970 rados --pool $poolname put $objname $dir/change
971 objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi
972 rm $dir/oi $dir/change
973 ;;
974
975 # ROBJ10 must be handled after digests are re-computed by a deep scrub below
976 # ROBJ11 must be handled with config change before deep scrub
977 # ROBJ12 must be handled with config change before scrubs
978 # ROBJ13 must be handled before scrubs
979
980 14)
981 echo -n bad-val > $dir/bad-val
982 objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1
983 objectstore_tool $dir 1 $objname rm-attr _ || return 1
984 rm $dir/bad-val
985 ;;
986
987 15)
988 objectstore_tool $dir $osd $objname rm-attr _ || return 1
989 ;;
990
991 16)
992 objectstore_tool $dir 0 $objname rm-attr snapset || return 1
993 echo -n bad-val > $dir/bad-val
994 objectstore_tool $dir 1 $objname set-attr snapset $dir/bad-val || return 1
995 ;;
996
997 17)
998 # Deep-scrub only (all replicas are diffent than the object info
999 local payload=ROBJ17
1000 echo $payload > $dir/new.ROBJ17
1001 objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ17 || return 1
1002 objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ17 || return 1
1003 ;;
1004
1005 18)
1006 # Deep-scrub only (all replicas are diffent than the object info
1007 local payload=ROBJ18
1008 echo $payload > $dir/new.ROBJ18
1009 objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ18 || return 1
1010 objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ18 || return 1
1011 # Make one replica have a different object info, so a full repair must happen too
1012 objectstore_tool $dir $osd $objname corrupt-info || return 1
1013
1014 esac
1015 done
1016
1017 local pg=$(get_pg $poolname ROBJ0)
1018
1019 inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1020 inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1021 inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1022 inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1023
1024 pg_scrub $pg
1025
1026 ERRORS=0
1027 declare -a s_err_strings
1028 err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:30259878:::ROBJ15:head : candidate had a missing info key"
1029 err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:33aca486:::ROBJ18:head : object info inconsistent "
1030 err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:5c7b2c47:::ROBJ16:head : candidate had a corrupt snapset"
1031 err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:5c7b2c47:::ROBJ16:head : candidate had a missing snapset key"
1032 err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:5c7b2c47:::ROBJ16:head : failed to pick suitable object info"
1033 err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:86586531:::ROBJ8:head : attr value mismatch '_key1-ROBJ8', attr name mismatch '_key3-ROBJ8', attr name mismatch '_key2-ROBJ8'"
1034 err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:bc819597:::ROBJ12:head : candidate had a stat error"
1035 err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:c0c86b1d:::ROBJ14:head : candidate had a missing info key"
1036 err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:c0c86b1d:::ROBJ14:head : candidate had a corrupt info"
1037 err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:c0c86b1d:::ROBJ14:head : failed to pick suitable object info"
1038 err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : candidate size 9 info size 7 mismatch"
1039 err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : size 9 != size 7 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from shard 0"
1040 err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:d60617f9:::ROBJ13:head : candidate had a stat error"
1041 err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 3:f2a5b2a4:::ROBJ3:head : missing"
1042 err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ffdb2004:::ROBJ9:head : candidate size 1 info size 7 mismatch"
1043 err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ffdb2004:::ROBJ9:head : object info inconsistent "
1044 err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 3:c0c86b1d:::ROBJ14:head : no '_' attr"
1045 err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 3:5c7b2c47:::ROBJ16:head : can't decode 'snapset' attr buffer::malformed_input: .* no longer understand old encoding version 3 < 97"
1046 err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub : stat mismatch, got 18/18 objects, 0/0 clones, 17/18 dirty, 17/18 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 113/120 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
1047 err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 1 missing, 7 inconsistent objects"
1048 err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 17 errors"
1049
1050 for err_string in "${err_strings[@]}"
1051 do
1052 if ! grep -q "$err_string" $dir/osd.${primary}.log
1053 then
1054 echo "Missing log message '$err_string'"
1055 ERRORS=$(expr $ERRORS + 1)
1056 fi
1057 done
1058
1059 rados list-inconsistent-pg $poolname > $dir/json || return 1
1060 # Check pg count
1061 test $(jq '. | length' $dir/json) = "1" || return 1
1062 # Check pgid
1063 test $(jq -r '.[0]' $dir/json) = $pg || return 1
1064
1065 rados list-inconsistent-obj $pg > $dir/json || return 1
1066 # Get epoch for repair-get requests
1067 epoch=$(jq .epoch $dir/json)
1068
1069 jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
1070 {
1071 "inconsistents": [
1072 {
1073 "shards": [
1074 {
1075 "size": 7,
1076 "errors": [],
1077 "osd": 0,
1078 "primary": false
1079 },
1080 {
1081 "object_info": {
1082 "oid": {
1083 "oid": "ROBJ1",
1084 "key": "",
1085 "snapid": -2,
1086 "hash": 1454963827,
1087 "max": 0,
1088 "pool": 3,
1089 "namespace": ""
1090 },
1091 "version": "51'58",
1092 "prior_version": "21'3",
1093 "last_reqid": "osd.1.0:57",
1094 "user_version": 3,
1095 "size": 7,
1096 "mtime": "",
1097 "local_mtime": "",
1098 "lost": 0,
1099 "flags": [
1100 "dirty",
1101 "omap",
1102 "data_digest",
1103 "omap_digest"
1104 ],
1105 "truncate_seq": 0,
1106 "truncate_size": 0,
1107 "data_digest": "0x2ddbf8f5",
1108 "omap_digest": "0xf5fba2c6",
1109 "expected_object_size": 0,
1110 "expected_write_size": 0,
1111 "alloc_hint_flags": 0,
1112 "manifest": {
1113 "type": 0
1114 },
1115 "watchers": {}
1116 },
1117 "size": 9,
1118 "errors": [
1119 "size_mismatch_info",
1120 "obj_size_info_mismatch"
1121 ],
1122 "osd": 1,
1123 "primary": true
1124 }
1125 ],
1126 "selected_object_info": {
1127 "oid": {
1128 "oid": "ROBJ1",
1129 "key": "",
1130 "snapid": -2,
1131 "hash": 1454963827,
1132 "max": 0,
1133 "pool": 3,
1134 "namespace": ""
1135 },
1136 "version": "51'58",
1137 "prior_version": "21'3",
1138 "last_reqid": "osd.1.0:57",
1139 "user_version": 3,
1140 "size": 7,
1141 "mtime": "2018-04-05 14:33:19.804040",
1142 "local_mtime": "2018-04-05 14:33:19.804839",
1143 "lost": 0,
1144 "flags": [
1145 "dirty",
1146 "omap",
1147 "data_digest",
1148 "omap_digest"
1149 ],
1150 "truncate_seq": 0,
1151 "truncate_size": 0,
1152 "data_digest": "0x2ddbf8f5",
1153 "omap_digest": "0xf5fba2c6",
1154 "expected_object_size": 0,
1155 "expected_write_size": 0,
1156 "alloc_hint_flags": 0,
1157 "manifest": {
1158 "type": 0
1159 },
1160 "watchers": {}
1161 },
1162 "union_shard_errors": [
1163 "size_mismatch_info",
1164 "obj_size_info_mismatch"
1165 ],
1166 "errors": [
1167 "size_mismatch"
1168 ],
1169 "object": {
1170 "version": 3,
1171 "snap": "head",
1172 "locator": "",
1173 "nspace": "",
1174 "name": "ROBJ1"
1175 }
1176 },
1177 {
1178 "shards": [
1179 {
1180 "errors": [
1181 "stat_error"
1182 ],
1183 "osd": 0,
1184 "primary": false
1185 },
1186 {
1187 "size": 7,
1188 "errors": [],
1189 "osd": 1,
1190 "primary": true
1191 }
1192 ],
1193 "selected_object_info": {
1194 "oid": {
1195 "oid": "ROBJ12",
1196 "key": "",
1197 "snapid": -2,
1198 "hash": 3920199997,
1199 "max": 0,
1200 "pool": 3,
1201 "namespace": ""
1202 },
1203 "version": "51'56",
1204 "prior_version": "43'36",
1205 "last_reqid": "osd.1.0:55",
1206 "user_version": 36,
1207 "size": 7,
1208 "mtime": "",
1209 "local_mtime": "",
1210 "lost": 0,
1211 "flags": [
1212 "dirty",
1213 "omap",
1214 "data_digest",
1215 "omap_digest"
1216 ],
1217 "truncate_seq": 0,
1218 "truncate_size": 0,
1219 "data_digest": "0x2ddbf8f5",
1220 "omap_digest": "0x067f306a",
1221 "expected_object_size": 0,
1222 "expected_write_size": 0,
1223 "alloc_hint_flags": 0,
1224 "manifest": {
1225 "type": 0
1226 },
1227 "watchers": {}
1228 },
1229 "union_shard_errors": [
1230 "stat_error"
1231 ],
1232 "errors": [],
1233 "object": {
1234 "version": 36,
1235 "snap": "head",
1236 "locator": "",
1237 "nspace": "",
1238 "name": "ROBJ12"
1239 }
1240 },
1241 {
1242 "shards": [
1243 {
1244 "errors": [
1245 "stat_error"
1246 ],
1247 "osd": 0,
1248 "primary": false
1249 },
1250 {
1251 "size": 7,
1252 "errors": [],
1253 "osd": 1,
1254 "primary": true
1255 }
1256 ],
1257 "selected_object_info": {
1258 "oid": {
1259 "oid": "ROBJ13",
1260 "key": "",
1261 "snapid": -2,
1262 "hash": 2682806379,
1263 "max": 0,
1264 "pool": 3,
1265 "namespace": ""
1266 },
1267 "version": "51'59",
1268 "prior_version": "45'39",
1269 "last_reqid": "osd.1.0:58",
1270 "user_version": 39,
1271 "size": 7,
1272 "mtime": "",
1273 "local_mtime": "",
1274 "lost": 0,
1275 "flags": [
1276 "dirty",
1277 "omap",
1278 "data_digest",
1279 "omap_digest"
1280 ],
1281 "truncate_seq": 0,
1282 "truncate_size": 0,
1283 "data_digest": "0x2ddbf8f5",
1284 "omap_digest": "0x6441854d",
1285 "expected_object_size": 0,
1286 "expected_write_size": 0,
1287 "alloc_hint_flags": 0,
1288 "manifest": {
1289 "type": 0
1290 },
1291 "watchers": {}
1292 },
1293 "union_shard_errors": [
1294 "stat_error"
1295 ],
1296 "errors": [],
1297 "object": {
1298 "version": 39,
1299 "snap": "head",
1300 "locator": "",
1301 "nspace": "",
1302 "name": "ROBJ13"
1303 }
1304 },
1305 {
1306 "shards": [
1307 {
1308 "object_info": "bad-val",
1309 "size": 7,
1310 "errors": [
1311 "info_corrupted"
1312 ],
1313 "osd": 0,
1314 "primary": false
1315 },
1316 {
1317 "size": 7,
1318 "errors": [
1319 "info_missing"
1320 ],
1321 "osd": 1,
1322 "primary": true
1323 }
1324 ],
1325 "union_shard_errors": [
1326 "info_missing",
1327 "info_corrupted"
1328 ],
1329 "errors": [],
1330 "object": {
1331 "version": 0,
1332 "snap": "head",
1333 "locator": "",
1334 "nspace": "",
1335 "name": "ROBJ14"
1336 }
1337 },
1338 {
1339 "shards": [
1340 {
1341 "object_info": {
1342 "oid": {
1343 "oid": "ROBJ15",
1344 "key": "",
1345 "snapid": -2,
1346 "hash": 504996876,
1347 "max": 0,
1348 "pool": 3,
1349 "namespace": ""
1350 },
1351 "version": "51'49",
1352 "prior_version": "49'45",
1353 "last_reqid": "osd.1.0:48",
1354 "user_version": 45,
1355 "size": 7,
1356 "mtime": "2018-04-05 14:33:29.498969",
1357 "local_mtime": "2018-04-05 14:33:29.499890",
1358 "lost": 0,
1359 "flags": [
1360 "dirty",
1361 "omap",
1362 "data_digest",
1363 "omap_digest"
1364 ],
1365 "truncate_seq": 0,
1366 "truncate_size": 0,
1367 "data_digest": "0x2ddbf8f5",
1368 "omap_digest": "0x2d2a4d6e",
1369 "expected_object_size": 0,
1370 "expected_write_size": 0,
1371 "alloc_hint_flags": 0,
1372 "manifest": {
1373 "type": 0
1374 },
1375 "watchers": {}
1376 },
1377 "size": 7,
1378 "errors": [],
1379 "osd": 0,
1380 "primary": false
1381 },
1382 {
1383 "size": 7,
1384 "errors": [
1385 "info_missing"
1386 ],
1387 "osd": 1,
1388 "primary": true
1389 }
1390 ],
1391 "selected_object_info": {
1392 "oid": {
1393 "oid": "ROBJ15",
1394 "key": "",
1395 "snapid": -2,
1396 "hash": 504996876,
1397 "max": 0,
1398 "pool": 3,
1399 "namespace": ""
1400 },
1401 "version": "51'49",
1402 "prior_version": "49'45",
1403 "last_reqid": "osd.1.0:48",
1404 "user_version": 45,
1405 "size": 7,
1406 "mtime": "",
1407 "local_mtime": "",
1408 "lost": 0,
1409 "flags": [
1410 "dirty",
1411 "omap",
1412 "data_digest",
1413 "omap_digest"
1414 ],
1415 "truncate_seq": 0,
1416 "truncate_size": 0,
1417 "data_digest": "0x2ddbf8f5",
1418 "omap_digest": "0x2d2a4d6e",
1419 "expected_object_size": 0,
1420 "expected_write_size": 0,
1421 "alloc_hint_flags": 0,
1422 "manifest": {
1423 "type": 0
1424 },
1425 "watchers": {}
1426 },
1427 "union_shard_errors": [
1428 "info_missing"
1429 ],
1430 "errors": [],
1431 "object": {
1432 "version": 45,
1433 "snap": "head",
1434 "locator": "",
1435 "nspace": "",
1436 "name": "ROBJ15"
1437 }
1438 },
1439 {
1440 "errors": [],
1441 "object": {
1442 "locator": "",
1443 "name": "ROBJ16",
1444 "nspace": "",
1445 "snap": "head",
1446 "version": 0
1447 },
1448 "shards": [
1449 {
1450 "errors": [
1451 "snapset_missing"
1452 ],
1453 "osd": 0,
1454 "primary": false,
1455 "size": 7
1456 },
1457 {
1458 "errors": [
1459 "snapset_corrupted"
1460 ],
1461 "osd": 1,
1462 "primary": true,
1463 "snapset": "bad-val",
1464 "size": 7
1465 }
1466 ],
1467 "union_shard_errors": [
1468 "snapset_missing",
1469 "snapset_corrupted"
1470 ]
1471 },
1472 {
1473 "errors": [
1474 "object_info_inconsistency"
1475 ],
1476 "object": {
1477 "locator": "",
1478 "name": "ROBJ18",
1479 "nspace": "",
1480 "snap": "head"
1481 },
1482 "selected_object_info": {
1483 "alloc_hint_flags": 255,
1484 "data_digest": "0x2ddbf8f5",
1485 "expected_object_size": 0,
1486 "expected_write_size": 0,
1487 "flags": [
1488 "dirty",
1489 "omap",
1490 "data_digest",
1491 "omap_digest"
1492 ],
1493 "lost": 0,
1494 "manifest": {
1495 "type": 0
1496 },
1497 "oid": {
1498 "hash": 1629828556,
1499 "key": "",
1500 "max": 0,
1501 "namespace": "",
1502 "oid": "ROBJ18",
1503 "pool": 3,
1504 "snapid": -2
1505 },
1506 "omap_digest": "0xddc3680f",
1507 "size": 7,
1508 "truncate_seq": 0,
1509 "truncate_size": 0,
1510 "user_version": 54,
1511 "watchers": {}
1512 },
1513 "shards": [
1514 {
1515 "errors": [],
1516 "object_info": {
1517 "alloc_hint_flags": 0,
1518 "data_digest": "0x2ddbf8f5",
1519 "expected_object_size": 0,
1520 "expected_write_size": 0,
1521 "flags": [
1522 "dirty",
1523 "omap",
1524 "data_digest",
1525 "omap_digest"
1526 ],
1527 "lost": 0,
1528 "manifest": {
1529 "type": 0
1530 },
1531 "oid": {
1532 "hash": 1629828556,
1533 "key": "",
1534 "max": 0,
1535 "namespace": "",
1536 "oid": "ROBJ18",
1537 "pool": 3,
1538 "snapid": -2
1539 },
1540 "omap_digest": "0xddc3680f",
1541 "size": 7,
1542 "truncate_seq": 0,
1543 "truncate_size": 0,
1544 "user_version": 54,
1545 "watchers": {}
1546 },
1547 "osd": 0,
1548 "primary": false,
1549 "size": 7
1550 },
1551 {
1552 "errors": [],
1553 "object_info": {
1554 "alloc_hint_flags": 255,
1555 "data_digest": "0x2ddbf8f5",
1556 "expected_object_size": 0,
1557 "expected_write_size": 0,
1558 "flags": [
1559 "dirty",
1560 "omap",
1561 "data_digest",
1562 "omap_digest"
1563 ],
1564 "lost": 0,
1565 "manifest": {
1566 "type": 0
1567 },
1568 "oid": {
1569 "hash": 1629828556,
1570 "key": "",
1571 "max": 0,
1572 "namespace": "",
1573 "oid": "ROBJ18",
1574 "pool": 3,
1575 "snapid": -2
1576 },
1577 "omap_digest": "0xddc3680f",
1578 "size": 7,
1579 "truncate_seq": 0,
1580 "truncate_size": 0,
1581 "user_version": 54,
1582 "watchers": {}
1583 },
1584 "osd": 1,
1585 "primary": true,
1586 "size": 7
1587 }
1588 ],
1589 "union_shard_errors": []
1590 },
1591 {
1592 "shards": [
1593 {
1594 "size": 7,
1595 "errors": [],
1596 "osd": 0,
1597 "primary": false
1598 },
1599 {
1600 "errors": [
1601 "missing"
1602 ],
1603 "osd": 1,
1604 "primary": true
1605 }
1606 ],
1607 "selected_object_info": {
1608 "oid": {
1609 "oid": "ROBJ3",
1610 "key": "",
1611 "snapid": -2,
1612 "hash": 625845583,
1613 "max": 0,
1614 "pool": 3,
1615 "namespace": ""
1616 },
1617 "version": "51'61",
1618 "prior_version": "25'9",
1619 "last_reqid": "osd.1.0:60",
1620 "user_version": 9,
1621 "size": 7,
1622 "mtime": "",
1623 "local_mtime": "",
1624 "lost": 0,
1625 "flags": [
1626 "dirty",
1627 "omap",
1628 "data_digest",
1629 "omap_digest"
1630 ],
1631 "truncate_seq": 0,
1632 "truncate_size": 0,
1633 "data_digest": "0x2ddbf8f5",
1634 "omap_digest": "0x00b35dfd",
1635 "expected_object_size": 0,
1636 "expected_write_size": 0,
1637 "alloc_hint_flags": 0,
1638 "manifest": {
1639 "type": 0
1640 },
1641 "watchers": {}
1642 },
1643 "union_shard_errors": [
1644 "missing"
1645 ],
1646 "errors": [],
1647 "object": {
1648 "version": 9,
1649 "snap": "head",
1650 "locator": "",
1651 "nspace": "",
1652 "name": "ROBJ3"
1653 }
1654 },
1655 {
1656 "shards": [
1657 {
1658 "attrs": [
1659 {
1660 "Base64": false,
1661 "value": "bad-val",
1662 "name": "key1-ROBJ8"
1663 },
1664 {
1665 "Base64": false,
1666 "value": "val2-ROBJ8",
1667 "name": "key2-ROBJ8"
1668 }
1669 ],
1670 "size": 7,
1671 "errors": [],
1672 "osd": 0,
1673 "primary": false
1674 },
1675 {
1676 "attrs": [
1677 {
1678 "Base64": false,
1679 "value": "val1-ROBJ8",
1680 "name": "key1-ROBJ8"
1681 },
1682 {
1683 "Base64": false,
1684 "value": "val3-ROBJ8",
1685 "name": "key3-ROBJ8"
1686 }
1687 ],
1688 "size": 7,
1689 "errors": [],
1690 "osd": 1,
1691 "primary": true
1692 }
1693 ],
1694 "selected_object_info": {
1695 "oid": {
1696 "oid": "ROBJ8",
1697 "key": "",
1698 "snapid": -2,
1699 "hash": 2359695969,
1700 "max": 0,
1701 "pool": 3,
1702 "namespace": ""
1703 },
1704 "version": "79'66",
1705 "prior_version": "79'65",
1706 "last_reqid": "client.4554.0:1",
1707 "user_version": 74,
1708 "size": 7,
1709 "mtime": "",
1710 "local_mtime": "",
1711 "lost": 0,
1712 "flags": [
1713 "dirty",
1714 "omap",
1715 "data_digest",
1716 "omap_digest"
1717 ],
1718 "truncate_seq": 0,
1719 "truncate_size": 0,
1720 "data_digest": "0x2ddbf8f5",
1721 "omap_digest": "0xd6be81dc",
1722 "expected_object_size": 0,
1723 "expected_write_size": 0,
1724 "alloc_hint_flags": 0,
1725 "manifest": {
1726 "type": 0
1727 },
1728 "watchers": {}
1729 },
1730 "union_shard_errors": [],
1731 "errors": [
1732 "attr_value_mismatch",
1733 "attr_name_mismatch"
1734 ],
1735 "object": {
1736 "version": 66,
1737 "snap": "head",
1738 "locator": "",
1739 "nspace": "",
1740 "name": "ROBJ8"
1741 }
1742 },
1743 {
1744 "shards": [
1745 {
1746 "object_info": {
1747 "oid": {
1748 "oid": "ROBJ9",
1749 "key": "",
1750 "snapid": -2,
1751 "hash": 537189375,
1752 "max": 0,
1753 "pool": 3,
1754 "namespace": ""
1755 },
1756 "version": "95'67",
1757 "prior_version": "51'64",
1758 "last_reqid": "client.4649.0:1",
1759 "user_version": 75,
1760 "size": 1,
1761 "mtime": "",
1762 "local_mtime": "",
1763 "lost": 0,
1764 "flags": [
1765 "dirty",
1766 "omap",
1767 "data_digest",
1768 "omap_digest"
1769 ],
1770 "truncate_seq": 0,
1771 "truncate_size": 0,
1772 "data_digest": "0x2b63260d",
1773 "omap_digest": "0x2eecc539",
1774 "expected_object_size": 0,
1775 "expected_write_size": 0,
1776 "alloc_hint_flags": 0,
1777 "manifest": {
1778 "type": 0
1779 },
1780 "watchers": {}
1781 },
1782 "size": 1,
1783 "errors": [],
1784 "osd": 0,
1785 "primary": false
1786 },
1787 {
1788 "object_info": {
1789 "oid": {
1790 "oid": "ROBJ9",
1791 "key": "",
1792 "snapid": -2,
1793 "hash": 537189375,
1794 "max": 0,
1795 "pool": 3,
1796 "namespace": ""
1797 },
1798 "version": "51'64",
1799 "prior_version": "37'27",
1800 "last_reqid": "osd.1.0:63",
1801 "user_version": 27,
1802 "size": 7,
1803 "mtime": "2018-04-05 14:33:25.352485",
1804 "local_mtime": "2018-04-05 14:33:25.353746",
1805 "lost": 0,
1806 "flags": [
1807 "dirty",
1808 "omap",
1809 "data_digest",
1810 "omap_digest"
1811 ],
1812 "truncate_seq": 0,
1813 "truncate_size": 0,
1814 "data_digest": "0x2ddbf8f5",
1815 "omap_digest": "0x2eecc539",
1816 "expected_object_size": 0,
1817 "expected_write_size": 0,
1818 "alloc_hint_flags": 0,
1819 "manifest": {
1820 "type": 0
1821 },
1822 "watchers": {}
1823 },
1824 "size": 1,
1825 "errors": [
1826 "obj_size_info_mismatch"
1827 ],
1828 "osd": 1,
1829 "primary": true
1830 }
1831 ],
1832 "selected_object_info": {
1833 "oid": {
1834 "oid": "ROBJ9",
1835 "key": "",
1836 "snapid": -2,
1837 "hash": 537189375,
1838 "max": 0,
1839 "pool": 3,
1840 "namespace": ""
1841 },
1842 "version": "95'67",
1843 "prior_version": "51'64",
1844 "last_reqid": "client.4649.0:1",
1845 "user_version": 75,
1846 "size": 1,
1847 "mtime": "",
1848 "local_mtime": "",
1849 "lost": 0,
1850 "flags": [
1851 "dirty",
1852 "omap",
1853 "data_digest",
1854 "omap_digest"
1855 ],
1856 "truncate_seq": 0,
1857 "truncate_size": 0,
1858 "data_digest": "0x2b63260d",
1859 "omap_digest": "0x2eecc539",
1860 "expected_object_size": 0,
1861 "expected_write_size": 0,
1862 "alloc_hint_flags": 0,
1863 "manifest": {
1864 "type": 0
1865 },
1866 "watchers": {}
1867 },
1868 "union_shard_errors": [
1869 "obj_size_info_mismatch"
1870 ],
1871 "errors": [
1872 "object_info_inconsistency"
1873 ],
1874 "object": {
1875 "version": 67,
1876 "snap": "head",
1877 "locator": "",
1878 "nspace": "",
1879 "name": "ROBJ9"
1880 }
1881 }
1882 ],
1883 "epoch": 0
1884 }
1885 EOF
1886
1887 jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
1888 multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
1889 if test $getjson = "yes"
1890 then
1891 jq '.' $dir/json > save1.json
1892 fi
1893
1894 if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
1895 then
1896 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
1897 fi
1898
1899 objname=ROBJ9
1900 # Change data and size again because digest was recomputed
1901 echo -n ZZZ > $dir/change
1902 rados --pool $poolname put $objname $dir/change
1903 # Set one to an even older value
1904 objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi
1905 rm $dir/oi $dir/change
1906
1907 objname=ROBJ10
1908 objectstore_tool $dir 1 $objname get-attr _ > $dir/oi
1909 rados --pool $poolname setomapval $objname key2-$objname val2-$objname
1910 objectstore_tool $dir 0 $objname set-attr _ $dir/oi
1911 objectstore_tool $dir 1 $objname set-attr _ $dir/oi
1912 rm $dir/oi
1913
1914 inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1915 inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1916 inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1917 inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1918 pg_deep_scrub $pg
1919
1920 err_strings=()
1921 err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:30259878:::ROBJ15:head : candidate had a missing info key"
1922 err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:33aca486:::ROBJ18:head : data_digest 0xbd89c912 != data_digest 0x2ddbf8f5 from auth oi 3:33aca486:::ROBJ18:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 54 dd 2ddbf8f5 od ddc3680f alloc_hint [[]0 0 255[]][)], object info inconsistent "
1923 err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:33aca486:::ROBJ18:head : data_digest 0xbd89c912 != data_digest 0x2ddbf8f5 from auth oi 3:33aca486:::ROBJ18:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 54 dd 2ddbf8f5 od ddc3680f alloc_hint [[]0 0 255[]][)]"
1924 err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:33aca486:::ROBJ18:head : failed to pick suitable auth object"
1925 err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:5c7b2c47:::ROBJ16:head : candidate had a corrupt snapset"
1926 err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:5c7b2c47:::ROBJ16:head : candidate had a missing snapset key"
1927 err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:5c7b2c47:::ROBJ16:head : failed to pick suitable object info"
1928 err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:86586531:::ROBJ8:head : attr value mismatch '_key1-ROBJ8', attr name mismatch '_key3-ROBJ8', attr name mismatch '_key2-ROBJ8'"
1929 err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:87abbf36:::ROBJ11:head : candidate had a read error"
1930 err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:8aa5320e:::ROBJ17:head : data_digest 0x5af0c3ef != data_digest 0x2ddbf8f5 from auth oi 3:8aa5320e:::ROBJ17:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 51 dd 2ddbf8f5 od e9572720 alloc_hint [[]0 0 0[]][)]"
1931 err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:8aa5320e:::ROBJ17:head : data_digest 0x5af0c3ef != data_digest 0x2ddbf8f5 from auth oi 3:8aa5320e:::ROBJ17:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 51 dd 2ddbf8f5 od e9572720 alloc_hint [[]0 0 0[]][)]"
1932 err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:8aa5320e:::ROBJ17:head : failed to pick suitable auth object"
1933 err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:8b55fa4b:::ROBJ7:head : omap_digest 0xefced57a != omap_digest 0x6a73cc07 from shard 1"
1934 err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:8b55fa4b:::ROBJ7:head : omap_digest 0x6a73cc07 != omap_digest 0xefced57a from auth oi 3:8b55fa4b:::ROBJ7:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [[]0 0 0[]][)]"
1935 err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:a53c12e8:::ROBJ6:head : omap_digest 0x689ee887 != omap_digest 0x179c919f from shard 1, omap_digest 0x689ee887 != omap_digest 0x179c919f from auth oi 3:a53c12e8:::ROBJ6:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [[]0 0 0[]][)]"
1936 err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:b1f19cbd:::ROBJ10:head : omap_digest 0xa8dd5adc != omap_digest 0xc2025a24 from auth oi 3:b1f19cbd:::ROBJ10:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [[]0 0 0[]][)]"
1937 err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:b1f19cbd:::ROBJ10:head : omap_digest 0xa8dd5adc != omap_digest 0xc2025a24 from auth oi 3:b1f19cbd:::ROBJ10:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [[]0 0 0[]][)]"
1938 err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:b1f19cbd:::ROBJ10:head : failed to pick suitable auth object"
1939 err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:bc819597:::ROBJ12:head : candidate had a stat error"
1940 err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:c0c86b1d:::ROBJ14:head : candidate had a missing info key"
1941 err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:c0c86b1d:::ROBJ14:head : candidate had a corrupt info"
1942 err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:c0c86b1d:::ROBJ14:head : failed to pick suitable object info"
1943 err_strings[22]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : candidate size 9 info size 7 mismatch"
1944 err_strings[23]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : data_digest 0x2d4a11c2 != data_digest 0x2ddbf8f5 from shard 0, data_digest 0x2d4a11c2 != data_digest 0x2ddbf8f5 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:65 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from shard 0"
1945 err_strings[24]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:d60617f9:::ROBJ13:head : candidate had a read error"
1946 err_strings[25]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:d60617f9:::ROBJ13:head : candidate had a stat error"
1947 err_strings[26]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:d60617f9:::ROBJ13:head : failed to pick suitable object info"
1948 err_strings[27]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:e97ce31e:::ROBJ2:head : data_digest 0x578a4830 != data_digest 0x2ddbf8f5 from shard 1, data_digest 0x578a4830 != data_digest 0x2ddbf8f5 from auth oi 3:e97ce31e:::ROBJ2:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [[]0 0 0[]][)]"
1949 err_strings[28]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 3:f2a5b2a4:::ROBJ3:head : missing"
1950 err_strings[29]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:f4981d31:::ROBJ4:head : omap_digest 0xd7178dfe != omap_digest 0xe2d46ea4 from shard 1, omap_digest 0xd7178dfe != omap_digest 0xe2d46ea4 from auth oi 3:f4981d31:::ROBJ4:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [[]0 0 0[]][)]"
1951 err_strings[30]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:f4bfd4d1:::ROBJ5:head : omap_digest 0x1a862a41 != omap_digest 0x6cac8f6 from shard 1"
1952 err_strings[31]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:f4bfd4d1:::ROBJ5:head : omap_digest 0x6cac8f6 != omap_digest 0x1a862a41 from auth oi 3:f4bfd4d1:::ROBJ5:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [[]0 0 0[]][)]"
1953 err_strings[32]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:ffdb2004:::ROBJ9:head : candidate size 3 info size 7 mismatch"
1954 err_strings[33]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:ffdb2004:::ROBJ9:head : object info inconsistent "
1955 err_strings[34]="log_channel[(]cluster[)] log [[]ERR[]] : deep-scrub [0-9]*[.]0 3:c0c86b1d:::ROBJ14:head : no '_' attr"
1956 err_strings[35]="log_channel[(]cluster[)] log [[]ERR[]] : deep-scrub [0-9]*[.]0 3:5c7b2c47:::ROBJ16:head : can't decode 'snapset' attr buffer::malformed_input: .* no longer understand old encoding version 3 < 97"
1957 err_strings[36]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub : stat mismatch, got 18/18 objects, 0/0 clones, 17/18 dirty, 17/18 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 115/116 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
1958 err_strings[37]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub 1 missing, 11 inconsistent objects"
1959 err_strings[38]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub 35 errors"
1960
1961 for err_string in "${err_strings[@]}"
1962 do
1963 if ! grep -q "$err_string" $dir/osd.${primary}.log
1964 then
1965 echo "Missing log message '$err_string'"
1966 ERRORS=$(expr $ERRORS + 1)
1967 fi
1968 done
1969
1970 rados list-inconsistent-pg $poolname > $dir/json || return 1
1971 # Check pg count
1972 test $(jq '. | length' $dir/json) = "1" || return 1
1973 # Check pgid
1974 test $(jq -r '.[0]' $dir/json) = $pg || return 1
1975
1976 rados list-inconsistent-obj $pg > $dir/json || return 1
1977 # Get epoch for repair-get requests
1978 epoch=$(jq .epoch $dir/json)
1979
1980 jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
1981 {
1982 "inconsistents": [
1983 {
1984 "shards": [
1985 {
1986 "data_digest": "0x2ddbf8f5",
1987 "omap_digest": "0xf5fba2c6",
1988 "size": 7,
1989 "errors": [],
1990 "osd": 0,
1991 "primary": false
1992 },
1993 {
1994 "object_info": {
1995 "oid": {
1996 "oid": "ROBJ1",
1997 "key": "",
1998 "snapid": -2,
1999 "hash": 1454963827,
2000 "max": 0,
2001 "pool": 3,
2002 "namespace": ""
2003 },
2004 "version": "51'58",
2005 "prior_version": "21'3",
2006 "last_reqid": "osd.1.0:57",
2007 "user_version": 3,
2008 "size": 7,
2009 "mtime": "2018-04-05 14:33:19.804040",
2010 "local_mtime": "2018-04-05 14:33:19.804839",
2011 "lost": 0,
2012 "flags": [
2013 "dirty",
2014 "omap",
2015 "data_digest",
2016 "omap_digest"
2017 ],
2018 "truncate_seq": 0,
2019 "truncate_size": 0,
2020 "data_digest": "0x2ddbf8f5",
2021 "omap_digest": "0xf5fba2c6",
2022 "expected_object_size": 0,
2023 "expected_write_size": 0,
2024 "alloc_hint_flags": 0,
2025 "manifest": {
2026 "type": 0
2027 },
2028 "watchers": {}
2029 },
2030 "data_digest": "0x2d4a11c2",
2031 "omap_digest": "0xf5fba2c6",
2032 "size": 9,
2033 "errors": [
2034 "data_digest_mismatch_info",
2035 "size_mismatch_info",
2036 "obj_size_info_mismatch"
2037 ],
2038 "osd": 1,
2039 "primary": true
2040 }
2041 ],
2042 "selected_object_info": {
2043 "oid": {
2044 "oid": "ROBJ1",
2045 "key": "",
2046 "snapid": -2,
2047 "hash": 1454963827,
2048 "max": 0,
2049 "pool": 3,
2050 "namespace": ""
2051 },
2052 "version": "51'58",
2053 "prior_version": "21'3",
2054 "last_reqid": "osd.1.0:57",
2055 "user_version": 3,
2056 "size": 7,
2057 "mtime": "2018-04-05 14:33:19.804040",
2058 "local_mtime": "2018-04-05 14:33:19.804839",
2059 "lost": 0,
2060 "flags": [
2061 "dirty",
2062 "omap",
2063 "data_digest",
2064 "omap_digest"
2065 ],
2066 "truncate_seq": 0,
2067 "truncate_size": 0,
2068 "data_digest": "0x2ddbf8f5",
2069 "omap_digest": "0xf5fba2c6",
2070 "expected_object_size": 0,
2071 "expected_write_size": 0,
2072 "alloc_hint_flags": 0,
2073 "manifest": {
2074 "type": 0
2075 },
2076 "watchers": {}
2077 },
2078 "union_shard_errors": [
2079 "data_digest_mismatch_info",
2080 "size_mismatch_info",
2081 "obj_size_info_mismatch"
2082 ],
2083 "errors": [
2084 "data_digest_mismatch",
2085 "size_mismatch"
2086 ],
2087 "object": {
2088 "version": 3,
2089 "snap": "head",
2090 "locator": "",
2091 "nspace": "",
2092 "name": "ROBJ1"
2093 }
2094 },
2095 {
2096 "shards": [
2097 {
2098 "data_digest": "0x2ddbf8f5",
2099 "omap_digest": "0xa8dd5adc",
2100 "size": 7,
2101 "errors": [
2102 "omap_digest_mismatch_info"
2103 ],
2104 "osd": 0,
2105 "primary": false
2106 },
2107 {
2108 "data_digest": "0x2ddbf8f5",
2109 "omap_digest": "0xa8dd5adc",
2110 "size": 7,
2111 "errors": [
2112 "omap_digest_mismatch_info"
2113 ],
2114 "osd": 1,
2115 "primary": true
2116 }
2117 ],
2118 "selected_object_info": {
2119 "alloc_hint_flags": 0,
2120 "data_digest": "0x2ddbf8f5",
2121 "expected_object_size": 0,
2122 "expected_write_size": 0,
2123 "flags": [
2124 "dirty",
2125 "omap",
2126 "data_digest",
2127 "omap_digest"
2128 ],
2129 "lost": 0,
2130 "manifest": {
2131 "type": 0
2132 },
2133 "oid": {
2134 "hash": 3174666125,
2135 "key": "",
2136 "max": 0,
2137 "namespace": "",
2138 "oid": "ROBJ10",
2139 "pool": 3,
2140 "snapid": -2
2141 },
2142 "omap_digest": "0xc2025a24",
2143 "size": 7,
2144 "truncate_seq": 0,
2145 "truncate_size": 0,
2146 "user_version": 30,
2147 "watchers": {}
2148 },
2149 "union_shard_errors": [
2150 "omap_digest_mismatch_info"
2151 ],
2152 "errors": [],
2153 "object": {
2154 "version": 30,
2155 "snap": "head",
2156 "locator": "",
2157 "nspace": "",
2158 "name": "ROBJ10"
2159 }
2160 },
2161 {
2162 "shards": [
2163 {
2164 "data_digest": "0x2ddbf8f5",
2165 "omap_digest": "0xa03cef03",
2166 "size": 7,
2167 "errors": [],
2168 "osd": 0,
2169 "primary": false
2170 },
2171 {
2172 "size": 7,
2173 "errors": [
2174 "read_error"
2175 ],
2176 "osd": 1,
2177 "primary": true
2178 }
2179 ],
2180 "selected_object_info": {
2181 "oid": {
2182 "oid": "ROBJ11",
2183 "key": "",
2184 "snapid": -2,
2185 "hash": 1828574689,
2186 "max": 0,
2187 "pool": 3,
2188 "namespace": ""
2189 },
2190 "version": "51'52",
2191 "prior_version": "41'33",
2192 "last_reqid": "osd.1.0:51",
2193 "user_version": 33,
2194 "size": 7,
2195 "mtime": "2018-04-05 14:33:26.761286",
2196 "local_mtime": "2018-04-05 14:33:26.762368",
2197 "lost": 0,
2198 "flags": [
2199 "dirty",
2200 "omap",
2201 "data_digest",
2202 "omap_digest"
2203 ],
2204 "truncate_seq": 0,
2205 "truncate_size": 0,
2206 "data_digest": "0x2ddbf8f5",
2207 "omap_digest": "0xa03cef03",
2208 "expected_object_size": 0,
2209 "expected_write_size": 0,
2210 "alloc_hint_flags": 0,
2211 "manifest": {
2212 "type": 0
2213 },
2214 "watchers": {}
2215 },
2216 "union_shard_errors": [
2217 "read_error"
2218 ],
2219 "errors": [],
2220 "object": {
2221 "version": 33,
2222 "snap": "head",
2223 "locator": "",
2224 "nspace": "",
2225 "name": "ROBJ11"
2226 }
2227 },
2228 {
2229 "shards": [
2230 {
2231 "errors": [
2232 "stat_error"
2233 ],
2234 "osd": 0,
2235 "primary": false
2236 },
2237 {
2238 "data_digest": "0x2ddbf8f5",
2239 "omap_digest": "0x067f306a",
2240 "size": 7,
2241 "errors": [],
2242 "osd": 1,
2243 "primary": true
2244 }
2245 ],
2246 "selected_object_info": {
2247 "oid": {
2248 "oid": "ROBJ12",
2249 "key": "",
2250 "snapid": -2,
2251 "hash": 3920199997,
2252 "max": 0,
2253 "pool": 3,
2254 "namespace": ""
2255 },
2256 "version": "51'56",
2257 "prior_version": "43'36",
2258 "last_reqid": "osd.1.0:55",
2259 "user_version": 36,
2260 "size": 7,
2261 "mtime": "2018-04-05 14:33:27.460958",
2262 "local_mtime": "2018-04-05 14:33:27.462109",
2263 "lost": 0,
2264 "flags": [
2265 "dirty",
2266 "omap",
2267 "data_digest",
2268 "omap_digest"
2269 ],
2270 "truncate_seq": 0,
2271 "truncate_size": 0,
2272 "data_digest": "0x2ddbf8f5",
2273 "omap_digest": "0x067f306a",
2274 "expected_object_size": 0,
2275 "expected_write_size": 0,
2276 "alloc_hint_flags": 0,
2277 "manifest": {
2278 "type": 0
2279 },
2280 "watchers": {}
2281 },
2282 "union_shard_errors": [
2283 "stat_error"
2284 ],
2285 "errors": [],
2286 "object": {
2287 "version": 36,
2288 "snap": "head",
2289 "locator": "",
2290 "nspace": "",
2291 "name": "ROBJ12"
2292 }
2293 },
2294 {
2295 "shards": [
2296 {
2297 "errors": [
2298 "stat_error"
2299 ],
2300 "osd": 0,
2301 "primary": false
2302 },
2303 {
2304 "size": 7,
2305 "errors": [
2306 "read_error"
2307 ],
2308 "osd": 1,
2309 "primary": true
2310 }
2311 ],
2312 "union_shard_errors": [
2313 "stat_error",
2314 "read_error"
2315 ],
2316 "errors": [],
2317 "object": {
2318 "version": 0,
2319 "snap": "head",
2320 "locator": "",
2321 "nspace": "",
2322 "name": "ROBJ13"
2323 }
2324 },
2325 {
2326 "shards": [
2327 {
2328 "object_info": "bad-val",
2329 "data_digest": "0x2ddbf8f5",
2330 "omap_digest": "0x4f14f849",
2331 "size": 7,
2332 "errors": [
2333 "info_corrupted"
2334 ],
2335 "osd": 0,
2336 "primary": false
2337 },
2338 {
2339 "data_digest": "0x2ddbf8f5",
2340 "omap_digest": "0x4f14f849",
2341 "size": 7,
2342 "errors": [
2343 "info_missing"
2344 ],
2345 "osd": 1,
2346 "primary": true
2347 }
2348 ],
2349 "union_shard_errors": [
2350 "info_missing",
2351 "info_corrupted"
2352 ],
2353 "errors": [],
2354 "object": {
2355 "version": 0,
2356 "snap": "head",
2357 "locator": "",
2358 "nspace": "",
2359 "name": "ROBJ14"
2360 }
2361 },
2362 {
2363 "shards": [
2364 {
2365 "object_info": {
2366 "oid": {
2367 "oid": "ROBJ15",
2368 "key": "",
2369 "snapid": -2,
2370 "hash": 504996876,
2371 "max": 0,
2372 "pool": 3,
2373 "namespace": ""
2374 },
2375 "version": "51'49",
2376 "prior_version": "49'45",
2377 "last_reqid": "osd.1.0:48",
2378 "user_version": 45,
2379 "size": 7,
2380 "mtime": "2018-04-05 14:33:29.498969",
2381 "local_mtime": "2018-04-05 14:33:29.499890",
2382 "lost": 0,
2383 "flags": [
2384 "dirty",
2385 "omap",
2386 "data_digest",
2387 "omap_digest"
2388 ],
2389 "truncate_seq": 0,
2390 "truncate_size": 0,
2391 "data_digest": "0x2ddbf8f5",
2392 "omap_digest": "0x2d2a4d6e",
2393 "expected_object_size": 0,
2394 "expected_write_size": 0,
2395 "alloc_hint_flags": 0,
2396 "manifest": {
2397 "type": 0
2398 },
2399 "watchers": {}
2400 },
2401 "data_digest": "0x2ddbf8f5",
2402 "omap_digest": "0x2d2a4d6e",
2403 "size": 7,
2404 "errors": [],
2405 "osd": 0,
2406 "primary": false
2407 },
2408 {
2409 "data_digest": "0x2ddbf8f5",
2410 "omap_digest": "0x2d2a4d6e",
2411 "size": 7,
2412 "errors": [
2413 "info_missing"
2414 ],
2415 "osd": 1,
2416 "primary": true
2417 }
2418 ],
2419 "selected_object_info": {
2420 "oid": {
2421 "oid": "ROBJ15",
2422 "key": "",
2423 "snapid": -2,
2424 "hash": 504996876,
2425 "max": 0,
2426 "pool": 3,
2427 "namespace": ""
2428 },
2429 "version": "51'49",
2430 "prior_version": "49'45",
2431 "last_reqid": "osd.1.0:48",
2432 "user_version": 45,
2433 "size": 7,
2434 "mtime": "2018-04-05 14:33:29.498969",
2435 "local_mtime": "2018-04-05 14:33:29.499890",
2436 "lost": 0,
2437 "flags": [
2438 "dirty",
2439 "omap",
2440 "data_digest",
2441 "omap_digest"
2442 ],
2443 "truncate_seq": 0,
2444 "truncate_size": 0,
2445 "data_digest": "0x2ddbf8f5",
2446 "omap_digest": "0x2d2a4d6e",
2447 "expected_object_size": 0,
2448 "expected_write_size": 0,
2449 "alloc_hint_flags": 0,
2450 "manifest": {
2451 "type": 0
2452 },
2453 "watchers": {}
2454 },
2455 "union_shard_errors": [
2456 "info_missing"
2457 ],
2458 "errors": [],
2459 "object": {
2460 "version": 45,
2461 "snap": "head",
2462 "locator": "",
2463 "nspace": "",
2464 "name": "ROBJ15"
2465 }
2466 },
2467 {
2468 "errors": [],
2469 "object": {
2470 "locator": "",
2471 "name": "ROBJ16",
2472 "nspace": "",
2473 "snap": "head",
2474 "version": 0
2475 },
2476 "shards": [
2477 {
2478 "data_digest": "0x2ddbf8f5",
2479 "errors": [
2480 "snapset_missing"
2481 ],
2482 "omap_digest": "0x8b699207",
2483 "osd": 0,
2484 "primary": false,
2485 "size": 7
2486 },
2487 {
2488 "snapset": "bad-val",
2489 "data_digest": "0x2ddbf8f5",
2490 "errors": [
2491 "snapset_corrupted"
2492 ],
2493 "omap_digest": "0x8b699207",
2494 "osd": 1,
2495 "primary": true,
2496 "size": 7
2497 }
2498 ],
2499 "union_shard_errors": [
2500 "snapset_missing",
2501 "snapset_corrupted"
2502 ]
2503 },
2504 {
2505 "errors": [],
2506 "object": {
2507 "locator": "",
2508 "name": "ROBJ17",
2509 "nspace": "",
2510 "snap": "head"
2511 },
2512 "selected_object_info": {
2513 "alloc_hint_flags": 0,
2514 "data_digest": "0x2ddbf8f5",
2515 "expected_object_size": 0,
2516 "expected_write_size": 0,
2517 "flags": [
2518 "dirty",
2519 "omap",
2520 "data_digest",
2521 "omap_digest"
2522 ],
2523 "lost": 0,
2524 "manifest": {
2525 "type": 0
2526 },
2527 "oid": {
2528 "hash": 1884071249,
2529 "key": "",
2530 "max": 0,
2531 "namespace": "",
2532 "oid": "ROBJ17",
2533 "pool": 3,
2534 "snapid": -2
2535 },
2536 "omap_digest": "0xe9572720",
2537 "size": 7,
2538 "truncate_seq": 0,
2539 "truncate_size": 0,
2540 "user_version": 51,
2541 "watchers": {}
2542 },
2543 "shards": [
2544 {
2545 "data_digest": "0x5af0c3ef",
2546 "errors": [
2547 "data_digest_mismatch_info"
2548 ],
2549 "omap_digest": "0xe9572720",
2550 "osd": 0,
2551 "primary": false,
2552 "size": 7
2553 },
2554 {
2555 "data_digest": "0x5af0c3ef",
2556 "errors": [
2557 "data_digest_mismatch_info"
2558 ],
2559 "omap_digest": "0xe9572720",
2560 "osd": 1,
2561 "primary": true,
2562 "size": 7
2563 }
2564 ],
2565 "union_shard_errors": [
2566 "data_digest_mismatch_info"
2567 ]
2568 },
2569 {
2570 "errors": [
2571 "object_info_inconsistency"
2572 ],
2573 "object": {
2574 "locator": "",
2575 "name": "ROBJ18",
2576 "nspace": "",
2577 "snap": "head"
2578 },
2579 "selected_object_info": {
2580 "alloc_hint_flags": 255,
2581 "data_digest": "0x2ddbf8f5",
2582 "expected_object_size": 0,
2583 "expected_write_size": 0,
2584 "flags": [
2585 "dirty",
2586 "omap",
2587 "data_digest",
2588 "omap_digest"
2589 ],
2590 "lost": 0,
2591 "manifest": {
2592 "type": 0
2593 },
2594 "oid": {
2595 "hash": 1629828556,
2596 "key": "",
2597 "max": 0,
2598 "namespace": "",
2599 "oid": "ROBJ18",
2600 "pool": 3,
2601 "snapid": -2
2602 },
2603 "omap_digest": "0xddc3680f",
2604 "size": 7,
2605 "truncate_seq": 0,
2606 "truncate_size": 0,
2607 "user_version": 54,
2608 "watchers": {}
2609 },
2610 "shards": [
2611 {
2612 "data_digest": "0xbd89c912",
2613 "errors": [
2614 "data_digest_mismatch_info"
2615 ],
2616 "object_info": {
2617 "alloc_hint_flags": 0,
2618 "data_digest": "0x2ddbf8f5",
2619 "expected_object_size": 0,
2620 "expected_write_size": 0,
2621 "flags": [
2622 "dirty",
2623 "omap",
2624 "data_digest",
2625 "omap_digest"
2626 ],
2627 "lost": 0,
2628 "manifest": {
2629 "type": 0
2630 },
2631 "oid": {
2632 "hash": 1629828556,
2633 "key": "",
2634 "max": 0,
2635 "namespace": "",
2636 "oid": "ROBJ18",
2637 "pool": 3,
2638 "snapid": -2
2639 },
2640 "omap_digest": "0xddc3680f",
2641 "size": 7,
2642 "truncate_seq": 0,
2643 "truncate_size": 0,
2644 "user_version": 54,
2645 "watchers": {}
2646 },
2647 "omap_digest": "0xddc3680f",
2648 "osd": 0,
2649 "primary": false,
2650 "size": 7
2651 },
2652 {
2653 "data_digest": "0xbd89c912",
2654 "errors": [
2655 "data_digest_mismatch_info"
2656 ],
2657 "object_info": {
2658 "alloc_hint_flags": 255,
2659 "data_digest": "0x2ddbf8f5",
2660 "expected_object_size": 0,
2661 "expected_write_size": 0,
2662 "flags": [
2663 "dirty",
2664 "omap",
2665 "data_digest",
2666 "omap_digest"
2667 ],
2668 "lost": 0,
2669 "manifest": {
2670 "type": 0
2671 },
2672 "oid": {
2673 "hash": 1629828556,
2674 "key": "",
2675 "max": 0,
2676 "namespace": "",
2677 "oid": "ROBJ18",
2678 "pool": 3,
2679 "snapid": -2
2680 },
2681 "omap_digest": "0xddc3680f",
2682 "size": 7,
2683 "truncate_seq": 0,
2684 "truncate_size": 0,
2685 "user_version": 54,
2686 "watchers": {}
2687 },
2688 "omap_digest": "0xddc3680f",
2689 "osd": 1,
2690 "primary": true,
2691 "size": 7
2692 }
2693 ],
2694 "union_shard_errors": [
2695 "data_digest_mismatch_info"
2696 ]
2697 },
2698 {
2699 "shards": [
2700 {
2701 "data_digest": "0x578a4830",
2702 "omap_digest": "0xf8e11918",
2703 "size": 7,
2704 "errors": [
2705 "data_digest_mismatch_info"
2706 ],
2707 "osd": 0,
2708 "primary": false
2709 },
2710 {
2711 "data_digest": "0x2ddbf8f5",
2712 "omap_digest": "0xf8e11918",
2713 "size": 7,
2714 "errors": [],
2715 "osd": 1,
2716 "primary": true
2717 }
2718 ],
2719 "selected_object_info": {
2720 "oid": {
2721 "oid": "ROBJ2",
2722 "key": "",
2723 "snapid": -2,
2724 "hash": 2026323607,
2725 "max": 0,
2726 "pool": 3,
2727 "namespace": ""
2728 },
2729 "version": "51'60",
2730 "prior_version": "23'6",
2731 "last_reqid": "osd.1.0:59",
2732 "user_version": 6,
2733 "size": 7,
2734 "mtime": "2018-04-05 14:33:20.498756",
2735 "local_mtime": "2018-04-05 14:33:20.499704",
2736 "lost": 0,
2737 "flags": [
2738 "dirty",
2739 "omap",
2740 "data_digest",
2741 "omap_digest"
2742 ],
2743 "truncate_seq": 0,
2744 "truncate_size": 0,
2745 "data_digest": "0x2ddbf8f5",
2746 "omap_digest": "0xf8e11918",
2747 "expected_object_size": 0,
2748 "expected_write_size": 0,
2749 "alloc_hint_flags": 0,
2750 "manifest": {
2751 "type": 0
2752 },
2753 "watchers": {}
2754 },
2755 "union_shard_errors": [
2756 "data_digest_mismatch_info"
2757 ],
2758 "errors": [
2759 "data_digest_mismatch"
2760 ],
2761 "object": {
2762 "version": 6,
2763 "snap": "head",
2764 "locator": "",
2765 "nspace": "",
2766 "name": "ROBJ2"
2767 }
2768 },
2769 {
2770 "shards": [
2771 {
2772 "data_digest": "0x2ddbf8f5",
2773 "omap_digest": "0x00b35dfd",
2774 "size": 7,
2775 "errors": [],
2776 "osd": 0,
2777 "primary": false
2778 },
2779 {
2780 "errors": [
2781 "missing"
2782 ],
2783 "osd": 1,
2784 "primary": true
2785 }
2786 ],
2787 "selected_object_info": {
2788 "oid": {
2789 "oid": "ROBJ3",
2790 "key": "",
2791 "snapid": -2,
2792 "hash": 625845583,
2793 "max": 0,
2794 "pool": 3,
2795 "namespace": ""
2796 },
2797 "version": "51'61",
2798 "prior_version": "25'9",
2799 "last_reqid": "osd.1.0:60",
2800 "user_version": 9,
2801 "size": 7,
2802 "mtime": "2018-04-05 14:33:21.189382",
2803 "local_mtime": "2018-04-05 14:33:21.190446",
2804 "lost": 0,
2805 "flags": [
2806 "dirty",
2807 "omap",
2808 "data_digest",
2809 "omap_digest"
2810 ],
2811 "truncate_seq": 0,
2812 "truncate_size": 0,
2813 "data_digest": "0x2ddbf8f5",
2814 "omap_digest": "0x00b35dfd",
2815 "expected_object_size": 0,
2816 "expected_write_size": 0,
2817 "alloc_hint_flags": 0,
2818 "manifest": {
2819 "type": 0
2820 },
2821 "watchers": {}
2822 },
2823 "union_shard_errors": [
2824 "missing"
2825 ],
2826 "errors": [],
2827 "object": {
2828 "version": 9,
2829 "snap": "head",
2830 "locator": "",
2831 "nspace": "",
2832 "name": "ROBJ3"
2833 }
2834 },
2835 {
2836 "shards": [
2837 {
2838 "data_digest": "0x2ddbf8f5",
2839 "omap_digest": "0xd7178dfe",
2840 "size": 7,
2841 "errors": [
2842 "omap_digest_mismatch_info"
2843 ],
2844 "osd": 0,
2845 "primary": false
2846 },
2847 {
2848 "data_digest": "0x2ddbf8f5",
2849 "omap_digest": "0xe2d46ea4",
2850 "size": 7,
2851 "errors": [],
2852 "osd": 1,
2853 "primary": true
2854 }
2855 ],
2856 "selected_object_info": {
2857 "oid": {
2858 "oid": "ROBJ4",
2859 "key": "",
2860 "snapid": -2,
2861 "hash": 2360875311,
2862 "max": 0,
2863 "pool": 3,
2864 "namespace": ""
2865 },
2866 "version": "51'62",
2867 "prior_version": "27'12",
2868 "last_reqid": "osd.1.0:61",
2869 "user_version": 12,
2870 "size": 7,
2871 "mtime": "2018-04-05 14:33:21.862313",
2872 "local_mtime": "2018-04-05 14:33:21.863261",
2873 "lost": 0,
2874 "flags": [
2875 "dirty",
2876 "omap",
2877 "data_digest",
2878 "omap_digest"
2879 ],
2880 "truncate_seq": 0,
2881 "truncate_size": 0,
2882 "data_digest": "0x2ddbf8f5",
2883 "omap_digest": "0xe2d46ea4",
2884 "expected_object_size": 0,
2885 "expected_write_size": 0,
2886 "alloc_hint_flags": 0,
2887 "manifest": {
2888 "type": 0
2889 },
2890 "watchers": {}
2891 },
2892 "union_shard_errors": [
2893 "omap_digest_mismatch_info"
2894 ],
2895 "errors": [
2896 "omap_digest_mismatch"
2897 ],
2898 "object": {
2899 "version": 12,
2900 "snap": "head",
2901 "locator": "",
2902 "nspace": "",
2903 "name": "ROBJ4"
2904 }
2905 },
2906 {
2907 "shards": [
2908 {
2909 "data_digest": "0x2ddbf8f5",
2910 "omap_digest": "0x1a862a41",
2911 "size": 7,
2912 "errors": [],
2913 "osd": 0,
2914 "primary": false
2915 },
2916 {
2917 "data_digest": "0x2ddbf8f5",
2918 "omap_digest": "0x06cac8f6",
2919 "size": 7,
2920 "errors": [
2921 "omap_digest_mismatch_info"
2922 ],
2923 "osd": 1,
2924 "primary": true
2925 }
2926 ],
2927 "selected_object_info": {
2928 "oid": {
2929 "oid": "ROBJ5",
2930 "key": "",
2931 "snapid": -2,
2932 "hash": 2334915887,
2933 "max": 0,
2934 "pool": 3,
2935 "namespace": ""
2936 },
2937 "version": "51'63",
2938 "prior_version": "29'15",
2939 "last_reqid": "osd.1.0:62",
2940 "user_version": 15,
2941 "size": 7,
2942 "mtime": "2018-04-05 14:33:22.589300",
2943 "local_mtime": "2018-04-05 14:33:22.590376",
2944 "lost": 0,
2945 "flags": [
2946 "dirty",
2947 "omap",
2948 "data_digest",
2949 "omap_digest"
2950 ],
2951 "truncate_seq": 0,
2952 "truncate_size": 0,
2953 "data_digest": "0x2ddbf8f5",
2954 "omap_digest": "0x1a862a41",
2955 "expected_object_size": 0,
2956 "expected_write_size": 0,
2957 "alloc_hint_flags": 0,
2958 "manifest": {
2959 "type": 0
2960 },
2961 "watchers": {}
2962 },
2963 "union_shard_errors": [
2964 "omap_digest_mismatch_info"
2965 ],
2966 "errors": [
2967 "omap_digest_mismatch"
2968 ],
2969 "object": {
2970 "version": 15,
2971 "snap": "head",
2972 "locator": "",
2973 "nspace": "",
2974 "name": "ROBJ5"
2975 }
2976 },
2977 {
2978 "shards": [
2979 {
2980 "data_digest": "0x2ddbf8f5",
2981 "omap_digest": "0x689ee887",
2982 "size": 7,
2983 "errors": [
2984 "omap_digest_mismatch_info"
2985 ],
2986 "osd": 0,
2987 "primary": false
2988 },
2989 {
2990 "data_digest": "0x2ddbf8f5",
2991 "omap_digest": "0x179c919f",
2992 "size": 7,
2993 "errors": [],
2994 "osd": 1,
2995 "primary": true
2996 }
2997 ],
2998 "selected_object_info": {
2999 "oid": {
3000 "oid": "ROBJ6",
3001 "key": "",
3002 "snapid": -2,
3003 "hash": 390610085,
3004 "max": 0,
3005 "pool": 3,
3006 "namespace": ""
3007 },
3008 "version": "51'54",
3009 "prior_version": "31'18",
3010 "last_reqid": "osd.1.0:53",
3011 "user_version": 18,
3012 "size": 7,
3013 "mtime": "2018-04-05 14:33:23.289188",
3014 "local_mtime": "2018-04-05 14:33:23.290130",
3015 "lost": 0,
3016 "flags": [
3017 "dirty",
3018 "omap",
3019 "data_digest",
3020 "omap_digest"
3021 ],
3022 "truncate_seq": 0,
3023 "truncate_size": 0,
3024 "data_digest": "0x2ddbf8f5",
3025 "omap_digest": "0x179c919f",
3026 "expected_object_size": 0,
3027 "expected_write_size": 0,
3028 "alloc_hint_flags": 0,
3029 "manifest": {
3030 "type": 0
3031 },
3032 "watchers": {}
3033 },
3034 "union_shard_errors": [
3035 "omap_digest_mismatch_info"
3036 ],
3037 "errors": [
3038 "omap_digest_mismatch"
3039 ],
3040 "object": {
3041 "version": 18,
3042 "snap": "head",
3043 "locator": "",
3044 "nspace": "",
3045 "name": "ROBJ6"
3046 }
3047 },
3048 {
3049 "shards": [
3050 {
3051 "data_digest": "0x2ddbf8f5",
3052 "omap_digest": "0xefced57a",
3053 "size": 7,
3054 "errors": [],
3055 "osd": 0,
3056 "primary": false
3057 },
3058 {
3059 "data_digest": "0x2ddbf8f5",
3060 "omap_digest": "0x6a73cc07",
3061 "size": 7,
3062 "errors": [
3063 "omap_digest_mismatch_info"
3064 ],
3065 "osd": 1,
3066 "primary": true
3067 }
3068 ],
3069 "selected_object_info": {
3070 "oid": {
3071 "oid": "ROBJ7",
3072 "key": "",
3073 "snapid": -2,
3074 "hash": 3529485009,
3075 "max": 0,
3076 "pool": 3,
3077 "namespace": ""
3078 },
3079 "version": "51'53",
3080 "prior_version": "33'21",
3081 "last_reqid": "osd.1.0:52",
3082 "user_version": 21,
3083 "size": 7,
3084 "mtime": "2018-04-05 14:33:23.979658",
3085 "local_mtime": "2018-04-05 14:33:23.980731",
3086 "lost": 0,
3087 "flags": [
3088 "dirty",
3089 "omap",
3090 "data_digest",
3091 "omap_digest"
3092 ],
3093 "truncate_seq": 0,
3094 "truncate_size": 0,
3095 "data_digest": "0x2ddbf8f5",
3096 "omap_digest": "0xefced57a",
3097 "expected_object_size": 0,
3098 "expected_write_size": 0,
3099 "alloc_hint_flags": 0,
3100 "manifest": {
3101 "type": 0
3102 },
3103 "watchers": {}
3104 },
3105 "union_shard_errors": [
3106 "omap_digest_mismatch_info"
3107 ],
3108 "errors": [
3109 "omap_digest_mismatch"
3110 ],
3111 "object": {
3112 "version": 21,
3113 "snap": "head",
3114 "locator": "",
3115 "nspace": "",
3116 "name": "ROBJ7"
3117 }
3118 },
3119 {
3120 "shards": [
3121 {
3122 "attrs": [
3123 {
3124 "Base64": false,
3125 "value": "bad-val",
3126 "name": "key1-ROBJ8"
3127 },
3128 {
3129 "Base64": false,
3130 "value": "val2-ROBJ8",
3131 "name": "key2-ROBJ8"
3132 }
3133 ],
3134 "data_digest": "0x2ddbf8f5",
3135 "omap_digest": "0xd6be81dc",
3136 "size": 7,
3137 "errors": [],
3138 "osd": 0,
3139 "primary": false
3140 },
3141 {
3142 "attrs": [
3143 {
3144 "Base64": false,
3145 "value": "val1-ROBJ8",
3146 "name": "key1-ROBJ8"
3147 },
3148 {
3149 "Base64": false,
3150 "value": "val3-ROBJ8",
3151 "name": "key3-ROBJ8"
3152 }
3153 ],
3154 "data_digest": "0x2ddbf8f5",
3155 "omap_digest": "0xd6be81dc",
3156 "size": 7,
3157 "errors": [],
3158 "osd": 1,
3159 "primary": true
3160 }
3161 ],
3162 "selected_object_info": {
3163 "oid": {
3164 "oid": "ROBJ8",
3165 "key": "",
3166 "snapid": -2,
3167 "hash": 2359695969,
3168 "max": 0,
3169 "pool": 3,
3170 "namespace": ""
3171 },
3172 "version": "79'66",
3173 "prior_version": "79'65",
3174 "last_reqid": "client.4554.0:1",
3175 "user_version": 74,
3176 "size": 7,
3177 "mtime": "2018-04-05 14:34:05.598688",
3178 "local_mtime": "2018-04-05 14:34:05.599698",
3179 "lost": 0,
3180 "flags": [
3181 "dirty",
3182 "omap",
3183 "data_digest",
3184 "omap_digest"
3185 ],
3186 "truncate_seq": 0,
3187 "truncate_size": 0,
3188 "data_digest": "0x2ddbf8f5",
3189 "omap_digest": "0xd6be81dc",
3190 "expected_object_size": 0,
3191 "expected_write_size": 0,
3192 "alloc_hint_flags": 0,
3193 "manifest": {
3194 "type": 0
3195 },
3196 "watchers": {}
3197 },
3198 "union_shard_errors": [],
3199 "errors": [
3200 "attr_value_mismatch",
3201 "attr_name_mismatch"
3202 ],
3203 "object": {
3204 "version": 66,
3205 "snap": "head",
3206 "locator": "",
3207 "nspace": "",
3208 "name": "ROBJ8"
3209 }
3210 },
3211 {
3212 "shards": [
3213 {
3214 "object_info": {
3215 "oid": {
3216 "oid": "ROBJ9",
3217 "key": "",
3218 "snapid": -2,
3219 "hash": 537189375,
3220 "max": 0,
3221 "pool": 3,
3222 "namespace": ""
3223 },
3224 "version": "51'64",
3225 "prior_version": "37'27",
3226 "last_reqid": "osd.1.0:63",
3227 "user_version": 27,
3228 "size": 7,
3229 "mtime": "2018-04-05 14:33:25.352485",
3230 "local_mtime": "2018-04-05 14:33:25.353746",
3231 "lost": 0,
3232 "flags": [
3233 "dirty",
3234 "omap",
3235 "data_digest",
3236 "omap_digest"
3237 ],
3238 "truncate_seq": 0,
3239 "truncate_size": 0,
3240 "data_digest": "0x2ddbf8f5",
3241 "omap_digest": "0x2eecc539",
3242 "expected_object_size": 0,
3243 "expected_write_size": 0,
3244 "alloc_hint_flags": 0,
3245 "manifest": {
3246 "type": 0
3247 },
3248 "watchers": {}
3249 },
3250 "data_digest": "0x1f26fb26",
3251 "omap_digest": "0x2eecc539",
3252 "size": 3,
3253 "errors": [
3254 "obj_size_info_mismatch"
3255 ],
3256 "osd": 0,
3257 "primary": false
3258 },
3259 {
3260 "object_info": {
3261 "oid": {
3262 "oid": "ROBJ9",
3263 "key": "",
3264 "snapid": -2,
3265 "hash": 537189375,
3266 "max": 0,
3267 "pool": 3,
3268 "namespace": ""
3269 },
3270 "version": "119'68",
3271 "prior_version": "51'64",
3272 "last_reqid": "client.4834.0:1",
3273 "user_version": 76,
3274 "size": 3,
3275 "mtime": "2018-04-05 14:35:01.500659",
3276 "local_mtime": "2018-04-05 14:35:01.502117",
3277 "lost": 0,
3278 "flags": [
3279 "dirty",
3280 "omap",
3281 "data_digest",
3282 "omap_digest"
3283 ],
3284 "truncate_seq": 0,
3285 "truncate_size": 0,
3286 "data_digest": "0x1f26fb26",
3287 "omap_digest": "0x2eecc539",
3288 "expected_object_size": 0,
3289 "expected_write_size": 0,
3290 "alloc_hint_flags": 0,
3291 "manifest": {
3292 "type": 0
3293 },
3294 "watchers": {}
3295 },
3296 "data_digest": "0x1f26fb26",
3297 "omap_digest": "0x2eecc539",
3298 "size": 3,
3299 "errors": [],
3300 "osd": 1,
3301 "primary": true
3302 }
3303 ],
3304 "selected_object_info": {
3305 "oid": {
3306 "oid": "ROBJ9",
3307 "key": "",
3308 "snapid": -2,
3309 "hash": 537189375,
3310 "max": 0,
3311 "pool": 3,
3312 "namespace": ""
3313 },
3314 "version": "119'68",
3315 "prior_version": "51'64",
3316 "last_reqid": "client.4834.0:1",
3317 "user_version": 76,
3318 "size": 3,
3319 "mtime": "2018-04-05 14:35:01.500659",
3320 "local_mtime": "2018-04-05 14:35:01.502117",
3321 "lost": 0,
3322 "flags": [
3323 "dirty",
3324 "omap",
3325 "data_digest",
3326 "omap_digest"
3327 ],
3328 "truncate_seq": 0,
3329 "truncate_size": 0,
3330 "data_digest": "0x1f26fb26",
3331 "omap_digest": "0x2eecc539",
3332 "expected_object_size": 0,
3333 "expected_write_size": 0,
3334 "alloc_hint_flags": 0,
3335 "manifest": {
3336 "type": 0
3337 },
3338 "watchers": {}
3339 },
3340 "union_shard_errors": [
3341 "obj_size_info_mismatch"
3342 ],
3343 "errors": [
3344 "object_info_inconsistency"
3345 ],
3346 "object": {
3347 "version": 68,
3348 "snap": "head",
3349 "locator": "",
3350 "nspace": "",
3351 "name": "ROBJ9"
3352 }
3353 }
3354 ],
3355 "epoch": 0
3356 }
3357 EOF
3358
3359 jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
3360 multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
3361 if test $getjson = "yes"
3362 then
3363 jq '.' $dir/json > save2.json
3364 fi
3365
3366 if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
3367 then
3368 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
3369 fi
3370
3371 repair $pg
3372 wait_for_clean
3373
3374 # This hangs if the repair doesn't work
3375 timeout 30 rados -p $poolname get ROBJ17 $dir/robj17.out || return 1
3376 timeout 30 rados -p $poolname get ROBJ18 $dir/robj18.out || return 1
3377 # Even though we couldn't repair all of the introduced errors, we can fix ROBJ17
3378 diff -q $dir/new.ROBJ17 $dir/robj17.out || return 1
3379 rm -f $dir/new.ROBJ17 $dir/robj17.out || return 1
3380 diff -q $dir/new.ROBJ18 $dir/robj18.out || return 1
3381 rm -f $dir/new.ROBJ18 $dir/robj18.out || return 1
3382
3383 if [ $ERRORS != "0" ];
3384 then
3385 echo "TEST FAILED WITH $ERRORS ERRORS"
3386 return 1
3387 fi
3388
3389 ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it
3390 teardown $dir || return 1
3391 }
3392
3393
3394 #
3395 # Test scrub errors for an erasure coded pool
3396 #
3397 function corrupt_scrub_erasure() {
3398 local dir=$1
3399 local allow_overwrites=$2
3400 local poolname=ecpool
3401 local total_objs=7
3402
3403 setup $dir || return 1
3404 run_mon $dir a || return 1
3405 run_mgr $dir x || return 1
3406 for id in $(seq 0 2) ; do
3407 if [ "$allow_overwrites" = "true" ]; then
3408 run_osd_bluestore $dir $id || return 1
3409 else
3410 run_osd $dir $id || return 1
3411 fi
3412 done
3413 create_rbd_pool || return 1
3414 create_pool foo 1
3415
3416 create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1
3417 wait_for_clean || return 1
3418
3419 for i in $(seq 1 $total_objs) ; do
3420 objname=EOBJ${i}
3421 add_something $dir $poolname $objname || return 1
3422
3423 local osd=$(expr $i % 2)
3424
3425 case $i in
3426 1)
3427 # Size (deep scrub data_digest too)
3428 local payload=UVWXYZZZ
3429 echo $payload > $dir/CORRUPT
3430 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
3431 ;;
3432
3433 2)
3434 # Corrupt EC shard
3435 dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1
3436 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
3437 ;;
3438
3439 3)
3440 # missing
3441 objectstore_tool $dir $osd $objname remove || return 1
3442 ;;
3443
3444 4)
3445 rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
3446 rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
3447
3448 # Break xattrs
3449 echo -n bad-val > $dir/bad-val
3450 objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
3451 objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
3452 echo -n val3-$objname > $dir/newval
3453 objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
3454 rm $dir/bad-val $dir/newval
3455 ;;
3456
3457 5)
3458 # Corrupt EC shard
3459 dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2
3460 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
3461 ;;
3462
3463 6)
3464 objectstore_tool $dir 0 $objname rm-attr hinfo_key || return 1
3465 echo -n bad-val > $dir/bad-val
3466 objectstore_tool $dir 1 $objname set-attr hinfo_key $dir/bad-val || return 1
3467 ;;
3468
3469 7)
3470 local payload=MAKETHISDIFFERENTFROMOTHEROBJECTS
3471 echo $payload > $dir/DIFFERENT
3472 rados --pool $poolname put $objname $dir/DIFFERENT || return 1
3473
3474 # Get hinfo_key from EOBJ1
3475 objectstore_tool $dir 0 EOBJ1 get-attr hinfo_key > $dir/hinfo
3476 objectstore_tool $dir 0 $objname set-attr hinfo_key $dir/hinfo || return 1
3477 rm -f $dir/hinfo
3478 ;;
3479
3480 esac
3481 done
3482
3483 local pg=$(get_pg $poolname EOBJ0)
3484
3485 pg_scrub $pg
3486
3487 rados list-inconsistent-pg $poolname > $dir/json || return 1
3488 # Check pg count
3489 test $(jq '. | length' $dir/json) = "1" || return 1
3490 # Check pgid
3491 test $(jq -r '.[0]' $dir/json) = $pg || return 1
3492
3493 rados list-inconsistent-obj $pg > $dir/json || return 1
3494 # Get epoch for repair-get requests
3495 epoch=$(jq .epoch $dir/json)
3496
3497 jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
3498 {
3499 "inconsistents": [
3500 {
3501 "shards": [
3502 {
3503 "size": 2048,
3504 "errors": [],
3505 "shard": 2,
3506 "osd": 0,
3507 "primary": false
3508 },
3509 {
3510 "object_info": {
3511 "oid": {
3512 "oid": "EOBJ1",
3513 "key": "",
3514 "snapid": -2,
3515 "hash": 560836233,
3516 "max": 0,
3517 "pool": 3,
3518 "namespace": ""
3519 },
3520 "version": "27'1",
3521 "prior_version": "0'0",
3522 "last_reqid": "client.4184.0:1",
3523 "user_version": 1,
3524 "size": 7,
3525 "mtime": "",
3526 "local_mtime": "",
3527 "lost": 0,
3528 "flags": [
3529 "dirty",
3530 "data_digest"
3531 ],
3532 "truncate_seq": 0,
3533 "truncate_size": 0,
3534 "data_digest": "0x2ddbf8f5",
3535 "omap_digest": "0xffffffff",
3536 "expected_object_size": 0,
3537 "expected_write_size": 0,
3538 "alloc_hint_flags": 0,
3539 "manifest": {
3540 "type": 0
3541 },
3542 "watchers": {}
3543 },
3544 "size": 9,
3545 "shard": 0,
3546 "errors": [
3547 "size_mismatch_info",
3548 "obj_size_info_mismatch"
3549 ],
3550 "osd": 1,
3551 "primary": true
3552 },
3553 {
3554 "size": 2048,
3555 "shard": 1,
3556 "errors": [],
3557 "osd": 2,
3558 "primary": false
3559 }
3560 ],
3561 "selected_object_info": {
3562 "oid": {
3563 "oid": "EOBJ1",
3564 "key": "",
3565 "snapid": -2,
3566 "hash": 560836233,
3567 "max": 0,
3568 "pool": 3,
3569 "namespace": ""
3570 },
3571 "version": "27'1",
3572 "prior_version": "0'0",
3573 "last_reqid": "client.4184.0:1",
3574 "user_version": 1,
3575 "size": 7,
3576 "mtime": "",
3577 "local_mtime": "",
3578 "lost": 0,
3579 "flags": [
3580 "dirty",
3581 "data_digest"
3582 ],
3583 "truncate_seq": 0,
3584 "truncate_size": 0,
3585 "data_digest": "0x2ddbf8f5",
3586 "omap_digest": "0xffffffff",
3587 "expected_object_size": 0,
3588 "expected_write_size": 0,
3589 "alloc_hint_flags": 0,
3590 "manifest": {
3591 "type": 0
3592 },
3593 "watchers": {}
3594 },
3595 "union_shard_errors": [
3596 "size_mismatch_info",
3597 "obj_size_info_mismatch"
3598 ],
3599 "errors": [
3600 "size_mismatch"
3601 ],
3602 "object": {
3603 "version": 1,
3604 "snap": "head",
3605 "locator": "",
3606 "nspace": "",
3607 "name": "EOBJ1"
3608 }
3609 },
3610 {
3611 "shards": [
3612 {
3613 "size": 2048,
3614 "errors": [],
3615 "shard": 2,
3616 "osd": 0,
3617 "primary": false
3618 },
3619 {
3620 "shard": 0,
3621 "errors": [
3622 "missing"
3623 ],
3624 "osd": 1,
3625 "primary": true
3626 },
3627 {
3628 "size": 2048,
3629 "shard": 1,
3630 "errors": [],
3631 "osd": 2,
3632 "primary": false
3633 }
3634 ],
3635 "selected_object_info": {
3636 "oid": {
3637 "oid": "EOBJ3",
3638 "key": "",
3639 "snapid": -2,
3640 "hash": 3125668237,
3641 "max": 0,
3642 "pool": 3,
3643 "namespace": ""
3644 },
3645 "version": "39'3",
3646 "prior_version": "0'0",
3647 "last_reqid": "client.4252.0:1",
3648 "user_version": 3,
3649 "size": 7,
3650 "mtime": "",
3651 "local_mtime": "",
3652 "lost": 0,
3653 "flags": [
3654 "dirty",
3655 "data_digest"
3656 ],
3657 "truncate_seq": 0,
3658 "truncate_size": 0,
3659 "data_digest": "0x2ddbf8f5",
3660 "omap_digest": "0xffffffff",
3661 "expected_object_size": 0,
3662 "expected_write_size": 0,
3663 "alloc_hint_flags": 0,
3664 "manifest": {
3665 "type": 0
3666 },
3667 "watchers": {}
3668 },
3669 "union_shard_errors": [
3670 "missing"
3671 ],
3672 "errors": [],
3673 "object": {
3674 "version": 3,
3675 "snap": "head",
3676 "locator": "",
3677 "nspace": "",
3678 "name": "EOBJ3"
3679 }
3680 },
3681 {
3682 "shards": [
3683 {
3684 "attrs": [
3685 {
3686 "Base64": false,
3687 "value": "bad-val",
3688 "name": "key1-EOBJ4"
3689 },
3690 {
3691 "Base64": false,
3692 "value": "val2-EOBJ4",
3693 "name": "key2-EOBJ4"
3694 }
3695 ],
3696 "size": 2048,
3697 "errors": [],
3698 "shard": 2,
3699 "osd": 0,
3700 "primary": false
3701 },
3702 {
3703 "osd": 1,
3704 "primary": true,
3705 "shard": 0,
3706 "errors": [],
3707 "size": 2048,
3708 "attrs": [
3709 {
3710 "Base64": false,
3711 "value": "val1-EOBJ4",
3712 "name": "key1-EOBJ4"
3713 },
3714 {
3715 "Base64": false,
3716 "value": "val2-EOBJ4",
3717 "name": "key2-EOBJ4"
3718 }
3719 ]
3720 },
3721 {
3722 "osd": 2,
3723 "primary": false,
3724 "shard": 1,
3725 "errors": [],
3726 "size": 2048,
3727 "attrs": [
3728 {
3729 "Base64": false,
3730 "value": "val1-EOBJ4",
3731 "name": "key1-EOBJ4"
3732 },
3733 {
3734 "Base64": false,
3735 "value": "val3-EOBJ4",
3736 "name": "key3-EOBJ4"
3737 }
3738 ]
3739 }
3740 ],
3741 "selected_object_info": {
3742 "oid": {
3743 "oid": "EOBJ4",
3744 "key": "",
3745 "snapid": -2,
3746 "hash": 1618759290,
3747 "max": 0,
3748 "pool": 3,
3749 "namespace": ""
3750 },
3751 "version": "45'6",
3752 "prior_version": "45'5",
3753 "last_reqid": "client.4294.0:1",
3754 "user_version": 6,
3755 "size": 7,
3756 "mtime": "",
3757 "local_mtime": "",
3758 "lost": 0,
3759 "flags": [
3760 "dirty",
3761 "data_digest"
3762 ],
3763 "truncate_seq": 0,
3764 "truncate_size": 0,
3765 "data_digest": "0x2ddbf8f5",
3766 "omap_digest": "0xffffffff",
3767 "expected_object_size": 0,
3768 "expected_write_size": 0,
3769 "alloc_hint_flags": 0,
3770 "manifest": {
3771 "type": 0
3772 },
3773 "watchers": {}
3774 },
3775 "union_shard_errors": [],
3776 "errors": [
3777 "attr_value_mismatch",
3778 "attr_name_mismatch"
3779 ],
3780 "object": {
3781 "version": 6,
3782 "snap": "head",
3783 "locator": "",
3784 "nspace": "",
3785 "name": "EOBJ4"
3786 }
3787 },
3788 {
3789 "shards": [
3790 {
3791 "size": 2048,
3792 "errors": [],
3793 "shard": 2,
3794 "osd": 0,
3795 "primary": false
3796 },
3797 {
3798 "object_info": {
3799 "oid": {
3800 "oid": "EOBJ5",
3801 "key": "",
3802 "snapid": -2,
3803 "hash": 2918945441,
3804 "max": 0,
3805 "pool": 3,
3806 "namespace": ""
3807 },
3808 "version": "59'7",
3809 "prior_version": "0'0",
3810 "last_reqid": "client.4382.0:1",
3811 "user_version": 7,
3812 "size": 7,
3813 "mtime": "",
3814 "local_mtime": "",
3815 "lost": 0,
3816 "flags": [
3817 "dirty",
3818 "data_digest"
3819 ],
3820 "truncate_seq": 0,
3821 "truncate_size": 0,
3822 "data_digest": "0x2ddbf8f5",
3823 "omap_digest": "0xffffffff",
3824 "expected_object_size": 0,
3825 "expected_write_size": 0,
3826 "alloc_hint_flags": 0,
3827 "manifest": {
3828 "type": 0
3829 },
3830 "watchers": {}
3831 },
3832 "size": 4096,
3833 "shard": 0,
3834 "errors": [
3835 "size_mismatch_info",
3836 "obj_size_info_mismatch"
3837 ],
3838 "osd": 1,
3839 "primary": true
3840 },
3841 {
3842 "size": 2048,
3843 "shard": 1,
3844 "errors": [],
3845 "osd": 2,
3846 "primary": false
3847 }
3848 ],
3849 "selected_object_info": {
3850 "oid": {
3851 "oid": "EOBJ5",
3852 "key": "",
3853 "snapid": -2,
3854 "hash": 2918945441,
3855 "max": 0,
3856 "pool": 3,
3857 "namespace": ""
3858 },
3859 "version": "59'7",
3860 "prior_version": "0'0",
3861 "last_reqid": "client.4382.0:1",
3862 "user_version": 7,
3863 "size": 7,
3864 "mtime": "",
3865 "local_mtime": "",
3866 "lost": 0,
3867 "flags": [
3868 "dirty",
3869 "data_digest"
3870 ],
3871 "truncate_seq": 0,
3872 "truncate_size": 0,
3873 "data_digest": "0x2ddbf8f5",
3874 "omap_digest": "0xffffffff",
3875 "expected_object_size": 0,
3876 "expected_write_size": 0,
3877 "alloc_hint_flags": 0,
3878 "manifest": {
3879 "type": 0
3880 },
3881 "watchers": {}
3882 },
3883 "union_shard_errors": [
3884 "size_mismatch_info",
3885 "obj_size_info_mismatch"
3886 ],
3887 "errors": [
3888 "size_mismatch"
3889 ],
3890 "object": {
3891 "version": 7,
3892 "snap": "head",
3893 "locator": "",
3894 "nspace": "",
3895 "name": "EOBJ5"
3896 }
3897 },
3898 {
3899 "errors": [],
3900 "object": {
3901 "locator": "",
3902 "name": "EOBJ6",
3903 "nspace": "",
3904 "snap": "head",
3905 "version": 8
3906 },
3907 "selected_object_info": {
3908 "oid": {
3909 "oid": "EOBJ6",
3910 "key": "",
3911 "snapid": -2,
3912 "hash": 3050890866,
3913 "max": 0,
3914 "pool": 3,
3915 "namespace": ""
3916 },
3917 "version": "65'8",
3918 "prior_version": "0'0",
3919 "last_reqid": "client.4418.0:1",
3920 "user_version": 8,
3921 "size": 7,
3922 "mtime": "",
3923 "local_mtime": "",
3924 "lost": 0,
3925 "flags": [
3926 "dirty",
3927 "data_digest"
3928 ],
3929 "truncate_seq": 0,
3930 "truncate_size": 0,
3931 "data_digest": "0x2ddbf8f5",
3932 "omap_digest": "0xffffffff",
3933 "expected_object_size": 0,
3934 "expected_write_size": 0,
3935 "alloc_hint_flags": 0,
3936 "manifest": {
3937 "type": 0
3938 },
3939 "watchers": {}
3940 },
3941 "shards": [
3942 {
3943 "errors": [
3944 "hinfo_missing"
3945 ],
3946 "osd": 0,
3947 "primary": false,
3948 "shard": 2,
3949 "size": 2048
3950 },
3951 {
3952 "errors": [
3953 "hinfo_corrupted"
3954 ],
3955 "osd": 1,
3956 "primary": true,
3957 "shard": 0,
3958 "hashinfo": "bad-val",
3959 "size": 2048
3960 },
3961 {
3962 "errors": [],
3963 "osd": 2,
3964 "primary": false,
3965 "shard": 1,
3966 "size": 2048,
3967 "hashinfo": {
3968 "cumulative_shard_hashes": [
3969 {
3970 "hash": 80717615,
3971 "shard": 0
3972 },
3973 {
3974 "hash": 1534491824,
3975 "shard": 1
3976 },
3977 {
3978 "hash": 80717615,
3979 "shard": 2
3980 }
3981 ],
3982 "total_chunk_size": 2048
3983 }
3984 }
3985 ],
3986 "union_shard_errors": [
3987 "hinfo_missing",
3988 "hinfo_corrupted"
3989 ]
3990 },
3991 {
3992 "errors": [
3993 "hinfo_inconsistency"
3994 ],
3995 "object": {
3996 "locator": "",
3997 "name": "EOBJ7",
3998 "nspace": "",
3999 "snap": "head",
4000 "version": 10
4001 },
4002 "selected_object_info": {
4003 "oid": {
4004 "oid": "EOBJ7",
4005 "key": "",
4006 "snapid": -2,
4007 "hash": 3258066308,
4008 "max": 0,
4009 "pool": 3,
4010 "namespace": ""
4011 },
4012 "version": "75'10",
4013 "prior_version": "75'9",
4014 "last_reqid": "client.4482.0:1",
4015 "user_version": 10,
4016 "size": 34,
4017 "mtime": "",
4018 "local_mtime": "",
4019 "lost": 0,
4020 "flags": [
4021 "dirty",
4022 "data_digest"
4023 ],
4024 "truncate_seq": 0,
4025 "truncate_size": 0,
4026 "data_digest": "0x136e4e27",
4027 "omap_digest": "0xffffffff",
4028 "expected_object_size": 0,
4029 "expected_write_size": 0,
4030 "alloc_hint_flags": 0,
4031 "manifest": {
4032 "type": 0
4033 },
4034 "watchers": {}
4035 },
4036 "shards": [
4037 {
4038 "hashinfo": {
4039 "cumulative_shard_hashes": [
4040 {
4041 "hash": 80717615,
4042 "shard": 0
4043 },
4044 {
4045 "hash": 1534491824,
4046 "shard": 1
4047 },
4048 {
4049 "hash": 80717615,
4050 "shard": 2
4051 }
4052 ],
4053 "total_chunk_size": 2048
4054 },
4055 "errors": [],
4056 "osd": 0,
4057 "primary": false,
4058 "shard": 2,
4059 "size": 2048
4060 },
4061 {
4062 "hashinfo": {
4063 "cumulative_shard_hashes": [
4064 {
4065 "hash": 1534350760,
4066 "shard": 0
4067 },
4068 {
4069 "hash": 1534491824,
4070 "shard": 1
4071 },
4072 {
4073 "hash": 1534350760,
4074 "shard": 2
4075 }
4076 ],
4077 "total_chunk_size": 2048
4078 },
4079 "errors": [],
4080 "osd": 1,
4081 "primary": true,
4082 "shard": 0,
4083 "size": 2048
4084 },
4085 {
4086 "hashinfo": {
4087 "cumulative_shard_hashes": [
4088 {
4089 "hash": 1534350760,
4090 "shard": 0
4091 },
4092 {
4093 "hash": 1534491824,
4094 "shard": 1
4095 },
4096 {
4097 "hash": 1534350760,
4098 "shard": 2
4099 }
4100 ],
4101 "total_chunk_size": 2048
4102 },
4103 "errors": [],
4104 "osd": 2,
4105 "primary": false,
4106 "shard": 1,
4107 "size": 2048
4108 }
4109 ],
4110 "union_shard_errors": []
4111 }
4112 ],
4113 "epoch": 0
4114 }
4115 EOF
4116
4117 jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
4118 multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
4119 if test $getjson = "yes"
4120 then
4121 jq '.' $dir/json > save3.json
4122 fi
4123
4124 if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
4125 then
4126 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
4127 fi
4128
4129 pg_deep_scrub $pg
4130
4131 rados list-inconsistent-pg $poolname > $dir/json || return 1
4132 # Check pg count
4133 test $(jq '. | length' $dir/json) = "1" || return 1
4134 # Check pgid
4135 test $(jq -r '.[0]' $dir/json) = $pg || return 1
4136
4137 rados list-inconsistent-obj $pg > $dir/json || return 1
4138 # Get epoch for repair-get requests
4139 epoch=$(jq .epoch $dir/json)
4140
4141 if [ "$allow_overwrites" = "true" ]
4142 then
4143 jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
4144 {
4145 "inconsistents": [
4146 {
4147 "shards": [
4148 {
4149 "data_digest": "0x00000000",
4150 "omap_digest": "0xffffffff",
4151 "size": 2048,
4152 "errors": [],
4153 "shard": 2,
4154 "osd": 0,
4155 "primary": false
4156 },
4157 {
4158 "object_info": {
4159 "oid": {
4160 "oid": "EOBJ1",
4161 "key": "",
4162 "snapid": -2,
4163 "hash": 560836233,
4164 "max": 0,
4165 "pool": 3,
4166 "namespace": ""
4167 },
4168 "version": "27'1",
4169 "prior_version": "0'0",
4170 "last_reqid": "client.4184.0:1",
4171 "user_version": 1,
4172 "size": 7,
4173 "mtime": "2018-04-05 14:31:33.837147",
4174 "local_mtime": "2018-04-05 14:31:33.840763",
4175 "lost": 0,
4176 "flags": [
4177 "dirty",
4178 "data_digest"
4179 ],
4180 "truncate_seq": 0,
4181 "truncate_size": 0,
4182 "data_digest": "0x2ddbf8f5",
4183 "omap_digest": "0xffffffff",
4184 "expected_object_size": 0,
4185 "expected_write_size": 0,
4186 "alloc_hint_flags": 0,
4187 "manifest": {
4188 "type": 0
4189 },
4190 "watchers": {}
4191 },
4192 "size": 9,
4193 "shard": 0,
4194 "errors": [
4195 "read_error",
4196 "size_mismatch_info",
4197 "obj_size_info_mismatch"
4198 ],
4199 "osd": 1,
4200 "primary": true
4201 },
4202 {
4203 "data_digest": "0x00000000",
4204 "omap_digest": "0xffffffff",
4205 "size": 2048,
4206 "shard": 1,
4207 "errors": [],
4208 "osd": 2,
4209 "primary": false
4210 }
4211 ],
4212 "selected_object_info": {
4213 "oid": {
4214 "oid": "EOBJ1",
4215 "key": "",
4216 "snapid": -2,
4217 "hash": 560836233,
4218 "max": 0,
4219 "pool": 3,
4220 "namespace": ""
4221 },
4222 "version": "27'1",
4223 "prior_version": "0'0",
4224 "last_reqid": "client.4184.0:1",
4225 "user_version": 1,
4226 "size": 7,
4227 "mtime": "2018-04-05 14:31:33.837147",
4228 "local_mtime": "2018-04-05 14:31:33.840763",
4229 "lost": 0,
4230 "flags": [
4231 "dirty",
4232 "data_digest"
4233 ],
4234 "truncate_seq": 0,
4235 "truncate_size": 0,
4236 "data_digest": "0x2ddbf8f5",
4237 "omap_digest": "0xffffffff",
4238 "expected_object_size": 0,
4239 "expected_write_size": 0,
4240 "alloc_hint_flags": 0,
4241 "manifest": {
4242 "type": 0
4243 },
4244 "watchers": {}
4245 },
4246 "union_shard_errors": [
4247 "read_error",
4248 "size_mismatch_info",
4249 "obj_size_info_mismatch"
4250 ],
4251 "errors": [
4252 "size_mismatch"
4253 ],
4254 "object": {
4255 "version": 1,
4256 "snap": "head",
4257 "locator": "",
4258 "nspace": "",
4259 "name": "EOBJ1"
4260 }
4261 },
4262 {
4263 "shards": [
4264 {
4265 "data_digest": "0x00000000",
4266 "omap_digest": "0xffffffff",
4267 "size": 2048,
4268 "errors": [],
4269 "shard": 2,
4270 "osd": 0,
4271 "primary": false
4272 },
4273 {
4274 "shard": 0,
4275 "errors": [
4276 "missing"
4277 ],
4278 "osd": 1,
4279 "primary": true
4280 },
4281 {
4282 "data_digest": "0x00000000",
4283 "omap_digest": "0xffffffff",
4284 "size": 2048,
4285 "shard": 1,
4286 "errors": [],
4287 "osd": 2,
4288 "primary": false
4289 }
4290 ],
4291 "selected_object_info": {
4292 "oid": {
4293 "oid": "EOBJ3",
4294 "key": "",
4295 "snapid": -2,
4296 "hash": 3125668237,
4297 "max": 0,
4298 "pool": 3,
4299 "namespace": ""
4300 },
4301 "version": "39'3",
4302 "prior_version": "0'0",
4303 "last_reqid": "client.4252.0:1",
4304 "user_version": 3,
4305 "size": 7,
4306 "mtime": "2018-04-05 14:31:46.841145",
4307 "local_mtime": "2018-04-05 14:31:46.844996",
4308 "lost": 0,
4309 "flags": [
4310 "dirty",
4311 "data_digest"
4312 ],
4313 "truncate_seq": 0,
4314 "truncate_size": 0,
4315 "data_digest": "0x2ddbf8f5",
4316 "omap_digest": "0xffffffff",
4317 "expected_object_size": 0,
4318 "expected_write_size": 0,
4319 "alloc_hint_flags": 0,
4320 "manifest": {
4321 "type": 0
4322 },
4323 "watchers": {}
4324 },
4325 "union_shard_errors": [
4326 "missing"
4327 ],
4328 "errors": [],
4329 "object": {
4330 "version": 3,
4331 "snap": "head",
4332 "locator": "",
4333 "nspace": "",
4334 "name": "EOBJ3"
4335 }
4336 },
4337 {
4338 "shards": [
4339 {
4340 "attrs": [
4341 {
4342 "Base64": false,
4343 "value": "bad-val",
4344 "name": "key1-EOBJ4"
4345 },
4346 {
4347 "Base64": false,
4348 "value": "val2-EOBJ4",
4349 "name": "key2-EOBJ4"
4350 }
4351 ],
4352 "data_digest": "0x00000000",
4353 "omap_digest": "0xffffffff",
4354 "size": 2048,
4355 "errors": [],
4356 "shard": 2,
4357 "osd": 0,
4358 "primary": false
4359 },
4360 {
4361 "attrs": [
4362 {
4363 "Base64": false,
4364 "value": "val1-EOBJ4",
4365 "name": "key1-EOBJ4"
4366 },
4367 {
4368 "Base64": false,
4369 "value": "val2-EOBJ4",
4370 "name": "key2-EOBJ4"
4371 }
4372 ],
4373 "data_digest": "0x00000000",
4374 "omap_digest": "0xffffffff",
4375 "size": 2048,
4376 "errors": [],
4377 "shard": 0,
4378 "osd": 1,
4379 "primary": true
4380 },
4381 {
4382 "attrs": [
4383 {
4384 "Base64": false,
4385 "value": "val1-EOBJ4",
4386 "name": "key1-EOBJ4"
4387 },
4388 {
4389 "Base64": false,
4390 "value": "val3-EOBJ4",
4391 "name": "key3-EOBJ4"
4392 }
4393 ],
4394 "data_digest": "0x00000000",
4395 "omap_digest": "0xffffffff",
4396 "size": 2048,
4397 "errors": [],
4398 "shard": 1,
4399 "osd": 2,
4400 "primary": false
4401 }
4402 ],
4403 "selected_object_info": {
4404 "oid": {
4405 "oid": "EOBJ4",
4406 "key": "",
4407 "snapid": -2,
4408 "hash": 1618759290,
4409 "max": 0,
4410 "pool": 3,
4411 "namespace": ""
4412 },
4413 "version": "45'6",
4414 "prior_version": "45'5",
4415 "last_reqid": "client.4294.0:1",
4416 "user_version": 6,
4417 "size": 7,
4418 "mtime": "2018-04-05 14:31:54.663622",
4419 "local_mtime": "2018-04-05 14:31:54.664527",
4420 "lost": 0,
4421 "flags": [
4422 "dirty",
4423 "data_digest"
4424 ],
4425 "truncate_seq": 0,
4426 "truncate_size": 0,
4427 "data_digest": "0x2ddbf8f5",
4428 "omap_digest": "0xffffffff",
4429 "expected_object_size": 0,
4430 "expected_write_size": 0,
4431 "alloc_hint_flags": 0,
4432 "manifest": {
4433 "type": 0
4434 },
4435 "watchers": {}
4436 },
4437 "union_shard_errors": [],
4438 "errors": [
4439 "attr_value_mismatch",
4440 "attr_name_mismatch"
4441 ],
4442 "object": {
4443 "version": 6,
4444 "snap": "head",
4445 "locator": "",
4446 "nspace": "",
4447 "name": "EOBJ4"
4448 }
4449 },
4450 {
4451 "shards": [
4452 {
4453 "data_digest": "0x00000000",
4454 "omap_digest": "0xffffffff",
4455 "size": 2048,
4456 "errors": [],
4457 "shard": 2,
4458 "osd": 0,
4459 "primary": false
4460 },
4461 {
4462 "data_digest": "0x00000000",
4463 "omap_digest": "0xffffffff",
4464 "object_info": {
4465 "oid": {
4466 "oid": "EOBJ5",
4467 "key": "",
4468 "snapid": -2,
4469 "hash": 2918945441,
4470 "max": 0,
4471 "pool": 3,
4472 "namespace": ""
4473 },
4474 "version": "59'7",
4475 "prior_version": "0'0",
4476 "last_reqid": "client.4382.0:1",
4477 "user_version": 7,
4478 "size": 7,
4479 "mtime": "2018-04-05 14:32:12.929161",
4480 "local_mtime": "2018-04-05 14:32:12.934707",
4481 "lost": 0,
4482 "flags": [
4483 "dirty",
4484 "data_digest"
4485 ],
4486 "truncate_seq": 0,
4487 "truncate_size": 0,
4488 "data_digest": "0x2ddbf8f5",
4489 "omap_digest": "0xffffffff",
4490 "expected_object_size": 0,
4491 "expected_write_size": 0,
4492 "alloc_hint_flags": 0,
4493 "manifest": {
4494 "type": 0
4495 },
4496 "watchers": {}
4497 },
4498 "size": 4096,
4499 "errors": [
4500 "size_mismatch_info",
4501 "obj_size_info_mismatch"
4502 ],
4503 "shard": 0,
4504 "osd": 1,
4505 "primary": true
4506 },
4507 {
4508 "data_digest": "0x00000000",
4509 "omap_digest": "0xffffffff",
4510 "size": 2048,
4511 "errors": [],
4512 "shard": 1,
4513 "osd": 2,
4514 "primary": false
4515 }
4516 ],
4517 "selected_object_info": {
4518 "oid": {
4519 "oid": "EOBJ5",
4520 "key": "",
4521 "snapid": -2,
4522 "hash": 2918945441,
4523 "max": 0,
4524 "pool": 3,
4525 "namespace": ""
4526 },
4527 "version": "59'7",
4528 "prior_version": "0'0",
4529 "last_reqid": "client.4382.0:1",
4530 "user_version": 7,
4531 "size": 7,
4532 "mtime": "2018-04-05 14:32:12.929161",
4533 "local_mtime": "2018-04-05 14:32:12.934707",
4534 "lost": 0,
4535 "flags": [
4536 "dirty",
4537 "data_digest"
4538 ],
4539 "truncate_seq": 0,
4540 "truncate_size": 0,
4541 "data_digest": "0x2ddbf8f5",
4542 "omap_digest": "0xffffffff",
4543 "expected_object_size": 0,
4544 "expected_write_size": 0,
4545 "alloc_hint_flags": 0,
4546 "manifest": {
4547 "type": 0
4548 },
4549 "watchers": {}
4550 },
4551 "union_shard_errors": [
4552 "size_mismatch_info",
4553 "obj_size_info_mismatch"
4554 ],
4555 "errors": [
4556 "size_mismatch"
4557 ],
4558 "object": {
4559 "version": 7,
4560 "snap": "head",
4561 "locator": "",
4562 "nspace": "",
4563 "name": "EOBJ5"
4564 }
4565 },
4566 {
4567 "object": {
4568 "name": "EOBJ6",
4569 "nspace": "",
4570 "locator": "",
4571 "snap": "head",
4572 "version": 8
4573 },
4574 "errors": [],
4575 "union_shard_errors": [
4576 "read_error",
4577 "hinfo_missing",
4578 "hinfo_corrupted"
4579 ],
4580 "selected_object_info": {
4581 "oid": {
4582 "oid": "EOBJ6",
4583 "key": "",
4584 "snapid": -2,
4585 "hash": 3050890866,
4586 "max": 0,
4587 "pool": 3,
4588 "namespace": ""
4589 },
4590 "version": "65'8",
4591 "prior_version": "0'0",
4592 "last_reqid": "client.4418.0:1",
4593 "user_version": 8,
4594 "size": 7,
4595 "mtime": "2018-04-05 14:32:20.634116",
4596 "local_mtime": "2018-04-05 14:32:20.637999",
4597 "lost": 0,
4598 "flags": [
4599 "dirty",
4600 "data_digest"
4601 ],
4602 "truncate_seq": 0,
4603 "truncate_size": 0,
4604 "data_digest": "0x2ddbf8f5",
4605 "omap_digest": "0xffffffff",
4606 "expected_object_size": 0,
4607 "expected_write_size": 0,
4608 "alloc_hint_flags": 0,
4609 "manifest": {
4610 "type": 0
4611 },
4612 "watchers": {}
4613 },
4614 "shards": [
4615 {
4616 "osd": 0,
4617 "primary": false,
4618 "shard": 2,
4619 "errors": [
4620 "read_error",
4621 "hinfo_missing"
4622 ],
4623 "size": 2048
4624 },
4625 {
4626 "osd": 1,
4627 "primary": true,
4628 "shard": 0,
4629 "errors": [
4630 "read_error",
4631 "hinfo_corrupted"
4632 ],
4633 "size": 2048,
4634 "hashinfo": "bad-val"
4635 },
4636 {
4637 "osd": 2,
4638 "primary": false,
4639 "shard": 1,
4640 "errors": [],
4641 "size": 2048,
4642 "omap_digest": "0xffffffff",
4643 "data_digest": "0x00000000",
4644 "hashinfo": {
4645 "cumulative_shard_hashes": [
4646 {
4647 "hash": 80717615,
4648 "shard": 0
4649 },
4650 {
4651 "hash": 1534491824,
4652 "shard": 1
4653 },
4654 {
4655 "hash": 80717615,
4656 "shard": 2
4657 }
4658 ],
4659 "total_chunk_size": 2048
4660 }
4661 }
4662 ]
4663 },
4664 {
4665 "object": {
4666 "name": "EOBJ7",
4667 "nspace": "",
4668 "locator": "",
4669 "snap": "head",
4670 "version": 10
4671 },
4672 "errors": [
4673 "hinfo_inconsistency"
4674 ],
4675 "union_shard_errors": [],
4676 "selected_object_info": {
4677 "oid": {
4678 "oid": "EOBJ7",
4679 "key": "",
4680 "snapid": -2,
4681 "hash": 3258066308,
4682 "max": 0,
4683 "pool": 3,
4684 "namespace": ""
4685 },
4686 "version": "75'10",
4687 "prior_version": "75'9",
4688 "last_reqid": "client.4482.0:1",
4689 "user_version": 10,
4690 "size": 34,
4691 "mtime": "2018-04-05 14:32:33.058782",
4692 "local_mtime": "2018-04-05 14:32:33.059679",
4693 "lost": 0,
4694 "flags": [
4695 "dirty",
4696 "data_digest"
4697 ],
4698 "truncate_seq": 0,
4699 "truncate_size": 0,
4700 "data_digest": "0x136e4e27",
4701 "omap_digest": "0xffffffff",
4702 "expected_object_size": 0,
4703 "expected_write_size": 0,
4704 "alloc_hint_flags": 0,
4705 "manifest": {
4706 "type": 0
4707 },
4708 "watchers": {}
4709 },
4710 "shards": [
4711 {
4712 "osd": 0,
4713 "primary": false,
4714 "shard": 2,
4715 "errors": [],
4716 "size": 2048,
4717 "omap_digest": "0xffffffff",
4718 "data_digest": "0x00000000",
4719 "hashinfo": {
4720 "cumulative_shard_hashes": [
4721 {
4722 "hash": 80717615,
4723 "shard": 0
4724 },
4725 {
4726 "hash": 1534491824,
4727 "shard": 1
4728 },
4729 {
4730 "hash": 80717615,
4731 "shard": 2
4732 }
4733 ],
4734 "total_chunk_size": 2048
4735 }
4736 },
4737 {
4738 "osd": 1,
4739 "primary": true,
4740 "shard": 0,
4741 "errors": [],
4742 "size": 2048,
4743 "omap_digest": "0xffffffff",
4744 "data_digest": "0x00000000",
4745 "hashinfo": {
4746 "cumulative_shard_hashes": [
4747 {
4748 "hash": 1534350760,
4749 "shard": 0
4750 },
4751 {
4752 "hash": 1534491824,
4753 "shard": 1
4754 },
4755 {
4756 "hash": 1534350760,
4757 "shard": 2
4758 }
4759 ],
4760 "total_chunk_size": 2048
4761 }
4762 },
4763 {
4764 "osd": 2,
4765 "primary": false,
4766 "shard": 1,
4767 "errors": [],
4768 "size": 2048,
4769 "omap_digest": "0xffffffff",
4770 "data_digest": "0x00000000",
4771 "hashinfo": {
4772 "cumulative_shard_hashes": [
4773 {
4774 "hash": 1534350760,
4775 "shard": 0
4776 },
4777 {
4778 "hash": 1534491824,
4779 "shard": 1
4780 },
4781 {
4782 "hash": 1534350760,
4783 "shard": 2
4784 }
4785 ],
4786 "total_chunk_size": 2048
4787 }
4788 }
4789 ]
4790 }
4791 ],
4792 "epoch": 0
4793 }
4794 EOF
4795
4796 else
4797
4798 jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
4799 {
4800 "inconsistents": [
4801 {
4802 "shards": [
4803 {
4804 "data_digest": "0x04cfa72f",
4805 "omap_digest": "0xffffffff",
4806 "size": 2048,
4807 "errors": [],
4808 "shard": 2,
4809 "osd": 0,
4810 "primary": false
4811 },
4812 {
4813 "object_info": {
4814 "oid": {
4815 "oid": "EOBJ1",
4816 "key": "",
4817 "snapid": -2,
4818 "hash": 560836233,
4819 "max": 0,
4820 "pool": 3,
4821 "namespace": ""
4822 },
4823 "version": "27'1",
4824 "prior_version": "0'0",
4825 "last_reqid": "client.4192.0:1",
4826 "user_version": 1,
4827 "size": 7,
4828 "mtime": "2018-04-05 14:30:10.688009",
4829 "local_mtime": "2018-04-05 14:30:10.691774",
4830 "lost": 0,
4831 "flags": [
4832 "dirty",
4833 "data_digest"
4834 ],
4835 "truncate_seq": 0,
4836 "truncate_size": 0,
4837 "data_digest": "0x2ddbf8f5",
4838 "omap_digest": "0xffffffff",
4839 "expected_object_size": 0,
4840 "expected_write_size": 0,
4841 "alloc_hint_flags": 0,
4842 "manifest": {
4843 "type": 0
4844 },
4845 "watchers": {}
4846 },
4847 "size": 9,
4848 "shard": 0,
4849 "errors": [
4850 "read_error",
4851 "size_mismatch_info",
4852 "obj_size_info_mismatch"
4853 ],
4854 "osd": 1,
4855 "primary": true
4856 },
4857 {
4858 "data_digest": "0x04cfa72f",
4859 "omap_digest": "0xffffffff",
4860 "size": 2048,
4861 "shard": 1,
4862 "errors": [],
4863 "osd": 2,
4864 "primary": false
4865 }
4866 ],
4867 "selected_object_info": {
4868 "oid": {
4869 "oid": "EOBJ1",
4870 "key": "",
4871 "snapid": -2,
4872 "hash": 560836233,
4873 "max": 0,
4874 "pool": 3,
4875 "namespace": ""
4876 },
4877 "version": "27'1",
4878 "prior_version": "0'0",
4879 "last_reqid": "client.4192.0:1",
4880 "user_version": 1,
4881 "size": 7,
4882 "mtime": "2018-04-05 14:30:10.688009",
4883 "local_mtime": "2018-04-05 14:30:10.691774",
4884 "lost": 0,
4885 "flags": [
4886 "dirty",
4887 "data_digest"
4888 ],
4889 "truncate_seq": 0,
4890 "truncate_size": 0,
4891 "data_digest": "0x2ddbf8f5",
4892 "omap_digest": "0xffffffff",
4893 "expected_object_size": 0,
4894 "expected_write_size": 0,
4895 "alloc_hint_flags": 0,
4896 "manifest": {
4897 "type": 0
4898 },
4899 "watchers": {}
4900 },
4901 "union_shard_errors": [
4902 "read_error",
4903 "size_mismatch_info",
4904 "obj_size_info_mismatch"
4905 ],
4906 "errors": [
4907 "size_mismatch"
4908 ],
4909 "object": {
4910 "version": 1,
4911 "snap": "head",
4912 "locator": "",
4913 "nspace": "",
4914 "name": "EOBJ1"
4915 }
4916 },
4917 {
4918 "shards": [
4919 {
4920 "size": 2048,
4921 "errors": [
4922 "ec_hash_error"
4923 ],
4924 "shard": 2,
4925 "osd": 0,
4926 "primary": false
4927 },
4928 {
4929 "data_digest": "0x04cfa72f",
4930 "omap_digest": "0xffffffff",
4931 "size": 2048,
4932 "errors": [],
4933 "shard": 0,
4934 "osd": 1,
4935 "primary": true
4936 },
4937 {
4938 "data_digest": "0x04cfa72f",
4939 "omap_digest": "0xffffffff",
4940 "size": 2048,
4941 "errors": [],
4942 "shard": 1,
4943 "osd": 2,
4944 "primary": false
4945 }
4946 ],
4947 "selected_object_info": {
4948 "oid": {
4949 "oid": "EOBJ2",
4950 "key": "",
4951 "snapid": -2,
4952 "hash": 562812377,
4953 "max": 0,
4954 "pool": 3,
4955 "namespace": ""
4956 },
4957 "version": "33'2",
4958 "prior_version": "0'0",
4959 "last_reqid": "client.4224.0:1",
4960 "user_version": 2,
4961 "size": 7,
4962 "mtime": "2018-04-05 14:30:14.152945",
4963 "local_mtime": "2018-04-05 14:30:14.154014",
4964 "lost": 0,
4965 "flags": [
4966 "dirty",
4967 "data_digest"
4968 ],
4969 "truncate_seq": 0,
4970 "truncate_size": 0,
4971 "data_digest": "0x2ddbf8f5",
4972 "omap_digest": "0xffffffff",
4973 "expected_object_size": 0,
4974 "expected_write_size": 0,
4975 "alloc_hint_flags": 0,
4976 "manifest": {
4977 "type": 0
4978 },
4979 "watchers": {}
4980 },
4981 "union_shard_errors": [
4982 "ec_hash_error"
4983 ],
4984 "errors": [],
4985 "object": {
4986 "version": 2,
4987 "snap": "head",
4988 "locator": "",
4989 "nspace": "",
4990 "name": "EOBJ2"
4991 }
4992 },
4993 {
4994 "shards": [
4995 {
4996 "data_digest": "0x04cfa72f",
4997 "omap_digest": "0xffffffff",
4998 "size": 2048,
4999 "errors": [],
5000 "shard": 2,
5001 "osd": 0,
5002 "primary": false
5003 },
5004 {
5005 "osd": 1,
5006 "primary": true,
5007 "shard": 0,
5008 "errors": [
5009 "missing"
5010 ]
5011 },
5012 {
5013 "data_digest": "0x04cfa72f",
5014 "omap_digest": "0xffffffff",
5015 "size": 2048,
5016 "shard": 1,
5017 "errors": [],
5018 "osd": 2,
5019 "primary": false
5020 }
5021 ],
5022 "selected_object_info": {
5023 "oid": {
5024 "oid": "EOBJ3",
5025 "key": "",
5026 "snapid": -2,
5027 "hash": 3125668237,
5028 "max": 0,
5029 "pool": 3,
5030 "namespace": ""
5031 },
5032 "version": "39'3",
5033 "prior_version": "0'0",
5034 "last_reqid": "client.4258.0:1",
5035 "user_version": 3,
5036 "size": 7,
5037 "mtime": "2018-04-05 14:30:18.875544",
5038 "local_mtime": "2018-04-05 14:30:18.880153",
5039 "lost": 0,
5040 "flags": [
5041 "dirty",
5042 "data_digest"
5043 ],
5044 "truncate_seq": 0,
5045 "truncate_size": 0,
5046 "data_digest": "0x2ddbf8f5",
5047 "omap_digest": "0xffffffff",
5048 "expected_object_size": 0,
5049 "expected_write_size": 0,
5050 "alloc_hint_flags": 0,
5051 "manifest": {
5052 "type": 0
5053 },
5054 "watchers": {}
5055 },
5056 "union_shard_errors": [
5057 "missing"
5058 ],
5059 "errors": [],
5060 "object": {
5061 "version": 3,
5062 "snap": "head",
5063 "locator": "",
5064 "nspace": "",
5065 "name": "EOBJ3"
5066 }
5067 },
5068 {
5069 "shards": [
5070 {
5071 "attrs": [
5072 {
5073 "Base64": false,
5074 "value": "bad-val",
5075 "name": "key1-EOBJ4"
5076 },
5077 {
5078 "Base64": false,
5079 "value": "val2-EOBJ4",
5080 "name": "key2-EOBJ4"
5081 }
5082 ],
5083 "data_digest": "0x04cfa72f",
5084 "omap_digest": "0xffffffff",
5085 "size": 2048,
5086 "errors": [],
5087 "shard": 2,
5088 "osd": 0,
5089 "primary": false
5090 },
5091 {
5092 "osd": 1,
5093 "primary": true,
5094 "shard": 0,
5095 "errors": [],
5096 "size": 2048,
5097 "omap_digest": "0xffffffff",
5098 "data_digest": "0x04cfa72f",
5099 "attrs": [
5100 {
5101 "Base64": false,
5102 "value": "val1-EOBJ4",
5103 "name": "key1-EOBJ4"
5104 },
5105 {
5106 "Base64": false,
5107 "value": "val2-EOBJ4",
5108 "name": "key2-EOBJ4"
5109 }
5110 ]
5111 },
5112 {
5113 "osd": 2,
5114 "primary": false,
5115 "shard": 1,
5116 "errors": [],
5117 "size": 2048,
5118 "omap_digest": "0xffffffff",
5119 "data_digest": "0x04cfa72f",
5120 "attrs": [
5121 {
5122 "Base64": false,
5123 "value": "val1-EOBJ4",
5124 "name": "key1-EOBJ4"
5125 },
5126 {
5127 "Base64": false,
5128 "value": "val3-EOBJ4",
5129 "name": "key3-EOBJ4"
5130 }
5131 ]
5132 }
5133 ],
5134 "selected_object_info": {
5135 "oid": {
5136 "oid": "EOBJ4",
5137 "key": "",
5138 "snapid": -2,
5139 "hash": 1618759290,
5140 "max": 0,
5141 "pool": 3,
5142 "namespace": ""
5143 },
5144 "version": "45'6",
5145 "prior_version": "45'5",
5146 "last_reqid": "client.4296.0:1",
5147 "user_version": 6,
5148 "size": 7,
5149 "mtime": "2018-04-05 14:30:22.271983",
5150 "local_mtime": "2018-04-05 14:30:22.272840",
5151 "lost": 0,
5152 "flags": [
5153 "dirty",
5154 "data_digest"
5155 ],
5156 "truncate_seq": 0,
5157 "truncate_size": 0,
5158 "data_digest": "0x2ddbf8f5",
5159 "omap_digest": "0xffffffff",
5160 "expected_object_size": 0,
5161 "expected_write_size": 0,
5162 "alloc_hint_flags": 0,
5163 "manifest": {
5164 "type": 0
5165 },
5166 "watchers": {}
5167 },
5168 "union_shard_errors": [],
5169 "errors": [
5170 "attr_value_mismatch",
5171 "attr_name_mismatch"
5172 ],
5173 "object": {
5174 "version": 6,
5175 "snap": "head",
5176 "locator": "",
5177 "nspace": "",
5178 "name": "EOBJ4"
5179 }
5180 },
5181 {
5182 "shards": [
5183 {
5184 "data_digest": "0x04cfa72f",
5185 "omap_digest": "0xffffffff",
5186 "size": 2048,
5187 "errors": [],
5188 "shard": 2,
5189 "osd": 0,
5190 "primary": false
5191 },
5192 {
5193 "object_info": {
5194 "oid": {
5195 "oid": "EOBJ5",
5196 "key": "",
5197 "snapid": -2,
5198 "hash": 2918945441,
5199 "max": 0,
5200 "pool": 3,
5201 "namespace": ""
5202 },
5203 "version": "59'7",
5204 "prior_version": "0'0",
5205 "last_reqid": "client.4384.0:1",
5206 "user_version": 7,
5207 "size": 7,
5208 "mtime": "2018-04-05 14:30:35.162395",
5209 "local_mtime": "2018-04-05 14:30:35.166390",
5210 "lost": 0,
5211 "flags": [
5212 "dirty",
5213 "data_digest"
5214 ],
5215 "truncate_seq": 0,
5216 "truncate_size": 0,
5217 "data_digest": "0x2ddbf8f5",
5218 "omap_digest": "0xffffffff",
5219 "expected_object_size": 0,
5220 "expected_write_size": 0,
5221 "alloc_hint_flags": 0,
5222 "manifest": {
5223 "type": 0
5224 },
5225 "watchers": {}
5226 },
5227 "size": 4096,
5228 "shard": 0,
5229 "errors": [
5230 "size_mismatch_info",
5231 "ec_size_error",
5232 "obj_size_info_mismatch"
5233 ],
5234 "osd": 1,
5235 "primary": true
5236 },
5237 {
5238 "data_digest": "0x04cfa72f",
5239 "omap_digest": "0xffffffff",
5240 "size": 2048,
5241 "shard": 1,
5242 "errors": [],
5243 "osd": 2,
5244 "primary": false
5245 }
5246 ],
5247 "selected_object_info": {
5248 "oid": {
5249 "oid": "EOBJ5",
5250 "key": "",
5251 "snapid": -2,
5252 "hash": 2918945441,
5253 "max": 0,
5254 "pool": 3,
5255 "namespace": ""
5256 },
5257 "version": "59'7",
5258 "prior_version": "0'0",
5259 "last_reqid": "client.4384.0:1",
5260 "user_version": 7,
5261 "size": 7,
5262 "mtime": "2018-04-05 14:30:35.162395",
5263 "local_mtime": "2018-04-05 14:30:35.166390",
5264 "lost": 0,
5265 "flags": [
5266 "dirty",
5267 "data_digest"
5268 ],
5269 "truncate_seq": 0,
5270 "truncate_size": 0,
5271 "data_digest": "0x2ddbf8f5",
5272 "omap_digest": "0xffffffff",
5273 "expected_object_size": 0,
5274 "expected_write_size": 0,
5275 "alloc_hint_flags": 0,
5276 "manifest": {
5277 "type": 0
5278 },
5279 "watchers": {}
5280 },
5281 "union_shard_errors": [
5282 "size_mismatch_info",
5283 "ec_size_error",
5284 "obj_size_info_mismatch"
5285 ],
5286 "errors": [
5287 "size_mismatch"
5288 ],
5289 "object": {
5290 "version": 7,
5291 "snap": "head",
5292 "locator": "",
5293 "nspace": "",
5294 "name": "EOBJ5"
5295 }
5296 },
5297 {
5298 "object": {
5299 "name": "EOBJ6",
5300 "nspace": "",
5301 "locator": "",
5302 "snap": "head",
5303 "version": 8
5304 },
5305 "errors": [],
5306 "union_shard_errors": [
5307 "read_error",
5308 "hinfo_missing",
5309 "hinfo_corrupted"
5310 ],
5311 "selected_object_info": {
5312 "oid": {
5313 "oid": "EOBJ6",
5314 "key": "",
5315 "snapid": -2,
5316 "hash": 3050890866,
5317 "max": 0,
5318 "pool": 3,
5319 "namespace": ""
5320 },
5321 "version": "65'8",
5322 "prior_version": "0'0",
5323 "last_reqid": "client.4420.0:1",
5324 "user_version": 8,
5325 "size": 7,
5326 "mtime": "2018-04-05 14:30:40.914673",
5327 "local_mtime": "2018-04-05 14:30:40.917705",
5328 "lost": 0,
5329 "flags": [
5330 "dirty",
5331 "data_digest"
5332 ],
5333 "truncate_seq": 0,
5334 "truncate_size": 0,
5335 "data_digest": "0x2ddbf8f5",
5336 "omap_digest": "0xffffffff",
5337 "expected_object_size": 0,
5338 "expected_write_size": 0,
5339 "alloc_hint_flags": 0,
5340 "manifest": {
5341 "type": 0
5342 },
5343 "watchers": {}
5344 },
5345 "shards": [
5346 {
5347 "osd": 0,
5348 "primary": false,
5349 "shard": 2,
5350 "errors": [
5351 "read_error",
5352 "hinfo_missing"
5353 ],
5354 "size": 2048
5355 },
5356 {
5357 "osd": 1,
5358 "primary": true,
5359 "shard": 0,
5360 "errors": [
5361 "read_error",
5362 "hinfo_corrupted"
5363 ],
5364 "size": 2048,
5365 "hashinfo": "bad-val"
5366 },
5367 {
5368 "osd": 2,
5369 "primary": false,
5370 "shard": 1,
5371 "errors": [],
5372 "size": 2048,
5373 "omap_digest": "0xffffffff",
5374 "data_digest": "0x04cfa72f",
5375 "hashinfo": {
5376 "cumulative_shard_hashes": [
5377 {
5378 "hash": 80717615,
5379 "shard": 0
5380 },
5381 {
5382 "hash": 1534491824,
5383 "shard": 1
5384 },
5385 {
5386 "hash": 80717615,
5387 "shard": 2
5388 }
5389 ],
5390 "total_chunk_size": 2048
5391 }
5392 }
5393 ]
5394 },
5395 {
5396 "object": {
5397 "name": "EOBJ7",
5398 "nspace": "",
5399 "locator": "",
5400 "snap": "head",
5401 "version": 10
5402 },
5403 "errors": [
5404 "hinfo_inconsistency"
5405 ],
5406 "union_shard_errors": [
5407 "ec_hash_error"
5408 ],
5409 "selected_object_info": {
5410 "oid": {
5411 "oid": "EOBJ7",
5412 "key": "",
5413 "snapid": -2,
5414 "hash": 3258066308,
5415 "max": 0,
5416 "pool": 3,
5417 "namespace": ""
5418 },
5419 "version": "75'10",
5420 "prior_version": "75'9",
5421 "last_reqid": "client.4486.0:1",
5422 "user_version": 10,
5423 "size": 34,
5424 "mtime": "2018-04-05 14:30:50.995009",
5425 "local_mtime": "2018-04-05 14:30:50.996112",
5426 "lost": 0,
5427 "flags": [
5428 "dirty",
5429 "data_digest"
5430 ],
5431 "truncate_seq": 0,
5432 "truncate_size": 0,
5433 "data_digest": "0x136e4e27",
5434 "omap_digest": "0xffffffff",
5435 "expected_object_size": 0,
5436 "expected_write_size": 0,
5437 "alloc_hint_flags": 0,
5438 "manifest": {
5439 "type": 0
5440 },
5441 "watchers": {}
5442 },
5443 "shards": [
5444 {
5445 "osd": 0,
5446 "primary": false,
5447 "shard": 2,
5448 "errors": [
5449 "ec_hash_error"
5450 ],
5451 "size": 2048,
5452 "hashinfo": {
5453 "cumulative_shard_hashes": [
5454 {
5455 "hash": 80717615,
5456 "shard": 0
5457 },
5458 {
5459 "hash": 1534491824,
5460 "shard": 1
5461 },
5462 {
5463 "hash": 80717615,
5464 "shard": 2
5465 }
5466 ],
5467 "total_chunk_size": 2048
5468 }
5469 },
5470 {
5471 "osd": 1,
5472 "primary": true,
5473 "shard": 0,
5474 "errors": [],
5475 "size": 2048,
5476 "omap_digest": "0xffffffff",
5477 "data_digest": "0x5b7455a8",
5478 "hashinfo": {
5479 "cumulative_shard_hashes": [
5480 {
5481 "hash": 1534350760,
5482 "shard": 0
5483 },
5484 {
5485 "hash": 1534491824,
5486 "shard": 1
5487 },
5488 {
5489 "hash": 1534350760,
5490 "shard": 2
5491 }
5492 ],
5493 "total_chunk_size": 2048
5494 }
5495 },
5496 {
5497 "osd": 2,
5498 "primary": false,
5499 "shard": 1,
5500 "errors": [],
5501 "size": 2048,
5502 "omap_digest": "0xffffffff",
5503 "data_digest": "0x5b7455a8",
5504 "hashinfo": {
5505 "cumulative_shard_hashes": [
5506 {
5507 "hash": 1534350760,
5508 "shard": 0
5509 },
5510 {
5511 "hash": 1534491824,
5512 "shard": 1
5513 },
5514 {
5515 "hash": 1534350760,
5516 "shard": 2
5517 }
5518 ],
5519 "total_chunk_size": 2048
5520 }
5521 }
5522 ]
5523 }
5524 ],
5525 "epoch": 0
5526 }
5527 EOF
5528
5529 fi
5530
5531 jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
5532 multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
5533 if test $getjson = "yes"
5534 then
5535 if [ "$allow_overwrites" = "true" ]
5536 then
5537 num=4
5538 else
5539 num=5
5540 fi
5541 jq '.' $dir/json > save${num}.json
5542 fi
5543
5544 if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
5545 then
5546 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
5547 fi
5548
5549 ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it
5550 teardown $dir || return 1
5551 }
5552
5553 function TEST_corrupt_scrub_erasure_appends() {
5554 corrupt_scrub_erasure $1 false
5555 }
5556
5557 function TEST_corrupt_scrub_erasure_overwrites() {
5558 if [ "$use_ec_overwrite" = "true" ]; then
5559 corrupt_scrub_erasure $1 true
5560 fi
5561 }
5562
5563 #
5564 # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost
5565 #
5566 function TEST_periodic_scrub_replicated() {
5567 local dir=$1
5568 local poolname=psr_pool
5569 local objname=POBJ
5570
5571 setup $dir || return 1
5572 run_mon $dir a --osd_pool_default_size=2 || return 1
5573 run_mgr $dir x || return 1
5574 local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
5575 ceph_osd_args+="--osd_scrub_backoff_ratio=0"
5576 run_osd $dir 0 $ceph_osd_args || return 1
5577 run_osd $dir 1 $ceph_osd_args || return 1
5578 create_rbd_pool || return 1
5579 wait_for_clean || return 1
5580
5581 create_pool $poolname 1 1 || return 1
5582 wait_for_clean || return 1
5583
5584 local osd=0
5585 add_something $dir $poolname $objname scrub || return 1
5586 local primary=$(get_primary $poolname $objname)
5587 local pg=$(get_pg $poolname $objname)
5588
5589 # Add deep-scrub only error
5590 local payload=UVWXYZ
5591 echo $payload > $dir/CORRUPT
5592 # Uses $ceph_osd_args for osd restart
5593 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
5594
5595 # No scrub information available, so expect failure
5596 set -o pipefail
5597 ! rados list-inconsistent-obj $pg | jq '.' || return 1
5598 set +o pipefail
5599
5600 pg_deep_scrub $pg || return 1
5601
5602 # Make sure bad object found
5603 rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5604
5605 flush_pg_stats
5606 local last_scrub=$(get_last_scrub_stamp $pg)
5607 # Fake a schedule scrub
5608 CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
5609 trigger_scrub $pg || return 1
5610 # Wait for schedule regular scrub
5611 wait_for_scrub $pg "$last_scrub"
5612
5613 # It needed to be upgraded
5614 grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1
5615
5616 # Bad object still known
5617 rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5618
5619 # Can't upgrade with this set
5620 ceph osd set nodeep-scrub
5621 # Let map change propagate to OSDs
5622 flush_pg_stats
5623 sleep 5
5624
5625 # Fake a schedule scrub
5626 CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
5627 trigger_scrub $pg || return 1
5628 # Wait for schedule regular scrub
5629 # to notice scrub and skip it
5630 local found=false
5631 for i in $(seq 14 -1 0)
5632 do
5633 sleep 1
5634 ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; }
5635 echo Time left: $i seconds
5636 done
5637 test $found = "true" || return 1
5638
5639 # Bad object still known
5640 rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5641
5642 flush_pg_stats
5643 # Request a regular scrub and it will be done
5644 pg_scrub $pg
5645 grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1
5646
5647 # deep-scrub error is no longer present
5648 rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1
5649 }
5650
5651 function TEST_scrub_warning() {
5652 local dir=$1
5653 local poolname=psr_pool
5654 local objname=POBJ
5655 local scrubs=5
5656 local deep_scrubs=5
5657 local i1_day=86400
5658 local i7_days=$(calc $i1_day \* 7)
5659 local i14_days=$(calc $i1_day \* 14)
5660 local overdue=0.5
5661 local conf_overdue_seconds=$(calc $i7_days + $i1_day + \( $i7_days \* $overdue \) )
5662 local pool_overdue_seconds=$(calc $i14_days + $i1_day + \( $i14_days \* $overdue \) )
5663
5664 setup $dir || return 1
5665 run_mon $dir a --osd_pool_default_size=1 || return 1
5666 run_mgr $dir x --mon_warn_pg_not_scrubbed_ratio=${overdue} --mon_warn_pg_not_deep_scrubbed_ratio=${overdue} || return 1
5667 run_osd $dir 0 $ceph_osd_args --osd_scrub_backoff_ratio=0 || return 1
5668
5669 for i in $(seq 1 $(expr $scrubs + $deep_scrubs))
5670 do
5671 create_pool $poolname-$i 1 1 || return 1
5672 wait_for_clean || return 1
5673 if [ $i = "1" ];
5674 then
5675 ceph osd pool set $poolname-$i scrub_max_interval $i14_days
5676 fi
5677 if [ $i = $(expr $scrubs + 1) ];
5678 then
5679 ceph osd pool set $poolname-$i deep_scrub_interval $i14_days
5680 fi
5681 done
5682
5683 # Only 1 osd
5684 local primary=0
5685
5686 ceph osd set noscrub || return 1
5687 ceph osd set nodeep-scrub || return 1
5688 ceph config set global osd_scrub_interval_randomize_ratio 0
5689 ceph config set global osd_deep_scrub_randomize_ratio 0
5690 ceph config set global osd_scrub_max_interval ${i7_days}
5691 ceph config set global osd_deep_scrub_interval ${i7_days}
5692
5693 # Fake schedule scrubs
5694 for i in $(seq 1 $scrubs)
5695 do
5696 if [ $i = "1" ];
5697 then
5698 overdue_seconds=$pool_overdue_seconds
5699 else
5700 overdue_seconds=$conf_overdue_seconds
5701 fi
5702 CEPH_ARGS='' ceph daemon $(get_asok_path osd.${primary}) \
5703 trigger_scrub ${i}.0 $(expr ${overdue_seconds} + ${i}00) || return 1
5704 done
5705 # Fake schedule deep scrubs
5706 for i in $(seq $(expr $scrubs + 1) $(expr $scrubs + $deep_scrubs))
5707 do
5708 if [ $i = "$(expr $scrubs + 1)" ];
5709 then
5710 overdue_seconds=$pool_overdue_seconds
5711 else
5712 overdue_seconds=$conf_overdue_seconds
5713 fi
5714 CEPH_ARGS='' ceph daemon $(get_asok_path osd.${primary}) \
5715 trigger_deep_scrub ${i}.0 $(expr ${overdue_seconds} + ${i}00) || return 1
5716 done
5717 flush_pg_stats
5718
5719 ceph health
5720 ceph health detail
5721 ceph health | grep -q "$deep_scrubs pgs not deep-scrubbed in time" || return 1
5722 ceph health | grep -q "$scrubs pgs not scrubbed in time" || return 1
5723 COUNT=$(ceph health detail | grep "not scrubbed since" | wc -l)
5724 if [ "$COUNT" != $scrubs ]; then
5725 ceph health detail | grep "not scrubbed since"
5726 return 1
5727 fi
5728 COUNT=$(ceph health detail | grep "not deep-scrubbed since" | wc -l)
5729 if [ "$COUNT" != $deep_scrubs ]; then
5730 ceph health detail | grep "not deep-scrubbed since"
5731 return 1
5732 fi
5733 return 0
5734 }
5735
5736 #
5737 # Corrupt snapset in replicated pool
5738 #
5739 function TEST_corrupt_snapset_scrub_rep() {
5740 local dir=$1
5741 local poolname=csr_pool
5742 local total_objs=2
5743
5744 setup $dir || return 1
5745 run_mon $dir a --osd_pool_default_size=2 || return 1
5746 run_mgr $dir x || return 1
5747 run_osd $dir 0 || return 1
5748 run_osd $dir 1 || return 1
5749 create_rbd_pool || return 1
5750 wait_for_clean || return 1
5751
5752 create_pool foo 1 || return 1
5753 create_pool $poolname 1 1 || return 1
5754 wait_for_clean || return 1
5755
5756 for i in $(seq 1 $total_objs) ; do
5757 objname=ROBJ${i}
5758 add_something $dir $poolname $objname || return 1
5759
5760 rados --pool $poolname setomapheader $objname hdr-$objname || return 1
5761 rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
5762 done
5763
5764 local pg=$(get_pg $poolname ROBJ0)
5765 local primary=$(get_primary $poolname ROBJ0)
5766
5767 rados -p $poolname mksnap snap1
5768 echo -n head_of_snapshot_data > $dir/change
5769
5770 for i in $(seq 1 $total_objs) ; do
5771 objname=ROBJ${i}
5772
5773 # Alternate corruption between osd.0 and osd.1
5774 local osd=$(expr $i % 2)
5775
5776 case $i in
5777 1)
5778 rados --pool $poolname put $objname $dir/change
5779 objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
5780 ;;
5781
5782 2)
5783 rados --pool $poolname put $objname $dir/change
5784 objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
5785 ;;
5786
5787 esac
5788 done
5789 rm $dir/change
5790
5791 pg_scrub $pg
5792
5793 rados list-inconsistent-pg $poolname > $dir/json || return 1
5794 # Check pg count
5795 test $(jq '. | length' $dir/json) = "1" || return 1
5796 # Check pgid
5797 test $(jq -r '.[0]' $dir/json) = $pg || return 1
5798
5799 rados list-inconsistent-obj $pg > $dir/json || return 1
5800
5801 jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
5802 {
5803 "epoch": 34,
5804 "inconsistents": [
5805 {
5806 "object": {
5807 "name": "ROBJ1",
5808 "nspace": "",
5809 "locator": "",
5810 "snap": "head",
5811 "version": 8
5812 },
5813 "errors": [
5814 "snapset_inconsistency"
5815 ],
5816 "union_shard_errors": [],
5817 "selected_object_info": {
5818 "oid": {
5819 "oid": "ROBJ1",
5820 "key": "",
5821 "snapid": -2,
5822 "hash": 1454963827,
5823 "max": 0,
5824 "pool": 3,
5825 "namespace": ""
5826 },
5827 "version": "24'8",
5828 "prior_version": "21'3",
5829 "last_reqid": "client.4195.0:1",
5830 "user_version": 8,
5831 "size": 21,
5832 "mtime": "2018-04-05 14:35:43.286117",
5833 "local_mtime": "2018-04-05 14:35:43.288990",
5834 "lost": 0,
5835 "flags": [
5836 "dirty",
5837 "omap",
5838 "data_digest"
5839 ],
5840 "truncate_seq": 0,
5841 "truncate_size": 0,
5842 "data_digest": "0x53acb008",
5843 "omap_digest": "0xffffffff",
5844 "expected_object_size": 0,
5845 "expected_write_size": 0,
5846 "alloc_hint_flags": 0,
5847 "manifest": {
5848 "type": 0
5849 },
5850 "watchers": {}
5851 },
5852 "shards": [
5853 {
5854 "osd": 0,
5855 "primary": false,
5856 "errors": [],
5857 "size": 21,
5858 "snapset": {
5859 "clones": [
5860 {
5861 "overlap": "[]",
5862 "size": 7,
5863 "snap": 1,
5864 "snaps": [
5865 1
5866 ]
5867 }
5868 ],
5869 "snap_context": {
5870 "seq": 1,
5871 "snaps": [
5872 1
5873 ]
5874 }
5875 }
5876 },
5877 {
5878 "osd": 1,
5879 "primary": true,
5880 "errors": [],
5881 "size": 21,
5882 "snapset": {
5883 "clones": [],
5884 "snap_context": {
5885 "seq": 0,
5886 "snaps": []
5887 }
5888 }
5889 }
5890 ]
5891 },
5892 {
5893 "object": {
5894 "name": "ROBJ2",
5895 "nspace": "",
5896 "locator": "",
5897 "snap": "head",
5898 "version": 10
5899 },
5900 "errors": [
5901 "snapset_inconsistency"
5902 ],
5903 "union_shard_errors": [],
5904 "selected_object_info": {
5905 "oid": {
5906 "oid": "ROBJ2",
5907 "key": "",
5908 "snapid": -2,
5909 "hash": 2026323607,
5910 "max": 0,
5911 "pool": 3,
5912 "namespace": ""
5913 },
5914 "version": "28'10",
5915 "prior_version": "23'6",
5916 "last_reqid": "client.4223.0:1",
5917 "user_version": 10,
5918 "size": 21,
5919 "mtime": "2018-04-05 14:35:48.326856",
5920 "local_mtime": "2018-04-05 14:35:48.328097",
5921 "lost": 0,
5922 "flags": [
5923 "dirty",
5924 "omap",
5925 "data_digest"
5926 ],
5927 "truncate_seq": 0,
5928 "truncate_size": 0,
5929 "data_digest": "0x53acb008",
5930 "omap_digest": "0xffffffff",
5931 "expected_object_size": 0,
5932 "expected_write_size": 0,
5933 "alloc_hint_flags": 0,
5934 "manifest": {
5935 "type": 0
5936 },
5937 "watchers": {}
5938 },
5939 "shards": [
5940 {
5941 "osd": 0,
5942 "primary": false,
5943 "errors": [],
5944 "size": 21,
5945 "snapset": {
5946 "clones": [],
5947 "snap_context": {
5948 "seq": 0,
5949 "snaps": []
5950 }
5951 }
5952 },
5953 {
5954 "osd": 1,
5955 "primary": true,
5956 "errors": [],
5957 "size": 21,
5958 "snapset": {
5959 "clones": [
5960 {
5961 "overlap": "[]",
5962 "size": 7,
5963 "snap": 1,
5964 "snaps": [
5965 1
5966 ]
5967 }
5968 ],
5969 "snap_context": {
5970 "seq": 1,
5971 "snaps": [
5972 1
5973 ]
5974 }
5975 }
5976 }
5977 ]
5978 }
5979 ]
5980 }
5981 EOF
5982
5983 jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
5984 multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
5985 if test $getjson = "yes"
5986 then
5987 jq '.' $dir/json > save6.json
5988 fi
5989
5990 if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
5991 then
5992 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
5993 fi
5994
5995 ERRORS=0
5996 declare -a err_strings
5997 err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid [0-9]*:.*:::ROBJ1:head : snapset inconsistent"
5998 err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid [0-9]*:.*:::ROBJ2:head : snapset inconsistent"
5999 err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 [0-9]*:.*:::ROBJ1:1 : is an unexpected clone"
6000 err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub : stat mismatch, got 3/4 objects, 1/2 clones, 3/4 dirty, 3/4 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 49/56 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
6001 err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 0 missing, 2 inconsistent objects"
6002 err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 4 errors"
6003
6004 for err_string in "${err_strings[@]}"
6005 do
6006 if ! grep -q "$err_string" $dir/osd.${primary}.log
6007 then
6008 echo "Missing log message '$err_string'"
6009 ERRORS=$(expr $ERRORS + 1)
6010 fi
6011 done
6012
6013 if [ $ERRORS != "0" ];
6014 then
6015 echo "TEST FAILED WITH $ERRORS ERRORS"
6016 return 1
6017 fi
6018
6019 ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it
6020 teardown $dir || return 1
6021 }
6022
6023 function TEST_request_scrub_priority() {
6024 local dir=$1
6025 local poolname=psr_pool
6026 local objname=POBJ
6027 local OBJECTS=64
6028 local PGS=8
6029
6030 setup $dir || return 1
6031 run_mon $dir a --osd_pool_default_size=1 || return 1
6032 run_mgr $dir x || return 1
6033 local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
6034 ceph_osd_args+="--osd_scrub_backoff_ratio=0"
6035 run_osd $dir 0 $ceph_osd_args || return 1
6036
6037 create_pool $poolname $PGS $PGS || return 1
6038 wait_for_clean || return 1
6039
6040 local osd=0
6041 add_something $dir $poolname $objname noscrub || return 1
6042 local primary=$(get_primary $poolname $objname)
6043 local pg=$(get_pg $poolname $objname)
6044 poolid=$(ceph osd dump | grep "^pool.*[']${poolname}[']" | awk '{ print $2 }')
6045
6046 local otherpgs
6047 for i in $(seq 0 $(expr $PGS - 1))
6048 do
6049 opg="${poolid}.${i}"
6050 if [ "$opg" = "$pg" ]; then
6051 continue
6052 fi
6053 otherpgs="${otherpgs}${opg} "
6054 local other_last_scrub=$(get_last_scrub_stamp $pg)
6055 # Fake a schedule scrub
6056 CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
6057 trigger_scrub $opg || return 1
6058 done
6059
6060 sleep 15
6061 flush_pg_stats
6062
6063 # Request a regular scrub and it will be done
6064 local last_scrub=$(get_last_scrub_stamp $pg)
6065 ceph pg scrub $pg
6066
6067 ceph osd unset noscrub || return 1
6068 ceph osd unset nodeep-scrub || return 1
6069
6070 wait_for_scrub $pg "$last_scrub"
6071
6072 for opg in $otherpgs $pg
6073 do
6074 wait_for_scrub $opg "$other_last_scrub"
6075 done
6076
6077 # Verify that the requested scrub ran first
6078 grep "log_channel.*scrub ok" $dir/osd.${primary}.log | head -1 | sed 's/.*[[]DBG[]]//' | grep -q $pg || return 1
6079
6080 return 0
6081 }
6082
6083
6084 main osd-scrub-repair "$@"
6085
6086 # Local Variables:
6087 # compile-command: "cd build ; make -j4 && \
6088 # ../qa/run-standalone.sh osd-scrub-repair.sh"
6089 # End:
Ceph