ceph/qa/standalone/scrub/osd-scrub-repair.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
   4 #
   5 # Author: Loic Dachary <loic@dachary.org>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17 set -x
  18 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  19
  20 if [ `uname` = FreeBSD ]; then
  21     # erasure coding overwrites are only tested on Bluestore
  22     # erasure coding on filestore is unsafe
  23     # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites
  24     use_ec_overwrite=false
  25 else
  26     use_ec_overwrite=true
  27 fi
  28
  29 # Test development and debugging
  30 # Set to "yes" in order to ignore diff errors and save results to update test
  31 getjson="no"
  32
  33 # Filter out mtime and local_mtime dates, version, prior_version and last_reqid (client) from any object_info.
  34 jqfilter='def walk(f):
  35   . as $in
  36   | if type == "object" then
  37       reduce keys[] as $key
  38         ( {}; . + { ($key):  ($in[$key] | walk(f)) } ) | f
  39     elif type == "array" then map( walk(f) ) | f
  40     else f
  41     end;
  42 walk(if type == "object" then del(.mtime) else . end)
  43 | walk(if type == "object" then del(.local_mtime) else . end)
  44 | walk(if type == "object" then del(.last_reqid) else . end)
  45 | walk(if type == "object" then del(.version) else . end)
  46 | walk(if type == "object" then del(.prior_version) else . end)'
  47
  48 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
  49
  50 function run() {
  51     local dir=$1
  52     shift
  53
  54     export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
  55     export CEPH_ARGS
  56     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  57     CEPH_ARGS+="--mon-host=$CEPH_MON "
  58     CEPH_ARGS+="--osd-skip-data-digest=false "
  59
  60     export -n CEPH_CLI_TEST_DUP_COMMAND
  61     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  62     for func in $funcs ; do
  63         $func $dir || return 1
  64     done
  65 }
  66
  67 function add_something() {
  68     local dir=$1
  69     local poolname=$2
  70     local obj=${3:-SOMETHING}
  71     local scrub=${4:-noscrub}
  72
  73     if [ "$scrub" = "noscrub" ];
  74     then
  75         ceph osd set noscrub || return 1
  76         ceph osd set nodeep-scrub || return 1
  77     else
  78         ceph osd unset noscrub || return 1
  79         ceph osd unset nodeep-scrub || return 1
  80     fi
  81
  82     local payload=ABCDEF
  83     echo $payload > $dir/ORIGINAL
  84     rados --pool $poolname put $obj $dir/ORIGINAL || return 1
  85 }
  86
  87 #
  88 # Corrupt one copy of a replicated pool
  89 #
  90 function TEST_corrupt_and_repair_replicated() {
  91     local dir=$1
  92     local poolname=rbd
  93
  94     setup $dir || return 1
  95     run_mon $dir a --osd_pool_default_size=2 || return 1
  96     run_mgr $dir x || return 1
  97     run_osd $dir 0 || return 1
  98     run_osd $dir 1 || return 1
  99     create_rbd_pool || return 1
 100     wait_for_clean || return 1
 101
 102     add_something $dir $poolname || return 1
 103     corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
 104     # Reproduces http://tracker.ceph.com/issues/8914
 105     corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1
 106
 107     teardown $dir || return 1
 108 }
 109
 110 #
 111 # Allow repair to be scheduled when some recovering is still undergoing on the same OSD
 112 #
 113 function TEST_allow_repair_during_recovery() {
 114     local dir=$1
 115     local poolname=rbd
 116
 117     setup $dir || return 1
 118     run_mon $dir a --osd_pool_default_size=2 || return 1
 119     run_mgr $dir x || return 1
 120     run_osd $dir 0 --osd_scrub_during_recovery=false \
 121                    --osd_repair_during_recovery=true \
 122                    --osd_debug_pretend_recovery_active=true || return 1
 123     run_osd $dir 1 --osd_scrub_during_recovery=false \
 124                    --osd_repair_during_recovery=true \
 125                    --osd_debug_pretend_recovery_active=true || return 1
 126     create_rbd_pool || return 1
 127     wait_for_clean || return 1
 128
 129     add_something $dir $poolname || return 1
 130     corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
 131
 132     teardown $dir || return 1
 133 }
 134
 135 #
 136 # Skip non-repair scrub correctly during recovery
 137 #
 138 function TEST_skip_non_repair_during_recovery() {
 139     local dir=$1
 140     local poolname=rbd
 141
 142     setup $dir || return 1
 143     run_mon $dir a --osd_pool_default_size=2 || return 1
 144     run_mgr $dir x || return 1
 145     run_osd $dir 0 --osd_scrub_during_recovery=false \
 146                    --osd_repair_during_recovery=true \
 147                    --osd_debug_pretend_recovery_active=true || return 1
 148     run_osd $dir 1 --osd_scrub_during_recovery=false \
 149                    --osd_repair_during_recovery=true \
 150                    --osd_debug_pretend_recovery_active=true || return 1
 151     create_rbd_pool || return 1
 152     wait_for_clean || return 1
 153
 154     add_something $dir $poolname || return 1
 155     scrub_and_not_schedule $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
 156
 157     teardown $dir || return 1
 158 }
 159
 160 function scrub_and_not_schedule() {
 161     local dir=$1
 162     local poolname=$2
 163     local osd=$3
 164
 165     #
 166     # 1) start a non-repair scrub
 167     #
 168     local pg=$(get_pg $poolname SOMETHING)
 169     local last_scrub=$(get_last_scrub_stamp $pg)
 170     ceph pg scrub $pg
 171
 172     #
 173     # 2) Assure the scrub is not scheduled
 174     #
 175     for ((i=0; i < 3; i++)); do
 176         if test "$(get_last_scrub_stamp $pg)" '>' "$last_scrub" ; then
 177             return 1
 178         fi
 179         sleep 1
 180     done
 181
 182     #
 183     # 3) Access to the file must OK
 184     #
 185     objectstore_tool $dir $osd SOMETHING list-attrs || return 1
 186     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 187     diff $dir/ORIGINAL $dir/COPY || return 1
 188 }
 189
 190 function corrupt_and_repair_two() {
 191     local dir=$1
 192     local poolname=$2
 193     local first=$3
 194     local second=$4
 195
 196     #
 197     # 1) remove the corresponding file from the OSDs
 198     #
 199     pids=""
 200     run_in_background pids objectstore_tool $dir $first SOMETHING remove
 201     run_in_background pids objectstore_tool $dir $second SOMETHING remove
 202     wait_background pids
 203     return_code=$?
 204     if [ $return_code -ne 0 ]; then return $return_code; fi
 205
 206     #
 207     # 2) repair the PG
 208     #
 209     local pg=$(get_pg $poolname SOMETHING)
 210     repair $pg
 211     #
 212     # 3) The files must be back
 213     #
 214     pids=""
 215     run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs
 216     run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs
 217     wait_background pids
 218     return_code=$?
 219     if [ $return_code -ne 0 ]; then return $return_code; fi
 220
 221     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 222     diff $dir/ORIGINAL $dir/COPY || return 1
 223 }
 224
 225 #
 226 # 1) add an object
 227 # 2) remove the corresponding file from a designated OSD
 228 # 3) repair the PG
 229 # 4) check that the file has been restored in the designated OSD
 230 #
 231 function corrupt_and_repair_one() {
 232     local dir=$1
 233     local poolname=$2
 234     local osd=$3
 235
 236     #
 237     # 1) remove the corresponding file from the OSD
 238     #
 239     objectstore_tool $dir $osd SOMETHING remove || return 1
 240     #
 241     # 2) repair the PG
 242     #
 243     local pg=$(get_pg $poolname SOMETHING)
 244     repair $pg
 245     #
 246     # 3) The file must be back
 247     #
 248     objectstore_tool $dir $osd SOMETHING list-attrs || return 1
 249     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 250     diff $dir/ORIGINAL $dir/COPY || return 1
 251 }
 252
 253 function corrupt_and_repair_erasure_coded() {
 254     local dir=$1
 255     local poolname=$2
 256
 257     add_something $dir $poolname || return 1
 258
 259     local primary=$(get_primary $poolname SOMETHING)
 260     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 261     local not_primary_first=${osds[0]}
 262     local not_primary_second=${osds[1]}
 263
 264     # Reproduces http://tracker.ceph.com/issues/10017
 265     corrupt_and_repair_one $dir $poolname $primary  || return 1
 266     # Reproduces http://tracker.ceph.com/issues/10409
 267     corrupt_and_repair_one $dir $poolname $not_primary_first || return 1
 268     corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1
 269     corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1
 270
 271 }
 272
 273 function auto_repair_erasure_coded() {
 274     local dir=$1
 275     local allow_overwrites=$2
 276     local poolname=ecpool
 277
 278     # Launch a cluster with 5 seconds scrub interval
 279     setup $dir || return 1
 280     run_mon $dir a || return 1
 281     run_mgr $dir x || return 1
 282     local ceph_osd_args="--osd-scrub-auto-repair=true \
 283             --osd-deep-scrub-interval=5 \
 284             --osd-scrub-max-interval=5 \
 285             --osd-scrub-min-interval=5 \
 286             --osd-scrub-interval-randomize-ratio=0"
 287     for id in $(seq 0 2) ; do
 288         if [ "$allow_overwrites" = "true" ]; then
 289             run_osd $dir $id $ceph_osd_args || return 1
 290         else
 291             run_osd_filestore $dir $id $ceph_osd_args || return 1
 292         fi
 293     done
 294     create_rbd_pool || return 1
 295     wait_for_clean || return 1
 296
 297     # Create an EC pool
 298     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 299
 300     # Put an object
 301     local payload=ABCDEF
 302     echo $payload > $dir/ORIGINAL
 303     rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
 304
 305     # Remove the object from one shard physically
 306     # Restarted osd get $ceph_osd_args passed
 307     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
 308     # Wait for auto repair
 309     local pgid=$(get_pg $poolname SOMETHING)
 310     wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)"
 311     wait_for_clean || return 1
 312     # Verify - the file should be back
 313     # Restarted osd get $ceph_osd_args passed
 314     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
 315     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 316     diff $dir/ORIGINAL $dir/COPY || return 1
 317
 318     # Tear down
 319     teardown $dir || return 1
 320 }
 321
 322 function TEST_auto_repair_erasure_coded_appends() {
 323     auto_repair_erasure_coded $1 false
 324 }
 325
 326 function TEST_auto_repair_erasure_coded_overwrites() {
 327     if [ "$use_ec_overwrite" = "true" ]; then
 328         auto_repair_erasure_coded $1 true
 329     fi
 330 }
 331
 332 function TEST_auto_repair_bluestore_basic() {
 333     local dir=$1
 334     local poolname=testpool
 335
 336     # Launch a cluster with 5 seconds scrub interval
 337     setup $dir || return 1
 338     run_mon $dir a || return 1
 339     run_mgr $dir x || return 1
 340     local ceph_osd_args="--osd-scrub-auto-repair=true \
 341             --osd_deep_scrub_randomize_ratio=0 \
 342             --osd-scrub-interval-randomize-ratio=0"
 343     for id in $(seq 0 2) ; do
 344         run_osd $dir $id $ceph_osd_args || return 1
 345     done
 346
 347     create_pool $poolname 1 1 || return 1
 348     ceph osd pool set $poolname size 2
 349     wait_for_clean || return 1
 350
 351     # Put an object
 352     local payload=ABCDEF
 353     echo $payload > $dir/ORIGINAL
 354     rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
 355
 356     # Remove the object from one shard physically
 357     # Restarted osd get $ceph_osd_args passed
 358     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
 359
 360     local pgid=$(get_pg $poolname SOMETHING)
 361     local primary=$(get_primary $poolname SOMETHING)
 362     local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
 363     CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_deep_scrub $pgid
 364     CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_scrub $pgid
 365
 366     # Wait for auto repair
 367     wait_for_scrub $pgid "$last_scrub_stamp" || return 1
 368     wait_for_clean || return 1
 369     ceph pg dump pgs
 370     # Verify - the file should be back
 371     # Restarted osd get $ceph_osd_args passed
 372     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
 373     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING get-bytes $dir/COPY || return 1
 374     diff $dir/ORIGINAL $dir/COPY || return 1
 375     grep scrub_finish $dir/osd.${primary}.log
 376
 377     # Tear down
 378     teardown $dir || return 1
 379 }
 380
 381 function TEST_auto_repair_bluestore_scrub() {
 382     local dir=$1
 383     local poolname=testpool
 384
 385     # Launch a cluster with 5 seconds scrub interval
 386     setup $dir || return 1
 387     run_mon $dir a || return 1
 388     run_mgr $dir x || return 1
 389     local ceph_osd_args="--osd-scrub-auto-repair=true \
 390             --osd_deep_scrub_randomize_ratio=0 \
 391             --osd-scrub-interval-randomize-ratio=0"
 392     for id in $(seq 0 2) ; do
 393         run_osd $dir $id $ceph_osd_args || return 1
 394     done
 395
 396     create_pool $poolname 1 1 || return 1
 397     ceph osd pool set $poolname size 2
 398     wait_for_clean || return 1
 399
 400     # Put an object
 401     local payload=ABCDEF
 402     echo $payload > $dir/ORIGINAL
 403     rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
 404
 405     # Remove the object from one shard physically
 406     # Restarted osd get $ceph_osd_args passed
 407     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
 408
 409     local pgid=$(get_pg $poolname SOMETHING)
 410     local primary=$(get_primary $poolname SOMETHING)
 411     local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
 412     CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_scrub $pgid
 413
 414     # Wait for scrub -> auto repair
 415     wait_for_scrub $pgid "$last_scrub_stamp" || return 1
 416     ceph pg dump pgs
 417     # Actually this causes 2 scrubs, so we better wait a little longer
 418     sleep 5
 419     wait_for_clean || return 1
 420     ceph pg dump pgs
 421     # Verify - the file should be back
 422     # Restarted osd get $ceph_osd_args passed
 423     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
 424     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 425     diff $dir/ORIGINAL $dir/COPY || return 1
 426     grep scrub_finish $dir/osd.${primary}.log
 427
 428     # This should have caused 1 object to be repaired
 429     COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
 430     test "$COUNT" = "1" || return 1
 431
 432     # Tear down
 433     teardown $dir || return 1
 434 }
 435
 436 function TEST_auto_repair_bluestore_failed() {
 437     local dir=$1
 438     local poolname=testpool
 439
 440     # Launch a cluster with 5 seconds scrub interval
 441     setup $dir || return 1
 442     run_mon $dir a || return 1
 443     run_mgr $dir x || return 1
 444     local ceph_osd_args="--osd-scrub-auto-repair=true \
 445             --osd_deep_scrub_randomize_ratio=0 \
 446             --osd-scrub-interval-randomize-ratio=0"
 447     for id in $(seq 0 2) ; do
 448         run_osd $dir $id $ceph_osd_args || return 1
 449     done
 450
 451     create_pool $poolname 1 1 || return 1
 452     ceph osd pool set $poolname size 2
 453     wait_for_clean || return 1
 454
 455     # Put an object
 456     local payload=ABCDEF
 457     echo $payload > $dir/ORIGINAL
 458     for i in $(seq 1 10)
 459     do
 460       rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
 461     done
 462
 463     # Remove the object from one shard physically
 464     # Restarted osd get $ceph_osd_args passed
 465     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj1 remove || return 1
 466     # obj2 can't be repaired
 467     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj2 remove || return 1
 468     objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 rm-attr _ || return 1
 469
 470     local pgid=$(get_pg $poolname obj1)
 471     local primary=$(get_primary $poolname obj1)
 472     local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
 473     CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_deep_scrub $pgid
 474     CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_scrub $pgid
 475
 476     # Wait for auto repair
 477     wait_for_scrub $pgid "$last_scrub_stamp" || return 1
 478     wait_for_clean || return 1
 479     flush_pg_stats
 480     grep scrub_finish $dir/osd.${primary}.log
 481     grep -q "scrub_finish.*still present after re-scrub" $dir/osd.${primary}.log || return 1
 482     ceph pg dump pgs
 483     ceph pg dump pgs | grep -q "^$(pgid).*+failed_repair" || return 1
 484
 485     # Verify - obj1 should be back
 486     # Restarted osd get $ceph_osd_args passed
 487     objectstore_tool $dir $(get_not_primary $poolname obj1) obj1 list-attrs || return 1
 488     rados --pool $poolname get obj1 $dir/COPY || return 1
 489     diff $dir/ORIGINAL $dir/COPY || return 1
 490     grep scrub_finish $dir/osd.${primary}.log
 491
 492     # Make it repairable
 493     objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 remove || return 1
 494     repair $pgid
 495     sleep 2
 496
 497     ceph pg dump pgs
 498     ceph pg dump pgs | grep -q "^$(pgid).* active+clean " || return 1
 499     grep scrub_finish $dir/osd.${primary}.log
 500
 501     # Tear down
 502     teardown $dir || return 1
 503 }
 504
 505 function TEST_auto_repair_bluestore_failed_norecov() {
 506     local dir=$1
 507     local poolname=testpool
 508
 509     # Launch a cluster with 5 seconds scrub interval
 510     setup $dir || return 1
 511     run_mon $dir a || return 1
 512     run_mgr $dir x || return 1
 513     local ceph_osd_args="--osd-scrub-auto-repair=true \
 514             --osd_deep_scrub_randomize_ratio=0 \
 515             --osd-scrub-interval-randomize-ratio=0"
 516     for id in $(seq 0 2) ; do
 517         run_osd $dir $id $ceph_osd_args || return 1
 518     done
 519
 520     create_pool $poolname 1 1 || return 1
 521     ceph osd pool set $poolname size 2
 522     wait_for_clean || return 1
 523
 524     # Put an object
 525     local payload=ABCDEF
 526     echo $payload > $dir/ORIGINAL
 527     for i in $(seq 1 10)
 528     do
 529       rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
 530     done
 531
 532     # Remove the object from one shard physically
 533     # Restarted osd get $ceph_osd_args passed
 534     # obj1 can't be repaired
 535     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj1 remove || return 1
 536     objectstore_tool $dir $(get_primary $poolname SOMETHING) obj1 rm-attr _ || return 1
 537     # obj2 can't be repaired
 538     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj2 remove || return 1
 539     objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 rm-attr _ || return 1
 540
 541     local pgid=$(get_pg $poolname obj1)
 542     local primary=$(get_primary $poolname obj1)
 543     local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
 544     CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_deep_scrub $pgid
 545     CEPH_ARGS='' ceph daemon $(get_asok_path osd.$primary) trigger_scrub $pgid
 546
 547     # Wait for auto repair
 548     wait_for_scrub $pgid "$last_scrub_stamp" || return 1
 549     wait_for_clean || return 1
 550     flush_pg_stats
 551     grep -q "scrub_finish.*present with no repair possible" $dir/osd.${primary}.log || return 1
 552     ceph pg dump pgs
 553     ceph pg dump pgs | grep -q "^$(pgid).*+failed_repair" || return 1
 554
 555     # Tear down
 556     teardown $dir || return 1
 557 }
 558
 559 function TEST_repair_stats() {
 560     local dir=$1
 561     local poolname=testpool
 562     local OSDS=2
 563     local OBJS=30
 564     # This need to be an even number
 565     local REPAIRS=20
 566
 567     # Launch a cluster with 5 seconds scrub interval
 568     setup $dir || return 1
 569     run_mon $dir a || return 1
 570     run_mgr $dir x || return 1
 571     local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \
 572             --osd-scrub-interval-randomize-ratio=0"
 573     for id in $(seq 0 $(expr $OSDS - 1)) ; do
 574         run_osd $dir $id $ceph_osd_args || return 1
 575     done
 576
 577     create_pool $poolname 1 1 || return 1
 578     ceph osd pool set $poolname size 2
 579     wait_for_clean || return 1
 580
 581     # Put an object
 582     local payload=ABCDEF
 583     echo $payload > $dir/ORIGINAL
 584     for i in $(seq 1 $OBJS)
 585     do
 586       rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
 587     done
 588
 589     # Remove the object from one shard physically
 590     # Restarted osd get $ceph_osd_args passed
 591     local other=$(get_not_primary $poolname obj1)
 592     local pgid=$(get_pg $poolname obj1)
 593     local primary=$(get_primary $poolname obj1)
 594
 595     kill_daemons $dir TERM osd.$other >&2 < /dev/null || return 1
 596     kill_daemons $dir TERM osd.$primary >&2 < /dev/null || return 1
 597     for i in $(seq 1 $REPAIRS)
 598     do
 599       # Remove from both osd.0 and osd.1
 600       OSD=$(expr $i % 2)
 601       _objectstore_tool_nodown $dir $OSD obj$i remove || return 1
 602     done
 603     run_osd $dir $primary $ceph_osd_args || return 1
 604     run_osd $dir $other $ceph_osd_args || return 1
 605     wait_for_clean || return 1
 606
 607     repair $pgid
 608     wait_for_clean || return 1
 609     ceph pg dump pgs
 610
 611     # This should have caused 1 object to be repaired
 612     ceph pg $pgid query | jq '.info.stats.stat_sum'
 613     COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
 614     test "$COUNT" = "$REPAIRS" || return 1
 615
 616     ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $primary )"
 617     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $primary ).num_shards_repaired")
 618     test "$COUNT" = "$(expr $REPAIRS / 2)" || return 1
 619
 620     ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $other )"
 621     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $other ).num_shards_repaired")
 622     test "$COUNT" = "$(expr $REPAIRS / 2)" || return 1
 623
 624     ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum"
 625     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
 626     test "$COUNT" = "$REPAIRS" || return 1
 627
 628     # Tear down
 629     teardown $dir || return 1
 630 }
 631
 632 function TEST_repair_stats_ec() {
 633     local dir=$1
 634     local poolname=testpool
 635     local OSDS=3
 636     local OBJS=30
 637     # This need to be an even number
 638     local REPAIRS=26
 639     local allow_overwrites=false
 640
 641     # Launch a cluster with 5 seconds scrub interval
 642     setup $dir || return 1
 643     run_mon $dir a || return 1
 644     run_mgr $dir x || return 1
 645     local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \
 646             --osd-scrub-interval-randomize-ratio=0"
 647     for id in $(seq 0 $(expr $OSDS - 1)) ; do
 648         run_osd $dir $id $ceph_osd_args || return 1
 649     done
 650
 651     # Create an EC pool
 652     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 653
 654     # Put an object
 655     local payload=ABCDEF
 656     echo $payload > $dir/ORIGINAL
 657     for i in $(seq 1 $OBJS)
 658     do
 659       rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
 660     done
 661
 662     # Remove the object from one shard physically
 663     # Restarted osd get $ceph_osd_args passed
 664     local other=$(get_not_primary $poolname obj1)
 665     local pgid=$(get_pg $poolname obj1)
 666     local primary=$(get_primary $poolname obj1)
 667
 668     kill_daemons $dir TERM osd.$other >&2 < /dev/null || return 1
 669     kill_daemons $dir TERM osd.$primary >&2 < /dev/null || return 1
 670     for i in $(seq 1 $REPAIRS)
 671     do
 672       # Remove from both osd.0 and osd.1
 673       OSD=$(expr $i % 2)
 674       _objectstore_tool_nodown $dir $OSD obj$i remove || return 1
 675     done
 676     run_osd $dir $primary $ceph_osd_args || return 1
 677     run_osd $dir $other $ceph_osd_args || return 1
 678     wait_for_clean || return 1
 679
 680     repair $pgid
 681     wait_for_clean || return 1
 682     ceph pg dump pgs
 683
 684     # This should have caused 1 object to be repaired
 685     ceph pg $pgid query | jq '.info.stats.stat_sum'
 686     COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
 687     test "$COUNT" = "$REPAIRS" || return 1
 688
 689     for osd in $(seq 0 $(expr $OSDS - 1)) ; do
 690       if [ $osd = $other -o $osd = $primary ]; then
 691         repair=$(expr $REPAIRS / 2)
 692       else
 693         repair="0"
 694       fi
 695
 696       ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $osd )"
 697       COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $osd ).num_shards_repaired")
 698       test "$COUNT" = "$repair" || return 1
 699     done
 700
 701     ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum"
 702     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
 703     test "$COUNT" = "$REPAIRS" || return 1
 704
 705     # Tear down
 706     teardown $dir || return 1
 707 }
 708
 709 function corrupt_and_repair_jerasure() {
 710     local dir=$1
 711     local allow_overwrites=$2
 712     local poolname=ecpool
 713
 714     setup $dir || return 1
 715     run_mon $dir a || return 1
 716     run_mgr $dir x || return 1
 717     for id in $(seq 0 3) ; do
 718         if [ "$allow_overwrites" = "true" ]; then
 719             run_osd $dir $id || return 1
 720         else
 721             run_osd_filestore $dir $id || return 1
 722         fi
 723     done
 724     create_rbd_pool || return 1
 725     wait_for_clean || return 1
 726
 727     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 728     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 729
 730     teardown $dir || return 1
 731 }
 732
 733 function TEST_corrupt_and_repair_jerasure_appends() {
 734     corrupt_and_repair_jerasure $1 false
 735 }
 736
 737 function TEST_corrupt_and_repair_jerasure_overwrites() {
 738     if [ "$use_ec_overwrite" = "true" ]; then
 739         corrupt_and_repair_jerasure $1 true
 740     fi
 741 }
 742
 743 function corrupt_and_repair_lrc() {
 744     local dir=$1
 745     local allow_overwrites=$2
 746     local poolname=ecpool
 747
 748     setup $dir || return 1
 749     run_mon $dir a || return 1
 750     run_mgr $dir x || return 1
 751     for id in $(seq 0 9) ; do
 752         if [ "$allow_overwrites" = "true" ]; then
 753             run_osd $dir $id || return 1
 754         else
 755             run_osd_filestore $dir $id || return 1
 756         fi
 757     done
 758     create_rbd_pool || return 1
 759     wait_for_clean || return 1
 760
 761     create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1
 762     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 763
 764     teardown $dir || return 1
 765 }
 766
 767 function TEST_corrupt_and_repair_lrc_appends() {
 768     corrupt_and_repair_lrc $1 false
 769 }
 770
 771 function TEST_corrupt_and_repair_lrc_overwrites() {
 772     if [ "$use_ec_overwrite" = "true" ]; then
 773         corrupt_and_repair_lrc $1 true
 774     fi
 775 }
 776
 777 function unfound_erasure_coded() {
 778     local dir=$1
 779     local allow_overwrites=$2
 780     local poolname=ecpool
 781     local payload=ABCDEF
 782
 783     setup $dir || return 1
 784     run_mon $dir a || return 1
 785     run_mgr $dir x || return 1
 786     for id in $(seq 0 3) ; do
 787         if [ "$allow_overwrites" = "true" ]; then
 788             run_osd $dir $id || return 1
 789         else
 790             run_osd_filestore $dir $id || return 1
 791         fi
 792     done
 793
 794     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 795
 796     add_something $dir $poolname || return 1
 797
 798     local primary=$(get_primary $poolname SOMETHING)
 799     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 800     local not_primary_first=${osds[0]}
 801     local not_primary_second=${osds[1]}
 802     local not_primary_third=${osds[2]}
 803
 804     #
 805     # 1) remove the corresponding file from the OSDs
 806     #
 807     pids=""
 808     run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove
 809     run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove
 810     run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove
 811     wait_background pids
 812     return_code=$?
 813     if [ $return_code -ne 0 ]; then return $return_code; fi
 814
 815     #
 816     # 2) repair the PG
 817     #
 818     local pg=$(get_pg $poolname SOMETHING)
 819     repair $pg
 820     #
 821     # 3) check pg state
 822     #
 823     # it may take a bit to appear due to mon/mgr asynchrony
 824     for f in `seq 1 60`; do
 825         ceph -s | grep "1/1 objects unfound" && break
 826         sleep 1
 827     done
 828     ceph -s|grep "4 up" || return 1
 829     ceph -s|grep "4 in" || return 1
 830     ceph -s|grep "1/1 objects unfound" || return 1
 831
 832     teardown $dir || return 1
 833 }
 834
 835 function TEST_unfound_erasure_coded_appends() {
 836     unfound_erasure_coded $1 false
 837 }
 838
 839 function TEST_unfound_erasure_coded_overwrites() {
 840     if [ "$use_ec_overwrite" = "true" ]; then
 841         unfound_erasure_coded $1 true
 842     fi
 843 }
 844
 845 #
 846 # list_missing for EC pool
 847 #
 848 function list_missing_erasure_coded() {
 849     local dir=$1
 850     local allow_overwrites=$2
 851     local poolname=ecpool
 852
 853     setup $dir || return 1
 854     run_mon $dir a || return 1
 855     run_mgr $dir x || return 1
 856     for id in $(seq 0 2) ; do
 857         if [ "$allow_overwrites" = "true" ]; then
 858             run_osd $dir $id || return 1
 859         else
 860             run_osd_filestore $dir $id || return 1
 861         fi
 862     done
 863     create_rbd_pool || return 1
 864     wait_for_clean || return 1
 865
 866     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 867
 868     # Put an object and remove the two shards (including primary)
 869     add_something $dir $poolname MOBJ0 || return 1
 870     local -a osds0=($(get_osds $poolname MOBJ0))
 871
 872     # Put another object and remove two shards (excluding primary)
 873     add_something $dir $poolname MOBJ1 || return 1
 874     local -a osds1=($(get_osds $poolname MOBJ1))
 875
 876     # Stop all osd daemons
 877     for id in $(seq 0 2) ; do
 878         kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1
 879     done
 880
 881     id=${osds0[0]}
 882     ceph-objectstore-tool --data-path $dir/$id \
 883         MOBJ0 remove || return 1
 884     id=${osds0[1]}
 885     ceph-objectstore-tool --data-path $dir/$id \
 886         MOBJ0 remove || return 1
 887
 888     id=${osds1[1]}
 889     ceph-objectstore-tool --data-path $dir/$id \
 890         MOBJ1 remove || return 1
 891     id=${osds1[2]}
 892     ceph-objectstore-tool --data-path $dir/$id \
 893         MOBJ1 remove || return 1
 894
 895     for id in $(seq 0 2) ; do
 896         activate_osd $dir $id >&2 || return 1
 897     done
 898     create_rbd_pool || return 1
 899     wait_for_clean || return 1
 900
 901     # Get get - both objects should in the same PG
 902     local pg=$(get_pg $poolname MOBJ0)
 903
 904     # Repair the PG, which triggers the recovering,
 905     # and should mark the object as unfound
 906     repair $pg
 907
 908     for i in $(seq 0 120) ; do
 909         [ $i -lt 60 ] || return 1
 910         matches=$(ceph pg $pg list_unfound | egrep "MOBJ0|MOBJ1" | wc -l)
 911         [ $matches -eq 2 ] && break
 912     done
 913
 914     teardown $dir || return 1
 915 }
 916
 917 function TEST_list_missing_erasure_coded_appends() {
 918     list_missing_erasure_coded $1 false
 919 }
 920
 921 function TEST_list_missing_erasure_coded_overwrites() {
 922     if [ "$use_ec_overwrite" = "true" ]; then
 923         list_missing_erasure_coded $1 true
 924     fi
 925 }
 926
 927 #
 928 # Corrupt one copy of a replicated pool
 929 #
 930 function TEST_corrupt_scrub_replicated() {
 931     local dir=$1
 932     local poolname=csr_pool
 933     local total_objs=19
 934
 935     setup $dir || return 1
 936     run_mon $dir a --osd_pool_default_size=2 || return 1
 937     run_mgr $dir x || return 1
 938     run_osd $dir 0 || return 1
 939     run_osd $dir 1 || return 1
 940     create_rbd_pool || return 1
 941     wait_for_clean || return 1
 942
 943     create_pool foo 1 || return 1
 944     create_pool $poolname 1 1 || return 1
 945     wait_for_clean || return 1
 946
 947     for i in $(seq 1 $total_objs) ; do
 948         objname=ROBJ${i}
 949         add_something $dir $poolname $objname || return 1
 950
 951         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
 952         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
 953     done
 954
 955     # Increase file 1 MB + 1KB
 956     dd if=/dev/zero of=$dir/new.ROBJ19 bs=1024 count=1025
 957     rados --pool $poolname put $objname $dir/new.ROBJ19 || return 1
 958     rm -f $dir/new.ROBJ19
 959
 960     local pg=$(get_pg $poolname ROBJ0)
 961     local primary=$(get_primary $poolname ROBJ0)
 962
 963     # Compute an old omap digest and save oi
 964     CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \
 965         config set osd_deep_scrub_update_digest_min_age 0
 966     CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \
 967         config set osd_deep_scrub_update_digest_min_age 0
 968     pg_deep_scrub $pg
 969
 970     for i in $(seq 1 $total_objs) ; do
 971         objname=ROBJ${i}
 972
 973         # Alternate corruption between osd.0 and osd.1
 974         local osd=$(expr $i % 2)
 975
 976         case $i in
 977         1)
 978             # Size (deep scrub data_digest too)
 979             local payload=UVWXYZZZ
 980             echo $payload > $dir/CORRUPT
 981             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 982             ;;
 983
 984         2)
 985             # digest (deep scrub only)
 986             local payload=UVWXYZ
 987             echo $payload > $dir/CORRUPT
 988             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 989             ;;
 990
 991         3)
 992              # missing
 993              objectstore_tool $dir $osd $objname remove || return 1
 994              ;;
 995
 996          4)
 997              # Modify omap value (deep scrub only)
 998              objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1
 999              ;;
1000
1001          5)
1002             # Delete omap key (deep scrub only)
1003             objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1
1004             ;;
1005
1006          6)
1007             # Add extra omap key (deep scrub only)
1008             echo extra > $dir/extra-val
1009             objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1
1010             rm $dir/extra-val
1011             ;;
1012
1013          7)
1014             # Modify omap header (deep scrub only)
1015             echo -n newheader > $dir/hdr
1016             objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1
1017             rm $dir/hdr
1018             ;;
1019
1020          8)
1021             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
1022             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
1023
1024             # Break xattrs
1025             echo -n bad-val > $dir/bad-val
1026             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
1027             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
1028             echo -n val3-$objname > $dir/newval
1029             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
1030             rm $dir/bad-val $dir/newval
1031             ;;
1032
1033         9)
1034             objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi
1035             echo -n D > $dir/change
1036             rados --pool $poolname put $objname $dir/change
1037             objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi
1038             rm $dir/oi $dir/change
1039             ;;
1040
1041           # ROBJ10 must be handled after digests are re-computed by a deep scrub below
1042           # ROBJ11 must be handled with config change before deep scrub
1043           # ROBJ12 must be handled with config change before scrubs
1044           # ROBJ13 must be handled before scrubs
1045
1046         14)
1047             echo -n bad-val > $dir/bad-val
1048             objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1
1049             objectstore_tool $dir 1 $objname rm-attr _ || return 1
1050             rm $dir/bad-val
1051             ;;
1052
1053         15)
1054             objectstore_tool $dir $osd $objname rm-attr _ || return 1
1055             ;;
1056
1057         16)
1058             objectstore_tool $dir 0 $objname rm-attr snapset || return 1
1059             echo -n bad-val > $dir/bad-val
1060             objectstore_tool $dir 1 $objname set-attr snapset $dir/bad-val || return 1
1061             ;;
1062
1063         17)
1064             # Deep-scrub only (all replicas are diffent than the object info
1065            local payload=ROBJ17
1066            echo $payload > $dir/new.ROBJ17
1067            objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ17 || return 1
1068            objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ17 || return 1
1069            ;;
1070
1071         18)
1072             # Deep-scrub only (all replicas are diffent than the object info
1073            local payload=ROBJ18
1074            echo $payload > $dir/new.ROBJ18
1075            objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ18 || return 1
1076            objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ18 || return 1
1077            # Make one replica have a different object info, so a full repair must happen too
1078            objectstore_tool $dir $osd $objname corrupt-info || return 1
1079            ;;
1080
1081         19)
1082            # Set osd-max-object-size smaller than this object's size
1083
1084         esac
1085     done
1086
1087     local pg=$(get_pg $poolname ROBJ0)
1088
1089     ceph tell osd.\* injectargs -- --osd-max-object-size=1048576
1090
1091     inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1092     inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1093     inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1094     inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1095
1096     pg_scrub $pg
1097
1098     ERRORS=0
1099     declare -a s_err_strings
1100     err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:30259878:::ROBJ15:head : candidate had a missing info key"
1101     err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:33aca486:::ROBJ18:head : object info inconsistent "
1102     err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:5c7b2c47:::ROBJ16:head : candidate had a corrupt snapset"
1103     err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:5c7b2c47:::ROBJ16:head : candidate had a missing snapset key"
1104     err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:5c7b2c47:::ROBJ16:head : failed to pick suitable object info"
1105     err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:86586531:::ROBJ8:head : attr value mismatch '_key1-ROBJ8', attr name mismatch '_key3-ROBJ8', attr name mismatch '_key2-ROBJ8'"
1106     err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:bc819597:::ROBJ12:head : candidate had a stat error"
1107     err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:c0c86b1d:::ROBJ14:head : candidate had a missing info key"
1108     err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:c0c86b1d:::ROBJ14:head : candidate had a corrupt info"
1109     err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:c0c86b1d:::ROBJ14:head : failed to pick suitable object info"
1110     err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : candidate size 9 info size 7 mismatch"
1111     err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : size 9 != size 7 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from shard 0"
1112     err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:d60617f9:::ROBJ13:head : candidate had a stat error"
1113     err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 3:f2a5b2a4:::ROBJ3:head : missing"
1114     err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ffdb2004:::ROBJ9:head : candidate size 1 info size 7 mismatch"
1115     err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ffdb2004:::ROBJ9:head : object info inconsistent "
1116     err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 3:c0c86b1d:::ROBJ14:head : no '_' attr"
1117     err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 3:5c7b2c47:::ROBJ16:head : can't decode 'snapset' attr buffer::malformed_input: .* no longer understand old encoding version 3 < 97"
1118     err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub : stat mismatch, got 19/19 objects, 0/0 clones, 18/19 dirty, 18/19 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 1049713/1049720 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
1119     err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 1 missing, 8 inconsistent objects"
1120     err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 18 errors"
1121     err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:123a5f55:::ROBJ19:head : size 1049600 > 1048576 is too large"
1122
1123     for err_string in "${err_strings[@]}"
1124     do
1125         if ! grep -q "$err_string" $dir/osd.${primary}.log
1126         then
1127             echo "Missing log message '$err_string'"
1128             ERRORS=$(expr $ERRORS + 1)
1129         fi
1130     done
1131
1132     rados list-inconsistent-pg $poolname > $dir/json || return 1
1133     # Check pg count
1134     test $(jq '. | length' $dir/json) = "1" || return 1
1135     # Check pgid
1136     test $(jq -r '.[0]' $dir/json) = $pg || return 1
1137
1138     rados list-inconsistent-obj $pg > $dir/json || return 1
1139     # Get epoch for repair-get requests
1140     epoch=$(jq .epoch $dir/json)
1141
1142     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
1143 {
1144   "inconsistents": [
1145     {
1146       "shards": [
1147         {
1148           "size": 7,
1149           "errors": [],
1150           "osd": 0,
1151           "primary": false
1152         },
1153         {
1154           "object_info": {
1155             "oid": {
1156               "oid": "ROBJ1",
1157               "key": "",
1158               "snapid": -2,
1159               "hash": 1454963827,
1160               "max": 0,
1161               "pool": 3,
1162               "namespace": ""
1163             },
1164             "version": "51'58",
1165             "prior_version": "21'3",
1166             "last_reqid": "osd.1.0:57",
1167             "user_version": 3,
1168             "size": 7,
1169             "mtime": "",
1170             "local_mtime": "",
1171             "lost": 0,
1172             "flags": [
1173               "dirty",
1174               "omap",
1175               "data_digest",
1176               "omap_digest"
1177             ],
1178             "truncate_seq": 0,
1179             "truncate_size": 0,
1180             "data_digest": "0x2ddbf8f5",
1181             "omap_digest": "0xf5fba2c6",
1182             "expected_object_size": 0,
1183             "expected_write_size": 0,
1184             "alloc_hint_flags": 0,
1185             "manifest": {
1186               "type": 0
1187             },
1188             "watchers": {}
1189           },
1190           "size": 9,
1191           "errors": [
1192             "size_mismatch_info",
1193             "obj_size_info_mismatch"
1194           ],
1195           "osd": 1,
1196           "primary": true
1197         }
1198       ],
1199       "selected_object_info": {
1200         "oid": {
1201           "oid": "ROBJ1",
1202           "key": "",
1203           "snapid": -2,
1204           "hash": 1454963827,
1205           "max": 0,
1206           "pool": 3,
1207           "namespace": ""
1208         },
1209         "version": "51'58",
1210         "prior_version": "21'3",
1211         "last_reqid": "osd.1.0:57",
1212         "user_version": 3,
1213         "size": 7,
1214         "mtime": "2018-04-05 14:33:19.804040",
1215         "local_mtime": "2018-04-05 14:33:19.804839",
1216         "lost": 0,
1217         "flags": [
1218           "dirty",
1219           "omap",
1220           "data_digest",
1221           "omap_digest"
1222         ],
1223         "truncate_seq": 0,
1224         "truncate_size": 0,
1225         "data_digest": "0x2ddbf8f5",
1226         "omap_digest": "0xf5fba2c6",
1227         "expected_object_size": 0,
1228         "expected_write_size": 0,
1229         "alloc_hint_flags": 0,
1230         "manifest": {
1231           "type": 0
1232         },
1233         "watchers": {}
1234       },
1235       "union_shard_errors": [
1236         "size_mismatch_info",
1237         "obj_size_info_mismatch"
1238       ],
1239       "errors": [
1240         "size_mismatch"
1241       ],
1242       "object": {
1243         "version": 3,
1244         "snap": "head",
1245         "locator": "",
1246         "nspace": "",
1247         "name": "ROBJ1"
1248       }
1249     },
1250     {
1251       "shards": [
1252         {
1253           "errors": [
1254             "stat_error"
1255           ],
1256           "osd": 0,
1257           "primary": false
1258         },
1259         {
1260           "size": 7,
1261           "errors": [],
1262           "osd": 1,
1263           "primary": true
1264         }
1265       ],
1266       "selected_object_info": {
1267         "oid": {
1268           "oid": "ROBJ12",
1269           "key": "",
1270           "snapid": -2,
1271           "hash": 3920199997,
1272           "max": 0,
1273           "pool": 3,
1274           "namespace": ""
1275         },
1276         "version": "51'56",
1277         "prior_version": "43'36",
1278         "last_reqid": "osd.1.0:55",
1279         "user_version": 36,
1280         "size": 7,
1281         "mtime": "",
1282         "local_mtime": "",
1283         "lost": 0,
1284         "flags": [
1285           "dirty",
1286           "omap",
1287           "data_digest",
1288           "omap_digest"
1289         ],
1290         "truncate_seq": 0,
1291         "truncate_size": 0,
1292         "data_digest": "0x2ddbf8f5",
1293         "omap_digest": "0x067f306a",
1294         "expected_object_size": 0,
1295         "expected_write_size": 0,
1296         "alloc_hint_flags": 0,
1297         "manifest": {
1298           "type": 0
1299         },
1300         "watchers": {}
1301       },
1302       "union_shard_errors": [
1303         "stat_error"
1304       ],
1305       "errors": [],
1306       "object": {
1307         "version": 36,
1308         "snap": "head",
1309         "locator": "",
1310         "nspace": "",
1311         "name": "ROBJ12"
1312       }
1313     },
1314     {
1315       "shards": [
1316         {
1317           "errors": [
1318             "stat_error"
1319           ],
1320           "osd": 0,
1321           "primary": false
1322         },
1323         {
1324           "size": 7,
1325           "errors": [],
1326           "osd": 1,
1327           "primary": true
1328         }
1329       ],
1330       "selected_object_info": {
1331         "oid": {
1332           "oid": "ROBJ13",
1333           "key": "",
1334           "snapid": -2,
1335           "hash": 2682806379,
1336           "max": 0,
1337           "pool": 3,
1338           "namespace": ""
1339         },
1340         "version": "51'59",
1341         "prior_version": "45'39",
1342         "last_reqid": "osd.1.0:58",
1343         "user_version": 39,
1344         "size": 7,
1345         "mtime": "",
1346         "local_mtime": "",
1347         "lost": 0,
1348         "flags": [
1349           "dirty",
1350           "omap",
1351           "data_digest",
1352           "omap_digest"
1353         ],
1354         "truncate_seq": 0,
1355         "truncate_size": 0,
1356         "data_digest": "0x2ddbf8f5",
1357         "omap_digest": "0x6441854d",
1358         "expected_object_size": 0,
1359         "expected_write_size": 0,
1360         "alloc_hint_flags": 0,
1361         "manifest": {
1362           "type": 0
1363         },
1364         "watchers": {}
1365       },
1366       "union_shard_errors": [
1367         "stat_error"
1368       ],
1369       "errors": [],
1370       "object": {
1371         "version": 39,
1372         "snap": "head",
1373         "locator": "",
1374         "nspace": "",
1375         "name": "ROBJ13"
1376       }
1377     },
1378     {
1379       "shards": [
1380         {
1381           "object_info": "bad-val",
1382           "size": 7,
1383           "errors": [
1384             "info_corrupted"
1385           ],
1386           "osd": 0,
1387           "primary": false
1388         },
1389         {
1390           "size": 7,
1391           "errors": [
1392             "info_missing"
1393           ],
1394           "osd": 1,
1395           "primary": true
1396         }
1397       ],
1398       "union_shard_errors": [
1399         "info_missing",
1400         "info_corrupted"
1401       ],
1402       "errors": [],
1403       "object": {
1404         "version": 0,
1405         "snap": "head",
1406         "locator": "",
1407         "nspace": "",
1408         "name": "ROBJ14"
1409       }
1410     },
1411     {
1412       "shards": [
1413         {
1414           "object_info": {
1415             "oid": {
1416               "oid": "ROBJ15",
1417               "key": "",
1418               "snapid": -2,
1419               "hash": 504996876,
1420               "max": 0,
1421               "pool": 3,
1422               "namespace": ""
1423             },
1424             "version": "51'49",
1425             "prior_version": "49'45",
1426             "last_reqid": "osd.1.0:48",
1427             "user_version": 45,
1428             "size": 7,
1429             "mtime": "2018-04-05 14:33:29.498969",
1430             "local_mtime": "2018-04-05 14:33:29.499890",
1431             "lost": 0,
1432             "flags": [
1433               "dirty",
1434               "omap",
1435               "data_digest",
1436               "omap_digest"
1437             ],
1438             "truncate_seq": 0,
1439             "truncate_size": 0,
1440             "data_digest": "0x2ddbf8f5",
1441             "omap_digest": "0x2d2a4d6e",
1442             "expected_object_size": 0,
1443             "expected_write_size": 0,
1444             "alloc_hint_flags": 0,
1445             "manifest": {
1446               "type": 0
1447             },
1448             "watchers": {}
1449           },
1450           "size": 7,
1451           "errors": [],
1452           "osd": 0,
1453           "primary": false
1454         },
1455         {
1456           "size": 7,
1457           "errors": [
1458             "info_missing"
1459           ],
1460           "osd": 1,
1461           "primary": true
1462         }
1463       ],
1464       "selected_object_info": {
1465         "oid": {
1466           "oid": "ROBJ15",
1467           "key": "",
1468           "snapid": -2,
1469           "hash": 504996876,
1470           "max": 0,
1471           "pool": 3,
1472           "namespace": ""
1473         },
1474         "version": "51'49",
1475         "prior_version": "49'45",
1476         "last_reqid": "osd.1.0:48",
1477         "user_version": 45,
1478         "size": 7,
1479         "mtime": "",
1480         "local_mtime": "",
1481         "lost": 0,
1482         "flags": [
1483           "dirty",
1484           "omap",
1485           "data_digest",
1486           "omap_digest"
1487         ],
1488         "truncate_seq": 0,
1489         "truncate_size": 0,
1490         "data_digest": "0x2ddbf8f5",
1491         "omap_digest": "0x2d2a4d6e",
1492         "expected_object_size": 0,
1493         "expected_write_size": 0,
1494         "alloc_hint_flags": 0,
1495         "manifest": {
1496           "type": 0
1497         },
1498         "watchers": {}
1499       },
1500       "union_shard_errors": [
1501         "info_missing"
1502       ],
1503       "errors": [],
1504       "object": {
1505         "version": 45,
1506         "snap": "head",
1507         "locator": "",
1508         "nspace": "",
1509         "name": "ROBJ15"
1510       }
1511     },
1512     {
1513       "errors": [],
1514       "object": {
1515       "locator": "",
1516       "name": "ROBJ16",
1517       "nspace": "",
1518       "snap": "head",
1519       "version": 0
1520        },
1521         "shards": [
1522       {
1523         "errors": [
1524           "snapset_missing"
1525         ],
1526         "osd": 0,
1527         "primary": false,
1528         "size": 7
1529       },
1530       {
1531         "errors": [
1532           "snapset_corrupted"
1533         ],
1534         "osd": 1,
1535         "primary": true,
1536         "snapset": "bad-val",
1537         "size": 7
1538       }
1539       ],
1540       "union_shard_errors": [
1541         "snapset_missing",
1542         "snapset_corrupted"
1543       ]
1544     },
1545     {
1546      "errors": [
1547        "object_info_inconsistency"
1548      ],
1549      "object": {
1550        "locator": "",
1551        "name": "ROBJ18",
1552        "nspace": "",
1553        "snap": "head"
1554      },
1555      "selected_object_info": {
1556        "alloc_hint_flags": 255,
1557        "data_digest": "0x2ddbf8f5",
1558        "expected_object_size": 0,
1559        "expected_write_size": 0,
1560        "flags": [
1561          "dirty",
1562          "omap",
1563          "data_digest",
1564          "omap_digest"
1565        ],
1566        "lost": 0,
1567        "manifest": {
1568          "type": 0
1569        },
1570        "oid": {
1571          "hash": 1629828556,
1572          "key": "",
1573          "max": 0,
1574          "namespace": "",
1575          "oid": "ROBJ18",
1576          "pool": 3,
1577          "snapid": -2
1578        },
1579        "omap_digest": "0xddc3680f",
1580        "size": 7,
1581        "truncate_seq": 0,
1582        "truncate_size": 0,
1583        "user_version": 54,
1584        "watchers": {}
1585      },
1586      "shards": [
1587        {
1588          "errors": [],
1589          "object_info": {
1590            "alloc_hint_flags": 0,
1591            "data_digest": "0x2ddbf8f5",
1592            "expected_object_size": 0,
1593            "expected_write_size": 0,
1594            "flags": [
1595              "dirty",
1596              "omap",
1597              "data_digest",
1598              "omap_digest"
1599            ],
1600            "lost": 0,
1601            "manifest": {
1602              "type": 0
1603            },
1604            "oid": {
1605              "hash": 1629828556,
1606              "key": "",
1607              "max": 0,
1608              "namespace": "",
1609              "oid": "ROBJ18",
1610              "pool": 3,
1611              "snapid": -2
1612            },
1613            "omap_digest": "0xddc3680f",
1614            "size": 7,
1615            "truncate_seq": 0,
1616            "truncate_size": 0,
1617            "user_version": 54,
1618            "watchers": {}
1619          },
1620          "osd": 0,
1621          "primary": false,
1622          "size": 7
1623        },
1624        {
1625          "errors": [],
1626          "object_info": {
1627            "alloc_hint_flags": 255,
1628            "data_digest": "0x2ddbf8f5",
1629            "expected_object_size": 0,
1630            "expected_write_size": 0,
1631            "flags": [
1632              "dirty",
1633              "omap",
1634              "data_digest",
1635              "omap_digest"
1636            ],
1637            "lost": 0,
1638            "manifest": {
1639              "type": 0
1640            },
1641            "oid": {
1642              "hash": 1629828556,
1643              "key": "",
1644              "max": 0,
1645              "namespace": "",
1646              "oid": "ROBJ18",
1647              "pool": 3,
1648              "snapid": -2
1649            },
1650            "omap_digest": "0xddc3680f",
1651            "size": 7,
1652            "truncate_seq": 0,
1653            "truncate_size": 0,
1654            "user_version": 54,
1655            "watchers": {}
1656          },
1657          "osd": 1,
1658          "primary": true,
1659          "size": 7
1660        }
1661      ],
1662      "union_shard_errors": []
1663    },
1664    {
1665       "object": {
1666         "name": "ROBJ19",
1667         "nspace": "",
1668         "locator": "",
1669         "snap": "head",
1670         "version": 58
1671       },
1672       "errors": [
1673         "size_too_large"
1674       ],
1675       "union_shard_errors": [],
1676       "selected_object_info": {
1677         "oid": {
1678           "oid": "ROBJ19",
1679           "key": "",
1680           "snapid": -2,
1681           "hash": 2868534344,
1682           "max": 0,
1683           "pool": 3,
1684           "namespace": ""
1685         },
1686         "version": "63'59",
1687         "prior_version": "63'58",
1688         "last_reqid": "osd.1.0:58",
1689         "user_version": 58,
1690         "size": 1049600,
1691         "mtime": "2019-08-09T23:33:58.340709+0000",
1692         "local_mtime": "2019-08-09T23:33:58.345676+0000",
1693         "lost": 0,
1694         "flags": [
1695           "dirty",
1696           "omap",
1697           "data_digest",
1698           "omap_digest"
1699         ],
1700         "truncate_seq": 0,
1701         "truncate_size": 0,
1702         "data_digest": "0x3dde0ef3",
1703         "omap_digest": "0xbffddd28",
1704         "expected_object_size": 0,
1705         "expected_write_size": 0,
1706         "alloc_hint_flags": 0,
1707         "manifest": {
1708           "type": 0
1709         },
1710         "watchers": {}
1711       },
1712       "shards": [
1713         {
1714           "osd": 0,
1715           "primary": false,
1716           "errors": [],
1717           "size": 1049600
1718         },
1719         {
1720           "osd": 1,
1721           "primary": true,
1722           "errors": [],
1723           "size": 1049600
1724         }
1725       ]
1726    },
1727    {
1728       "shards": [
1729         {
1730           "size": 7,
1731           "errors": [],
1732           "osd": 0,
1733           "primary": false
1734         },
1735         {
1736           "errors": [
1737             "missing"
1738           ],
1739           "osd": 1,
1740           "primary": true
1741         }
1742       ],
1743       "selected_object_info": {
1744         "oid": {
1745           "oid": "ROBJ3",
1746           "key": "",
1747           "snapid": -2,
1748           "hash": 625845583,
1749           "max": 0,
1750           "pool": 3,
1751           "namespace": ""
1752         },
1753         "version": "51'61",
1754         "prior_version": "25'9",
1755         "last_reqid": "osd.1.0:60",
1756         "user_version": 9,
1757         "size": 7,
1758         "mtime": "",
1759         "local_mtime": "",
1760         "lost": 0,
1761         "flags": [
1762           "dirty",
1763           "omap",
1764           "data_digest",
1765           "omap_digest"
1766         ],
1767         "truncate_seq": 0,
1768         "truncate_size": 0,
1769         "data_digest": "0x2ddbf8f5",
1770         "omap_digest": "0x00b35dfd",
1771         "expected_object_size": 0,
1772         "expected_write_size": 0,
1773         "alloc_hint_flags": 0,
1774         "manifest": {
1775           "type": 0
1776         },
1777         "watchers": {}
1778       },
1779       "union_shard_errors": [
1780         "missing"
1781       ],
1782       "errors": [],
1783       "object": {
1784         "version": 9,
1785         "snap": "head",
1786         "locator": "",
1787         "nspace": "",
1788         "name": "ROBJ3"
1789       }
1790     },
1791     {
1792       "shards": [
1793         {
1794           "attrs": [
1795             {
1796               "Base64": false,
1797               "value": "bad-val",
1798               "name": "key1-ROBJ8"
1799             },
1800             {
1801               "Base64": false,
1802               "value": "val2-ROBJ8",
1803               "name": "key2-ROBJ8"
1804             }
1805           ],
1806           "size": 7,
1807           "errors": [],
1808           "osd": 0,
1809           "primary": false
1810         },
1811         {
1812           "attrs": [
1813             {
1814               "Base64": false,
1815               "value": "val1-ROBJ8",
1816               "name": "key1-ROBJ8"
1817             },
1818             {
1819               "Base64": false,
1820               "value": "val3-ROBJ8",
1821               "name": "key3-ROBJ8"
1822             }
1823           ],
1824           "size": 7,
1825           "errors": [],
1826           "osd": 1,
1827           "primary": true
1828         }
1829       ],
1830       "selected_object_info": {
1831         "oid": {
1832           "oid": "ROBJ8",
1833           "key": "",
1834           "snapid": -2,
1835           "hash": 2359695969,
1836           "max": 0,
1837           "pool": 3,
1838           "namespace": ""
1839         },
1840         "version": "79'66",
1841         "prior_version": "79'65",
1842         "last_reqid": "client.4554.0:1",
1843         "user_version": 79,
1844         "size": 7,
1845         "mtime": "",
1846         "local_mtime": "",
1847         "lost": 0,
1848         "flags": [
1849           "dirty",
1850           "omap",
1851           "data_digest",
1852           "omap_digest"
1853         ],
1854         "truncate_seq": 0,
1855         "truncate_size": 0,
1856         "data_digest": "0x2ddbf8f5",
1857         "omap_digest": "0xd6be81dc",
1858         "expected_object_size": 0,
1859         "expected_write_size": 0,
1860         "alloc_hint_flags": 0,
1861         "manifest": {
1862           "type": 0
1863         },
1864         "watchers": {}
1865       },
1866       "union_shard_errors": [],
1867       "errors": [
1868         "attr_value_mismatch",
1869         "attr_name_mismatch"
1870       ],
1871       "object": {
1872         "version": 66,
1873         "snap": "head",
1874         "locator": "",
1875         "nspace": "",
1876         "name": "ROBJ8"
1877       }
1878     },
1879     {
1880       "shards": [
1881         {
1882           "object_info": {
1883             "oid": {
1884               "oid": "ROBJ9",
1885               "key": "",
1886               "snapid": -2,
1887               "hash": 537189375,
1888               "max": 0,
1889               "pool": 3,
1890               "namespace": ""
1891             },
1892             "version": "95'67",
1893             "prior_version": "51'64",
1894             "last_reqid": "client.4649.0:1",
1895             "user_version": 80,
1896             "size": 1,
1897             "mtime": "",
1898             "local_mtime": "",
1899             "lost": 0,
1900             "flags": [
1901               "dirty",
1902               "omap",
1903               "data_digest",
1904               "omap_digest"
1905             ],
1906             "truncate_seq": 0,
1907             "truncate_size": 0,
1908             "data_digest": "0x2b63260d",
1909             "omap_digest": "0x2eecc539",
1910             "expected_object_size": 0,
1911             "expected_write_size": 0,
1912             "alloc_hint_flags": 0,
1913             "manifest": {
1914               "type": 0
1915             },
1916             "watchers": {}
1917           },
1918           "size": 1,
1919           "errors": [],
1920           "osd": 0,
1921           "primary": false
1922         },
1923         {
1924           "object_info": {
1925             "oid": {
1926               "oid": "ROBJ9",
1927               "key": "",
1928               "snapid": -2,
1929               "hash": 537189375,
1930               "max": 0,
1931               "pool": 3,
1932               "namespace": ""
1933             },
1934             "version": "51'64",
1935             "prior_version": "37'27",
1936             "last_reqid": "osd.1.0:63",
1937             "user_version": 27,
1938             "size": 7,
1939             "mtime": "2018-04-05 14:33:25.352485",
1940             "local_mtime": "2018-04-05 14:33:25.353746",
1941             "lost": 0,
1942             "flags": [
1943               "dirty",
1944               "omap",
1945               "data_digest",
1946               "omap_digest"
1947             ],
1948             "truncate_seq": 0,
1949             "truncate_size": 0,
1950             "data_digest": "0x2ddbf8f5",
1951             "omap_digest": "0x2eecc539",
1952             "expected_object_size": 0,
1953             "expected_write_size": 0,
1954             "alloc_hint_flags": 0,
1955             "manifest": {
1956               "type": 0
1957             },
1958             "watchers": {}
1959           },
1960           "size": 1,
1961           "errors": [
1962             "obj_size_info_mismatch"
1963           ],
1964           "osd": 1,
1965           "primary": true
1966         }
1967       ],
1968       "selected_object_info": {
1969         "oid": {
1970           "oid": "ROBJ9",
1971           "key": "",
1972           "snapid": -2,
1973           "hash": 537189375,
1974           "max": 0,
1975           "pool": 3,
1976           "namespace": ""
1977         },
1978         "version": "95'67",
1979         "prior_version": "51'64",
1980         "last_reqid": "client.4649.0:1",
1981         "user_version": 80,
1982         "size": 1,
1983         "mtime": "",
1984         "local_mtime": "",
1985         "lost": 0,
1986         "flags": [
1987           "dirty",
1988           "omap",
1989           "data_digest",
1990           "omap_digest"
1991         ],
1992         "truncate_seq": 0,
1993         "truncate_size": 0,
1994         "data_digest": "0x2b63260d",
1995         "omap_digest": "0x2eecc539",
1996         "expected_object_size": 0,
1997         "expected_write_size": 0,
1998         "alloc_hint_flags": 0,
1999         "manifest": {
2000           "type": 0
2001         },
2002         "watchers": {}
2003       },
2004       "union_shard_errors": [
2005          "obj_size_info_mismatch"
2006       ],
2007       "errors": [
2008         "object_info_inconsistency"
2009       ],
2010       "object": {
2011         "version": 67,
2012         "snap": "head",
2013         "locator": "",
2014         "nspace": "",
2015         "name": "ROBJ9"
2016       }
2017     }
2018   ],
2019   "epoch": 0
2020 }
2021 EOF
2022
2023     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
2024     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
2025     if test $getjson = "yes"
2026     then
2027         jq '.' $dir/json > save1.json
2028     fi
2029
2030     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
2031     then
2032       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
2033     fi
2034
2035     objname=ROBJ9
2036     # Change data and size again because digest was recomputed
2037     echo -n ZZZ > $dir/change
2038     rados --pool $poolname put $objname $dir/change
2039     # Set one to an even older value
2040     objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi
2041     rm $dir/oi $dir/change
2042
2043     objname=ROBJ10
2044     objectstore_tool $dir 1 $objname get-attr _ > $dir/oi
2045     rados --pool $poolname setomapval $objname key2-$objname val2-$objname
2046     objectstore_tool $dir 0 $objname set-attr _ $dir/oi
2047     objectstore_tool $dir 1 $objname set-attr _ $dir/oi
2048     rm $dir/oi
2049
2050     inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
2051     inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
2052     inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
2053     inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
2054
2055     # ROBJ19 won't error this time
2056     ceph tell osd.\* injectargs -- --osd-max-object-size=134217728
2057
2058     pg_deep_scrub $pg
2059
2060     err_strings=()
2061     err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:30259878:::ROBJ15:head : candidate had a missing info key"
2062     err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:33aca486:::ROBJ18:head : data_digest 0xbd89c912 != data_digest 0x2ddbf8f5 from auth oi 3:33aca486:::ROBJ18:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 54 dd 2ddbf8f5 od ddc3680f alloc_hint [[]0 0 255[]][)], object info inconsistent "
2063     err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:33aca486:::ROBJ18:head : data_digest 0xbd89c912 != data_digest 0x2ddbf8f5 from auth oi 3:33aca486:::ROBJ18:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 54 dd 2ddbf8f5 od ddc3680f alloc_hint [[]0 0 255[]][)]"
2064     err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:33aca486:::ROBJ18:head : failed to pick suitable auth object"
2065     err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:5c7b2c47:::ROBJ16:head : candidate had a corrupt snapset"
2066     err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:5c7b2c47:::ROBJ16:head : candidate had a missing snapset key"
2067     err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:5c7b2c47:::ROBJ16:head : failed to pick suitable object info"
2068     err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:86586531:::ROBJ8:head : attr value mismatch '_key1-ROBJ8', attr name mismatch '_key3-ROBJ8', attr name mismatch '_key2-ROBJ8'"
2069     err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:87abbf36:::ROBJ11:head : candidate had a read error"
2070     err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:8aa5320e:::ROBJ17:head : data_digest 0x5af0c3ef != data_digest 0x2ddbf8f5 from auth oi 3:8aa5320e:::ROBJ17:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 51 dd 2ddbf8f5 od e9572720 alloc_hint [[]0 0 0[]][)]"
2071     err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:8aa5320e:::ROBJ17:head : data_digest 0x5af0c3ef != data_digest 0x2ddbf8f5 from auth oi 3:8aa5320e:::ROBJ17:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 51 dd 2ddbf8f5 od e9572720 alloc_hint [[]0 0 0[]][)]"
2072     err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:8aa5320e:::ROBJ17:head : failed to pick suitable auth object"
2073     err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:8b55fa4b:::ROBJ7:head : omap_digest 0xefced57a != omap_digest 0x6a73cc07 from shard 1"
2074     err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:8b55fa4b:::ROBJ7:head : omap_digest 0x6a73cc07 != omap_digest 0xefced57a from auth oi 3:8b55fa4b:::ROBJ7:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [[]0 0 0[]][)]"
2075     err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:a53c12e8:::ROBJ6:head : omap_digest 0x689ee887 != omap_digest 0x179c919f from shard 1, omap_digest 0x689ee887 != omap_digest 0x179c919f from auth oi 3:a53c12e8:::ROBJ6:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [[]0 0 0[]][)]"
2076     err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:b1f19cbd:::ROBJ10:head : omap_digest 0xa8dd5adc != omap_digest 0xc2025a24 from auth oi 3:b1f19cbd:::ROBJ10:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [[]0 0 0[]][)]"
2077     err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:b1f19cbd:::ROBJ10:head : omap_digest 0xa8dd5adc != omap_digest 0xc2025a24 from auth oi 3:b1f19cbd:::ROBJ10:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [[]0 0 0[]][)]"
2078     err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:b1f19cbd:::ROBJ10:head : failed to pick suitable auth object"
2079     err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:bc819597:::ROBJ12:head : candidate had a stat error"
2080     err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:c0c86b1d:::ROBJ14:head : candidate had a missing info key"
2081     err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:c0c86b1d:::ROBJ14:head : candidate had a corrupt info"
2082     err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:c0c86b1d:::ROBJ14:head : failed to pick suitable object info"
2083     err_strings[22]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : candidate size 9 info size 7 mismatch"
2084     err_strings[23]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : data_digest 0x2d4a11c2 != data_digest 0x2ddbf8f5 from shard 0, data_digest 0x2d4a11c2 != data_digest 0x2ddbf8f5 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from shard 0"
2085     err_strings[24]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:d60617f9:::ROBJ13:head : candidate had a read error"
2086     err_strings[25]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:d60617f9:::ROBJ13:head : candidate had a stat error"
2087     err_strings[26]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:d60617f9:::ROBJ13:head : failed to pick suitable object info"
2088     err_strings[27]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:e97ce31e:::ROBJ2:head : data_digest 0x578a4830 != data_digest 0x2ddbf8f5 from shard 1, data_digest 0x578a4830 != data_digest 0x2ddbf8f5 from auth oi 3:e97ce31e:::ROBJ2:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [[]0 0 0[]][)]"
2089     err_strings[28]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 3:f2a5b2a4:::ROBJ3:head : missing"
2090     err_strings[29]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:f4981d31:::ROBJ4:head : omap_digest 0xd7178dfe != omap_digest 0xe2d46ea4 from shard 1, omap_digest 0xd7178dfe != omap_digest 0xe2d46ea4 from auth oi 3:f4981d31:::ROBJ4:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [[]0 0 0[]][)]"
2091     err_strings[30]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:f4bfd4d1:::ROBJ5:head : omap_digest 0x1a862a41 != omap_digest 0x6cac8f6 from shard 1"
2092     err_strings[31]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:f4bfd4d1:::ROBJ5:head : omap_digest 0x6cac8f6 != omap_digest 0x1a862a41 from auth oi 3:f4bfd4d1:::ROBJ5:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [[]0 0 0[]][)]"
2093     err_strings[32]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:ffdb2004:::ROBJ9:head : candidate size 3 info size 7 mismatch"
2094     err_strings[33]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:ffdb2004:::ROBJ9:head : object info inconsistent "
2095     err_strings[34]="log_channel[(]cluster[)] log [[]ERR[]] : deep-scrub [0-9]*[.]0 3:c0c86b1d:::ROBJ14:head : no '_' attr"
2096     err_strings[35]="log_channel[(]cluster[)] log [[]ERR[]] : deep-scrub [0-9]*[.]0 3:5c7b2c47:::ROBJ16:head : can't decode 'snapset' attr buffer::malformed_input: .* no longer understand old encoding version 3 < 97"
2097     err_strings[36]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub : stat mismatch, got 19/19 objects, 0/0 clones, 18/19 dirty, 18/19 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 1049715/1049716 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
2098     err_strings[37]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub 1 missing, 11 inconsistent objects"
2099     err_strings[38]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub 35 errors"
2100
2101     for err_string in "${err_strings[@]}"
2102     do
2103         if ! grep -q "$err_string" $dir/osd.${primary}.log
2104         then
2105             echo "Missing log message '$err_string'"
2106             ERRORS=$(expr $ERRORS + 1)
2107         fi
2108     done
2109
2110     rados list-inconsistent-pg $poolname > $dir/json || return 1
2111     # Check pg count
2112     test $(jq '. | length' $dir/json) = "1" || return 1
2113     # Check pgid
2114     test $(jq -r '.[0]' $dir/json) = $pg || return 1
2115
2116     rados list-inconsistent-obj $pg > $dir/json || return 1
2117     # Get epoch for repair-get requests
2118     epoch=$(jq .epoch $dir/json)
2119
2120     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
2121 {
2122   "inconsistents": [
2123     {
2124       "shards": [
2125         {
2126           "data_digest": "0x2ddbf8f5",
2127           "omap_digest": "0xf5fba2c6",
2128           "size": 7,
2129           "errors": [],
2130           "osd": 0,
2131           "primary": false
2132         },
2133         {
2134           "object_info": {
2135             "oid": {
2136               "oid": "ROBJ1",
2137               "key": "",
2138               "snapid": -2,
2139               "hash": 1454963827,
2140               "max": 0,
2141               "pool": 3,
2142               "namespace": ""
2143             },
2144             "version": "51'58",
2145             "prior_version": "21'3",
2146             "last_reqid": "osd.1.0:57",
2147             "user_version": 3,
2148             "size": 7,
2149             "mtime": "2018-04-05 14:33:19.804040",
2150             "local_mtime": "2018-04-05 14:33:19.804839",
2151             "lost": 0,
2152             "flags": [
2153               "dirty",
2154               "omap",
2155               "data_digest",
2156               "omap_digest"
2157             ],
2158             "truncate_seq": 0,
2159             "truncate_size": 0,
2160             "data_digest": "0x2ddbf8f5",
2161             "omap_digest": "0xf5fba2c6",
2162             "expected_object_size": 0,
2163             "expected_write_size": 0,
2164             "alloc_hint_flags": 0,
2165             "manifest": {
2166               "type": 0
2167             },
2168             "watchers": {}
2169           },
2170           "data_digest": "0x2d4a11c2",
2171           "omap_digest": "0xf5fba2c6",
2172           "size": 9,
2173           "errors": [
2174             "data_digest_mismatch_info",
2175             "size_mismatch_info",
2176             "obj_size_info_mismatch"
2177           ],
2178           "osd": 1,
2179           "primary": true
2180         }
2181       ],
2182       "selected_object_info": {
2183         "oid": {
2184           "oid": "ROBJ1",
2185           "key": "",
2186           "snapid": -2,
2187           "hash": 1454963827,
2188           "max": 0,
2189           "pool": 3,
2190           "namespace": ""
2191         },
2192         "version": "51'58",
2193         "prior_version": "21'3",
2194         "last_reqid": "osd.1.0:57",
2195         "user_version": 3,
2196         "size": 7,
2197         "mtime": "2018-04-05 14:33:19.804040",
2198         "local_mtime": "2018-04-05 14:33:19.804839",
2199         "lost": 0,
2200         "flags": [
2201           "dirty",
2202           "omap",
2203           "data_digest",
2204           "omap_digest"
2205         ],
2206         "truncate_seq": 0,
2207         "truncate_size": 0,
2208         "data_digest": "0x2ddbf8f5",
2209         "omap_digest": "0xf5fba2c6",
2210         "expected_object_size": 0,
2211         "expected_write_size": 0,
2212         "alloc_hint_flags": 0,
2213         "manifest": {
2214           "type": 0
2215         },
2216         "watchers": {}
2217       },
2218       "union_shard_errors": [
2219         "data_digest_mismatch_info",
2220         "size_mismatch_info",
2221         "obj_size_info_mismatch"
2222       ],
2223       "errors": [
2224         "data_digest_mismatch",
2225         "size_mismatch"
2226       ],
2227       "object": {
2228         "version": 3,
2229         "snap": "head",
2230         "locator": "",
2231         "nspace": "",
2232         "name": "ROBJ1"
2233       }
2234     },
2235     {
2236       "shards": [
2237         {
2238           "data_digest": "0x2ddbf8f5",
2239           "omap_digest": "0xa8dd5adc",
2240           "size": 7,
2241           "errors": [
2242             "omap_digest_mismatch_info"
2243           ],
2244           "osd": 0,
2245           "primary": false
2246         },
2247         {
2248           "data_digest": "0x2ddbf8f5",
2249           "omap_digest": "0xa8dd5adc",
2250           "size": 7,
2251           "errors": [
2252             "omap_digest_mismatch_info"
2253           ],
2254           "osd": 1,
2255           "primary": true
2256         }
2257       ],
2258       "selected_object_info": {
2259         "alloc_hint_flags": 0,
2260         "data_digest": "0x2ddbf8f5",
2261         "expected_object_size": 0,
2262         "expected_write_size": 0,
2263         "flags": [
2264           "dirty",
2265           "omap",
2266           "data_digest",
2267           "omap_digest"
2268         ],
2269         "lost": 0,
2270         "manifest": {
2271           "type": 0
2272         },
2273         "oid": {
2274           "hash": 3174666125,
2275           "key": "",
2276           "max": 0,
2277           "namespace": "",
2278           "oid": "ROBJ10",
2279           "pool": 3,
2280           "snapid": -2
2281         },
2282         "omap_digest": "0xc2025a24",
2283         "size": 7,
2284         "truncate_seq": 0,
2285         "truncate_size": 0,
2286         "user_version": 30,
2287         "watchers": {}
2288       },
2289       "union_shard_errors": [
2290         "omap_digest_mismatch_info"
2291       ],
2292       "errors": [],
2293       "object": {
2294         "version": 30,
2295         "snap": "head",
2296         "locator": "",
2297         "nspace": "",
2298         "name": "ROBJ10"
2299       }
2300     },
2301     {
2302       "shards": [
2303         {
2304           "data_digest": "0x2ddbf8f5",
2305           "omap_digest": "0xa03cef03",
2306           "size": 7,
2307           "errors": [],
2308           "osd": 0,
2309           "primary": false
2310         },
2311         {
2312           "size": 7,
2313           "errors": [
2314             "read_error"
2315           ],
2316           "osd": 1,
2317           "primary": true
2318         }
2319       ],
2320       "selected_object_info": {
2321         "oid": {
2322           "oid": "ROBJ11",
2323           "key": "",
2324           "snapid": -2,
2325           "hash": 1828574689,
2326           "max": 0,
2327           "pool": 3,
2328           "namespace": ""
2329         },
2330         "version": "51'52",
2331         "prior_version": "41'33",
2332         "last_reqid": "osd.1.0:51",
2333         "user_version": 33,
2334         "size": 7,
2335         "mtime": "2018-04-05 14:33:26.761286",
2336         "local_mtime": "2018-04-05 14:33:26.762368",
2337         "lost": 0,
2338         "flags": [
2339           "dirty",
2340           "omap",
2341           "data_digest",
2342           "omap_digest"
2343         ],
2344         "truncate_seq": 0,
2345         "truncate_size": 0,
2346         "data_digest": "0x2ddbf8f5",
2347         "omap_digest": "0xa03cef03",
2348         "expected_object_size": 0,
2349         "expected_write_size": 0,
2350         "alloc_hint_flags": 0,
2351         "manifest": {
2352           "type": 0
2353         },
2354         "watchers": {}
2355       },
2356       "union_shard_errors": [
2357         "read_error"
2358       ],
2359       "errors": [],
2360       "object": {
2361         "version": 33,
2362         "snap": "head",
2363         "locator": "",
2364         "nspace": "",
2365         "name": "ROBJ11"
2366       }
2367     },
2368     {
2369       "shards": [
2370         {
2371           "errors": [
2372             "stat_error"
2373           ],
2374           "osd": 0,
2375           "primary": false
2376         },
2377         {
2378           "data_digest": "0x2ddbf8f5",
2379           "omap_digest": "0x067f306a",
2380           "size": 7,
2381           "errors": [],
2382           "osd": 1,
2383           "primary": true
2384         }
2385       ],
2386       "selected_object_info": {
2387         "oid": {
2388           "oid": "ROBJ12",
2389           "key": "",
2390           "snapid": -2,
2391           "hash": 3920199997,
2392           "max": 0,
2393           "pool": 3,
2394           "namespace": ""
2395         },
2396         "version": "51'56",
2397         "prior_version": "43'36",
2398         "last_reqid": "osd.1.0:55",
2399         "user_version": 36,
2400         "size": 7,
2401         "mtime": "2018-04-05 14:33:27.460958",
2402         "local_mtime": "2018-04-05 14:33:27.462109",
2403         "lost": 0,
2404         "flags": [
2405           "dirty",
2406           "omap",
2407           "data_digest",
2408           "omap_digest"
2409         ],
2410         "truncate_seq": 0,
2411         "truncate_size": 0,
2412         "data_digest": "0x2ddbf8f5",
2413         "omap_digest": "0x067f306a",
2414         "expected_object_size": 0,
2415         "expected_write_size": 0,
2416         "alloc_hint_flags": 0,
2417         "manifest": {
2418           "type": 0
2419         },
2420         "watchers": {}
2421       },
2422       "union_shard_errors": [
2423         "stat_error"
2424       ],
2425       "errors": [],
2426       "object": {
2427         "version": 36,
2428         "snap": "head",
2429         "locator": "",
2430         "nspace": "",
2431         "name": "ROBJ12"
2432       }
2433     },
2434     {
2435       "shards": [
2436         {
2437           "errors": [
2438             "stat_error"
2439           ],
2440           "osd": 0,
2441           "primary": false
2442         },
2443         {
2444           "size": 7,
2445           "errors": [
2446             "read_error"
2447           ],
2448           "osd": 1,
2449           "primary": true
2450         }
2451       ],
2452       "union_shard_errors": [
2453         "stat_error",
2454         "read_error"
2455       ],
2456       "errors": [],
2457       "object": {
2458         "version": 0,
2459         "snap": "head",
2460         "locator": "",
2461         "nspace": "",
2462         "name": "ROBJ13"
2463       }
2464     },
2465     {
2466       "shards": [
2467         {
2468           "object_info": "bad-val",
2469           "data_digest": "0x2ddbf8f5",
2470           "omap_digest": "0x4f14f849",
2471           "size": 7,
2472           "errors": [
2473             "info_corrupted"
2474           ],
2475           "osd": 0,
2476           "primary": false
2477         },
2478         {
2479           "data_digest": "0x2ddbf8f5",
2480           "omap_digest": "0x4f14f849",
2481           "size": 7,
2482           "errors": [
2483             "info_missing"
2484           ],
2485           "osd": 1,
2486           "primary": true
2487         }
2488       ],
2489       "union_shard_errors": [
2490         "info_missing",
2491         "info_corrupted"
2492       ],
2493       "errors": [],
2494       "object": {
2495         "version": 0,
2496         "snap": "head",
2497         "locator": "",
2498         "nspace": "",
2499         "name": "ROBJ14"
2500       }
2501     },
2502     {
2503       "shards": [
2504         {
2505           "object_info": {
2506             "oid": {
2507               "oid": "ROBJ15",
2508               "key": "",
2509               "snapid": -2,
2510               "hash": 504996876,
2511               "max": 0,
2512               "pool": 3,
2513               "namespace": ""
2514             },
2515             "version": "51'49",
2516             "prior_version": "49'45",
2517             "last_reqid": "osd.1.0:48",
2518             "user_version": 45,
2519             "size": 7,
2520             "mtime": "2018-04-05 14:33:29.498969",
2521             "local_mtime": "2018-04-05 14:33:29.499890",
2522             "lost": 0,
2523             "flags": [
2524               "dirty",
2525               "omap",
2526               "data_digest",
2527               "omap_digest"
2528             ],
2529             "truncate_seq": 0,
2530             "truncate_size": 0,
2531             "data_digest": "0x2ddbf8f5",
2532             "omap_digest": "0x2d2a4d6e",
2533             "expected_object_size": 0,
2534             "expected_write_size": 0,
2535             "alloc_hint_flags": 0,
2536             "manifest": {
2537               "type": 0
2538             },
2539             "watchers": {}
2540           },
2541           "data_digest": "0x2ddbf8f5",
2542           "omap_digest": "0x2d2a4d6e",
2543           "size": 7,
2544           "errors": [],
2545           "osd": 0,
2546           "primary": false
2547         },
2548         {
2549           "data_digest": "0x2ddbf8f5",
2550           "omap_digest": "0x2d2a4d6e",
2551           "size": 7,
2552           "errors": [
2553             "info_missing"
2554           ],
2555           "osd": 1,
2556           "primary": true
2557         }
2558       ],
2559       "selected_object_info": {
2560         "oid": {
2561           "oid": "ROBJ15",
2562           "key": "",
2563           "snapid": -2,
2564           "hash": 504996876,
2565           "max": 0,
2566           "pool": 3,
2567           "namespace": ""
2568         },
2569         "version": "51'49",
2570         "prior_version": "49'45",
2571         "last_reqid": "osd.1.0:48",
2572         "user_version": 45,
2573         "size": 7,
2574         "mtime": "2018-04-05 14:33:29.498969",
2575         "local_mtime": "2018-04-05 14:33:29.499890",
2576         "lost": 0,
2577         "flags": [
2578           "dirty",
2579           "omap",
2580           "data_digest",
2581           "omap_digest"
2582         ],
2583         "truncate_seq": 0,
2584         "truncate_size": 0,
2585         "data_digest": "0x2ddbf8f5",
2586         "omap_digest": "0x2d2a4d6e",
2587         "expected_object_size": 0,
2588         "expected_write_size": 0,
2589         "alloc_hint_flags": 0,
2590         "manifest": {
2591           "type": 0
2592         },
2593         "watchers": {}
2594       },
2595       "union_shard_errors": [
2596         "info_missing"
2597       ],
2598       "errors": [],
2599       "object": {
2600         "version": 45,
2601         "snap": "head",
2602         "locator": "",
2603         "nspace": "",
2604         "name": "ROBJ15"
2605       }
2606     },
2607     {
2608       "errors": [],
2609       "object": {
2610       "locator": "",
2611       "name": "ROBJ16",
2612       "nspace": "",
2613       "snap": "head",
2614       "version": 0
2615        },
2616         "shards": [
2617       {
2618         "data_digest": "0x2ddbf8f5",
2619         "errors": [
2620           "snapset_missing"
2621         ],
2622         "omap_digest": "0x8b699207",
2623         "osd": 0,
2624         "primary": false,
2625         "size": 7
2626       },
2627       {
2628         "snapset": "bad-val",
2629         "data_digest": "0x2ddbf8f5",
2630         "errors": [
2631           "snapset_corrupted"
2632         ],
2633         "omap_digest": "0x8b699207",
2634         "osd": 1,
2635         "primary": true,
2636         "size": 7
2637       }
2638       ],
2639       "union_shard_errors": [
2640         "snapset_missing",
2641         "snapset_corrupted"
2642       ]
2643     },
2644     {
2645      "errors": [],
2646      "object": {
2647        "locator": "",
2648        "name": "ROBJ17",
2649        "nspace": "",
2650        "snap": "head"
2651      },
2652      "selected_object_info": {
2653        "alloc_hint_flags": 0,
2654        "data_digest": "0x2ddbf8f5",
2655        "expected_object_size": 0,
2656        "expected_write_size": 0,
2657        "flags": [
2658          "dirty",
2659          "omap",
2660          "data_digest",
2661          "omap_digest"
2662        ],
2663        "lost": 0,
2664        "manifest": {
2665          "type": 0
2666        },
2667        "oid": {
2668          "hash": 1884071249,
2669          "key": "",
2670          "max": 0,
2671          "namespace": "",
2672          "oid": "ROBJ17",
2673          "pool": 3,
2674          "snapid": -2
2675        },
2676        "omap_digest": "0xe9572720",
2677        "size": 7,
2678        "truncate_seq": 0,
2679        "truncate_size": 0,
2680        "user_version": 51,
2681        "watchers": {}
2682      },
2683      "shards": [
2684        {
2685          "data_digest": "0x5af0c3ef",
2686          "errors": [
2687            "data_digest_mismatch_info"
2688          ],
2689          "omap_digest": "0xe9572720",
2690          "osd": 0,
2691          "primary": false,
2692          "size": 7
2693        },
2694        {
2695          "data_digest": "0x5af0c3ef",
2696          "errors": [
2697            "data_digest_mismatch_info"
2698          ],
2699          "omap_digest": "0xe9572720",
2700          "osd": 1,
2701          "primary": true,
2702          "size": 7
2703        }
2704      ],
2705      "union_shard_errors": [
2706        "data_digest_mismatch_info"
2707      ]
2708    },
2709    {
2710      "errors": [
2711        "object_info_inconsistency"
2712      ],
2713      "object": {
2714        "locator": "",
2715        "name": "ROBJ18",
2716        "nspace": "",
2717        "snap": "head"
2718      },
2719      "selected_object_info": {
2720        "alloc_hint_flags": 255,
2721        "data_digest": "0x2ddbf8f5",
2722        "expected_object_size": 0,
2723        "expected_write_size": 0,
2724        "flags": [
2725          "dirty",
2726          "omap",
2727          "data_digest",
2728          "omap_digest"
2729        ],
2730        "lost": 0,
2731        "manifest": {
2732          "type": 0
2733        },
2734        "oid": {
2735          "hash": 1629828556,
2736          "key": "",
2737          "max": 0,
2738          "namespace": "",
2739          "oid": "ROBJ18",
2740          "pool": 3,
2741          "snapid": -2
2742        },
2743        "omap_digest": "0xddc3680f",
2744        "size": 7,
2745        "truncate_seq": 0,
2746        "truncate_size": 0,
2747        "user_version": 54,
2748        "watchers": {}
2749      },
2750      "shards": [
2751        {
2752          "data_digest": "0xbd89c912",
2753          "errors": [
2754            "data_digest_mismatch_info"
2755          ],
2756          "object_info": {
2757            "alloc_hint_flags": 0,
2758            "data_digest": "0x2ddbf8f5",
2759            "expected_object_size": 0,
2760            "expected_write_size": 0,
2761            "flags": [
2762              "dirty",
2763              "omap",
2764              "data_digest",
2765              "omap_digest"
2766            ],
2767            "lost": 0,
2768            "manifest": {
2769              "type": 0
2770            },
2771            "oid": {
2772              "hash": 1629828556,
2773              "key": "",
2774              "max": 0,
2775              "namespace": "",
2776              "oid": "ROBJ18",
2777              "pool": 3,
2778              "snapid": -2
2779            },
2780            "omap_digest": "0xddc3680f",
2781            "size": 7,
2782            "truncate_seq": 0,
2783            "truncate_size": 0,
2784            "user_version": 54,
2785            "watchers": {}
2786          },
2787          "omap_digest": "0xddc3680f",
2788          "osd": 0,
2789          "primary": false,
2790          "size": 7
2791        },
2792        {
2793          "data_digest": "0xbd89c912",
2794          "errors": [
2795            "data_digest_mismatch_info"
2796          ],
2797          "object_info": {
2798            "alloc_hint_flags": 255,
2799            "data_digest": "0x2ddbf8f5",
2800            "expected_object_size": 0,
2801            "expected_write_size": 0,
2802            "flags": [
2803              "dirty",
2804              "omap",
2805              "data_digest",
2806              "omap_digest"
2807            ],
2808            "lost": 0,
2809            "manifest": {
2810              "type": 0
2811            },
2812            "oid": {
2813              "hash": 1629828556,
2814              "key": "",
2815              "max": 0,
2816              "namespace": "",
2817              "oid": "ROBJ18",
2818              "pool": 3,
2819              "snapid": -2
2820            },
2821            "omap_digest": "0xddc3680f",
2822            "size": 7,
2823            "truncate_seq": 0,
2824            "truncate_size": 0,
2825            "user_version": 54,
2826            "watchers": {}
2827          },
2828          "omap_digest": "0xddc3680f",
2829          "osd": 1,
2830          "primary": true,
2831          "size": 7
2832        }
2833      ],
2834      "union_shard_errors": [
2835        "data_digest_mismatch_info"
2836      ]
2837    },
2838    {
2839      "shards": [
2840         {
2841           "data_digest": "0x578a4830",
2842           "omap_digest": "0xf8e11918",
2843           "size": 7,
2844           "errors": [
2845             "data_digest_mismatch_info"
2846           ],
2847           "osd": 0,
2848           "primary": false
2849         },
2850         {
2851           "data_digest": "0x2ddbf8f5",
2852           "omap_digest": "0xf8e11918",
2853           "size": 7,
2854           "errors": [],
2855           "osd": 1,
2856           "primary": true
2857         }
2858       ],
2859       "selected_object_info": {
2860         "oid": {
2861           "oid": "ROBJ2",
2862           "key": "",
2863           "snapid": -2,
2864           "hash": 2026323607,
2865           "max": 0,
2866           "pool": 3,
2867           "namespace": ""
2868         },
2869         "version": "51'60",
2870         "prior_version": "23'6",
2871         "last_reqid": "osd.1.0:59",
2872         "user_version": 6,
2873         "size": 7,
2874         "mtime": "2018-04-05 14:33:20.498756",
2875         "local_mtime": "2018-04-05 14:33:20.499704",
2876         "lost": 0,
2877         "flags": [
2878           "dirty",
2879           "omap",
2880           "data_digest",
2881           "omap_digest"
2882         ],
2883         "truncate_seq": 0,
2884         "truncate_size": 0,
2885         "data_digest": "0x2ddbf8f5",
2886         "omap_digest": "0xf8e11918",
2887         "expected_object_size": 0,
2888         "expected_write_size": 0,
2889         "alloc_hint_flags": 0,
2890         "manifest": {
2891           "type": 0
2892         },
2893         "watchers": {}
2894       },
2895       "union_shard_errors": [
2896         "data_digest_mismatch_info"
2897       ],
2898       "errors": [
2899         "data_digest_mismatch"
2900       ],
2901       "object": {
2902         "version": 6,
2903         "snap": "head",
2904         "locator": "",
2905         "nspace": "",
2906         "name": "ROBJ2"
2907       }
2908     },
2909     {
2910       "shards": [
2911         {
2912           "data_digest": "0x2ddbf8f5",
2913           "omap_digest": "0x00b35dfd",
2914           "size": 7,
2915           "errors": [],
2916           "osd": 0,
2917           "primary": false
2918         },
2919         {
2920           "errors": [
2921             "missing"
2922           ],
2923           "osd": 1,
2924           "primary": true
2925         }
2926       ],
2927       "selected_object_info": {
2928         "oid": {
2929           "oid": "ROBJ3",
2930           "key": "",
2931           "snapid": -2,
2932           "hash": 625845583,
2933           "max": 0,
2934           "pool": 3,
2935           "namespace": ""
2936         },
2937         "version": "51'61",
2938         "prior_version": "25'9",
2939         "last_reqid": "osd.1.0:60",
2940         "user_version": 9,
2941         "size": 7,
2942         "mtime": "2018-04-05 14:33:21.189382",
2943         "local_mtime": "2018-04-05 14:33:21.190446",
2944         "lost": 0,
2945         "flags": [
2946           "dirty",
2947           "omap",
2948           "data_digest",
2949           "omap_digest"
2950         ],
2951         "truncate_seq": 0,
2952         "truncate_size": 0,
2953         "data_digest": "0x2ddbf8f5",
2954         "omap_digest": "0x00b35dfd",
2955         "expected_object_size": 0,
2956         "expected_write_size": 0,
2957         "alloc_hint_flags": 0,
2958         "manifest": {
2959           "type": 0
2960         },
2961         "watchers": {}
2962       },
2963       "union_shard_errors": [
2964         "missing"
2965       ],
2966       "errors": [],
2967       "object": {
2968         "version": 9,
2969         "snap": "head",
2970         "locator": "",
2971         "nspace": "",
2972         "name": "ROBJ3"
2973       }
2974     },
2975     {
2976       "shards": [
2977         {
2978           "data_digest": "0x2ddbf8f5",
2979           "omap_digest": "0xd7178dfe",
2980           "size": 7,
2981           "errors": [
2982             "omap_digest_mismatch_info"
2983           ],
2984           "osd": 0,
2985           "primary": false
2986         },
2987         {
2988           "data_digest": "0x2ddbf8f5",
2989           "omap_digest": "0xe2d46ea4",
2990           "size": 7,
2991           "errors": [],
2992           "osd": 1,
2993           "primary": true
2994         }
2995       ],
2996       "selected_object_info": {
2997         "oid": {
2998           "oid": "ROBJ4",
2999           "key": "",
3000           "snapid": -2,
3001           "hash": 2360875311,
3002           "max": 0,
3003           "pool": 3,
3004           "namespace": ""
3005         },
3006         "version": "51'62",
3007         "prior_version": "27'12",
3008         "last_reqid": "osd.1.0:61",
3009         "user_version": 12,
3010         "size": 7,
3011         "mtime": "2018-04-05 14:33:21.862313",
3012         "local_mtime": "2018-04-05 14:33:21.863261",
3013         "lost": 0,
3014         "flags": [
3015           "dirty",
3016           "omap",
3017           "data_digest",
3018           "omap_digest"
3019         ],
3020         "truncate_seq": 0,
3021         "truncate_size": 0,
3022         "data_digest": "0x2ddbf8f5",
3023         "omap_digest": "0xe2d46ea4",
3024         "expected_object_size": 0,
3025         "expected_write_size": 0,
3026         "alloc_hint_flags": 0,
3027         "manifest": {
3028           "type": 0
3029         },
3030         "watchers": {}
3031       },
3032       "union_shard_errors": [
3033         "omap_digest_mismatch_info"
3034       ],
3035       "errors": [
3036         "omap_digest_mismatch"
3037       ],
3038       "object": {
3039         "version": 12,
3040         "snap": "head",
3041         "locator": "",
3042         "nspace": "",
3043         "name": "ROBJ4"
3044       }
3045     },
3046     {
3047       "shards": [
3048         {
3049           "data_digest": "0x2ddbf8f5",
3050           "omap_digest": "0x1a862a41",
3051           "size": 7,
3052           "errors": [],
3053           "osd": 0,
3054           "primary": false
3055         },
3056         {
3057           "data_digest": "0x2ddbf8f5",
3058           "omap_digest": "0x06cac8f6",
3059           "size": 7,
3060           "errors": [
3061             "omap_digest_mismatch_info"
3062           ],
3063           "osd": 1,
3064           "primary": true
3065         }
3066       ],
3067       "selected_object_info": {
3068         "oid": {
3069           "oid": "ROBJ5",
3070           "key": "",
3071           "snapid": -2,
3072           "hash": 2334915887,
3073           "max": 0,
3074           "pool": 3,
3075           "namespace": ""
3076         },
3077         "version": "51'63",
3078         "prior_version": "29'15",
3079         "last_reqid": "osd.1.0:62",
3080         "user_version": 15,
3081         "size": 7,
3082         "mtime": "2018-04-05 14:33:22.589300",
3083         "local_mtime": "2018-04-05 14:33:22.590376",
3084         "lost": 0,
3085         "flags": [
3086           "dirty",
3087           "omap",
3088           "data_digest",
3089           "omap_digest"
3090         ],
3091         "truncate_seq": 0,
3092         "truncate_size": 0,
3093         "data_digest": "0x2ddbf8f5",
3094         "omap_digest": "0x1a862a41",
3095         "expected_object_size": 0,
3096         "expected_write_size": 0,
3097         "alloc_hint_flags": 0,
3098         "manifest": {
3099           "type": 0
3100         },
3101         "watchers": {}
3102       },
3103       "union_shard_errors": [
3104         "omap_digest_mismatch_info"
3105       ],
3106       "errors": [
3107         "omap_digest_mismatch"
3108       ],
3109       "object": {
3110         "version": 15,
3111         "snap": "head",
3112         "locator": "",
3113         "nspace": "",
3114         "name": "ROBJ5"
3115       }
3116     },
3117     {
3118       "shards": [
3119         {
3120           "data_digest": "0x2ddbf8f5",
3121           "omap_digest": "0x689ee887",
3122           "size": 7,
3123           "errors": [
3124             "omap_digest_mismatch_info"
3125           ],
3126           "osd": 0,
3127           "primary": false
3128         },
3129         {
3130           "data_digest": "0x2ddbf8f5",
3131           "omap_digest": "0x179c919f",
3132           "size": 7,
3133           "errors": [],
3134           "osd": 1,
3135           "primary": true
3136         }
3137       ],
3138       "selected_object_info": {
3139         "oid": {
3140           "oid": "ROBJ6",
3141           "key": "",
3142           "snapid": -2,
3143           "hash": 390610085,
3144           "max": 0,
3145           "pool": 3,
3146           "namespace": ""
3147         },
3148         "version": "51'54",
3149         "prior_version": "31'18",
3150         "last_reqid": "osd.1.0:53",
3151         "user_version": 18,
3152         "size": 7,
3153         "mtime": "2018-04-05 14:33:23.289188",
3154         "local_mtime": "2018-04-05 14:33:23.290130",
3155         "lost": 0,
3156         "flags": [
3157           "dirty",
3158           "omap",
3159           "data_digest",
3160           "omap_digest"
3161         ],
3162         "truncate_seq": 0,
3163         "truncate_size": 0,
3164         "data_digest": "0x2ddbf8f5",
3165         "omap_digest": "0x179c919f",
3166         "expected_object_size": 0,
3167         "expected_write_size": 0,
3168         "alloc_hint_flags": 0,
3169         "manifest": {
3170           "type": 0
3171         },
3172         "watchers": {}
3173       },
3174       "union_shard_errors": [
3175         "omap_digest_mismatch_info"
3176       ],
3177       "errors": [
3178         "omap_digest_mismatch"
3179       ],
3180       "object": {
3181         "version": 18,
3182         "snap": "head",
3183         "locator": "",
3184         "nspace": "",
3185         "name": "ROBJ6"
3186       }
3187     },
3188     {
3189       "shards": [
3190         {
3191           "data_digest": "0x2ddbf8f5",
3192           "omap_digest": "0xefced57a",
3193           "size": 7,
3194           "errors": [],
3195           "osd": 0,
3196           "primary": false
3197         },
3198         {
3199           "data_digest": "0x2ddbf8f5",
3200           "omap_digest": "0x6a73cc07",
3201           "size": 7,
3202           "errors": [
3203             "omap_digest_mismatch_info"
3204           ],
3205           "osd": 1,
3206           "primary": true
3207         }
3208       ],
3209       "selected_object_info": {
3210         "oid": {
3211           "oid": "ROBJ7",
3212           "key": "",
3213           "snapid": -2,
3214           "hash": 3529485009,
3215           "max": 0,
3216           "pool": 3,
3217           "namespace": ""
3218         },
3219         "version": "51'53",
3220         "prior_version": "33'21",
3221         "last_reqid": "osd.1.0:52",
3222         "user_version": 21,
3223         "size": 7,
3224         "mtime": "2018-04-05 14:33:23.979658",
3225         "local_mtime": "2018-04-05 14:33:23.980731",
3226         "lost": 0,
3227         "flags": [
3228           "dirty",
3229           "omap",
3230           "data_digest",
3231           "omap_digest"
3232         ],
3233         "truncate_seq": 0,
3234         "truncate_size": 0,
3235         "data_digest": "0x2ddbf8f5",
3236         "omap_digest": "0xefced57a",
3237         "expected_object_size": 0,
3238         "expected_write_size": 0,
3239         "alloc_hint_flags": 0,
3240         "manifest": {
3241           "type": 0
3242         },
3243         "watchers": {}
3244       },
3245       "union_shard_errors": [
3246         "omap_digest_mismatch_info"
3247       ],
3248       "errors": [
3249         "omap_digest_mismatch"
3250       ],
3251       "object": {
3252         "version": 21,
3253         "snap": "head",
3254         "locator": "",
3255         "nspace": "",
3256         "name": "ROBJ7"
3257       }
3258     },
3259     {
3260       "shards": [
3261         {
3262           "attrs": [
3263             {
3264               "Base64": false,
3265               "value": "bad-val",
3266               "name": "key1-ROBJ8"
3267             },
3268             {
3269               "Base64": false,
3270               "value": "val2-ROBJ8",
3271               "name": "key2-ROBJ8"
3272             }
3273           ],
3274           "data_digest": "0x2ddbf8f5",
3275           "omap_digest": "0xd6be81dc",
3276           "size": 7,
3277           "errors": [],
3278           "osd": 0,
3279           "primary": false
3280         },
3281         {
3282           "attrs": [
3283             {
3284               "Base64": false,
3285               "value": "val1-ROBJ8",
3286               "name": "key1-ROBJ8"
3287             },
3288             {
3289               "Base64": false,
3290               "value": "val3-ROBJ8",
3291               "name": "key3-ROBJ8"
3292             }
3293           ],
3294           "data_digest": "0x2ddbf8f5",
3295           "omap_digest": "0xd6be81dc",
3296           "size": 7,
3297           "errors": [],
3298           "osd": 1,
3299           "primary": true
3300         }
3301       ],
3302       "selected_object_info": {
3303         "oid": {
3304           "oid": "ROBJ8",
3305           "key": "",
3306           "snapid": -2,
3307           "hash": 2359695969,
3308           "max": 0,
3309           "pool": 3,
3310           "namespace": ""
3311         },
3312         "version": "79'66",
3313         "prior_version": "79'65",
3314         "last_reqid": "client.4554.0:1",
3315         "user_version": 79,
3316         "size": 7,
3317         "mtime": "2018-04-05 14:34:05.598688",
3318         "local_mtime": "2018-04-05 14:34:05.599698",
3319         "lost": 0,
3320         "flags": [
3321           "dirty",
3322           "omap",
3323           "data_digest",
3324           "omap_digest"
3325         ],
3326         "truncate_seq": 0,
3327         "truncate_size": 0,
3328         "data_digest": "0x2ddbf8f5",
3329         "omap_digest": "0xd6be81dc",
3330         "expected_object_size": 0,
3331         "expected_write_size": 0,
3332         "alloc_hint_flags": 0,
3333         "manifest": {
3334           "type": 0
3335         },
3336         "watchers": {}
3337       },
3338       "union_shard_errors": [],
3339       "errors": [
3340         "attr_value_mismatch",
3341         "attr_name_mismatch"
3342       ],
3343       "object": {
3344         "version": 66,
3345         "snap": "head",
3346         "locator": "",
3347         "nspace": "",
3348         "name": "ROBJ8"
3349       }
3350     },
3351     {
3352       "shards": [
3353         {
3354           "object_info": {
3355             "oid": {
3356               "oid": "ROBJ9",
3357               "key": "",
3358               "snapid": -2,
3359               "hash": 537189375,
3360               "max": 0,
3361               "pool": 3,
3362               "namespace": ""
3363             },
3364             "version": "51'64",
3365             "prior_version": "37'27",
3366             "last_reqid": "osd.1.0:63",
3367             "user_version": 27,
3368             "size": 7,
3369             "mtime": "2018-04-05 14:33:25.352485",
3370             "local_mtime": "2018-04-05 14:33:25.353746",
3371             "lost": 0,
3372             "flags": [
3373               "dirty",
3374               "omap",
3375               "data_digest",
3376               "omap_digest"
3377             ],
3378             "truncate_seq": 0,
3379             "truncate_size": 0,
3380             "data_digest": "0x2ddbf8f5",
3381             "omap_digest": "0x2eecc539",
3382             "expected_object_size": 0,
3383             "expected_write_size": 0,
3384             "alloc_hint_flags": 0,
3385             "manifest": {
3386               "type": 0
3387             },
3388             "watchers": {}
3389           },
3390           "data_digest": "0x1f26fb26",
3391           "omap_digest": "0x2eecc539",
3392           "size": 3,
3393           "errors": [
3394             "obj_size_info_mismatch"
3395           ],
3396           "osd": 0,
3397           "primary": false
3398         },
3399         {
3400           "object_info": {
3401             "oid": {
3402               "oid": "ROBJ9",
3403               "key": "",
3404               "snapid": -2,
3405               "hash": 537189375,
3406               "max": 0,
3407               "pool": 3,
3408               "namespace": ""
3409             },
3410             "version": "119'68",
3411             "prior_version": "51'64",
3412             "last_reqid": "client.4834.0:1",
3413             "user_version": 81,
3414             "size": 3,
3415             "mtime": "2018-04-05 14:35:01.500659",
3416             "local_mtime": "2018-04-05 14:35:01.502117",
3417             "lost": 0,
3418             "flags": [
3419               "dirty",
3420               "omap",
3421               "data_digest",
3422               "omap_digest"
3423             ],
3424             "truncate_seq": 0,
3425             "truncate_size": 0,
3426             "data_digest": "0x1f26fb26",
3427             "omap_digest": "0x2eecc539",
3428             "expected_object_size": 0,
3429             "expected_write_size": 0,
3430             "alloc_hint_flags": 0,
3431             "manifest": {
3432               "type": 0
3433             },
3434             "watchers": {}
3435           },
3436           "data_digest": "0x1f26fb26",
3437           "omap_digest": "0x2eecc539",
3438           "size": 3,
3439           "errors": [],
3440           "osd": 1,
3441           "primary": true
3442         }
3443       ],
3444       "selected_object_info": {
3445         "oid": {
3446           "oid": "ROBJ9",
3447           "key": "",
3448           "snapid": -2,
3449           "hash": 537189375,
3450           "max": 0,
3451           "pool": 3,
3452           "namespace": ""
3453         },
3454         "version": "119'68",
3455         "prior_version": "51'64",
3456         "last_reqid": "client.4834.0:1",
3457         "user_version": 81,
3458         "size": 3,
3459         "mtime": "2018-04-05 14:35:01.500659",
3460         "local_mtime": "2018-04-05 14:35:01.502117",
3461         "lost": 0,
3462         "flags": [
3463           "dirty",
3464           "omap",
3465           "data_digest",
3466           "omap_digest"
3467         ],
3468         "truncate_seq": 0,
3469         "truncate_size": 0,
3470         "data_digest": "0x1f26fb26",
3471         "omap_digest": "0x2eecc539",
3472         "expected_object_size": 0,
3473         "expected_write_size": 0,
3474         "alloc_hint_flags": 0,
3475         "manifest": {
3476           "type": 0
3477         },
3478         "watchers": {}
3479       },
3480       "union_shard_errors": [
3481         "obj_size_info_mismatch"
3482       ],
3483       "errors": [
3484         "object_info_inconsistency"
3485       ],
3486       "object": {
3487         "version": 68,
3488         "snap": "head",
3489         "locator": "",
3490         "nspace": "",
3491         "name": "ROBJ9"
3492       }
3493     }
3494   ],
3495   "epoch": 0
3496 }
3497 EOF
3498
3499     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
3500     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
3501     if test $getjson = "yes"
3502     then
3503         jq '.' $dir/json > save2.json
3504     fi
3505
3506     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
3507     then
3508       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
3509     fi
3510
3511     repair $pg
3512     wait_for_clean
3513
3514     # This hangs if the repair doesn't work
3515     timeout 30 rados -p $poolname get ROBJ17 $dir/robj17.out || return 1
3516     timeout 30 rados -p $poolname get ROBJ18 $dir/robj18.out || return 1
3517     # Even though we couldn't repair all of the introduced errors, we can fix ROBJ17
3518     diff -q $dir/new.ROBJ17 $dir/robj17.out || return 1
3519     rm -f $dir/new.ROBJ17 $dir/robj17.out || return 1
3520     diff -q $dir/new.ROBJ18 $dir/robj18.out || return 1
3521     rm -f $dir/new.ROBJ18 $dir/robj18.out || return 1
3522
3523     if [ $ERRORS != "0" ];
3524     then
3525         echo "TEST FAILED WITH $ERRORS ERRORS"
3526         return 1
3527     fi
3528
3529     ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it
3530     teardown $dir || return 1
3531 }
3532
3533
3534 #
3535 # Test scrub errors for an erasure coded pool
3536 #
3537 function corrupt_scrub_erasure() {
3538     local dir=$1
3539     local allow_overwrites=$2
3540     local poolname=ecpool
3541     local total_objs=7
3542
3543     setup $dir || return 1
3544     run_mon $dir a || return 1
3545     run_mgr $dir x || return 1
3546     for id in $(seq 0 2) ; do
3547         if [ "$allow_overwrites" = "true" ]; then
3548             run_osd $dir $id || return 1
3549         else
3550             run_osd_filestore $dir $id || return 1
3551         fi
3552     done
3553     create_rbd_pool || return 1
3554     create_pool foo 1
3555
3556     create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1
3557     wait_for_clean || return 1
3558
3559     for i in $(seq 1 $total_objs) ; do
3560         objname=EOBJ${i}
3561         add_something $dir $poolname $objname || return 1
3562
3563         local osd=$(expr $i % 2)
3564
3565         case $i in
3566         1)
3567             # Size (deep scrub data_digest too)
3568             local payload=UVWXYZZZ
3569             echo $payload > $dir/CORRUPT
3570             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
3571             ;;
3572
3573         2)
3574             # Corrupt EC shard
3575             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1
3576             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
3577             ;;
3578
3579         3)
3580              # missing
3581              objectstore_tool $dir $osd $objname remove || return 1
3582              ;;
3583
3584         4)
3585             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
3586             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
3587
3588             # Break xattrs
3589             echo -n bad-val > $dir/bad-val
3590             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
3591             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
3592             echo -n val3-$objname > $dir/newval
3593             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
3594             rm $dir/bad-val $dir/newval
3595             ;;
3596
3597         5)
3598             # Corrupt EC shard
3599             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2
3600             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
3601             ;;
3602
3603         6)
3604             objectstore_tool $dir 0 $objname rm-attr hinfo_key || return 1
3605             echo -n bad-val > $dir/bad-val
3606             objectstore_tool $dir 1 $objname set-attr hinfo_key $dir/bad-val || return 1
3607             ;;
3608
3609         7)
3610             local payload=MAKETHISDIFFERENTFROMOTHEROBJECTS
3611             echo $payload > $dir/DIFFERENT
3612             rados --pool $poolname put $objname $dir/DIFFERENT || return 1
3613
3614             # Get hinfo_key from EOBJ1
3615             objectstore_tool $dir 0 EOBJ1 get-attr hinfo_key > $dir/hinfo
3616             objectstore_tool $dir 0 $objname set-attr hinfo_key $dir/hinfo || return 1
3617             rm -f $dir/hinfo
3618             ;;
3619
3620         esac
3621     done
3622
3623     local pg=$(get_pg $poolname EOBJ0)
3624
3625     pg_scrub $pg
3626
3627     rados list-inconsistent-pg $poolname > $dir/json || return 1
3628     # Check pg count
3629     test $(jq '. | length' $dir/json) = "1" || return 1
3630     # Check pgid
3631     test $(jq -r '.[0]' $dir/json) = $pg || return 1
3632
3633     rados list-inconsistent-obj $pg > $dir/json || return 1
3634     # Get epoch for repair-get requests
3635     epoch=$(jq .epoch $dir/json)
3636
3637     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
3638 {
3639   "inconsistents": [
3640     {
3641       "shards": [
3642         {
3643           "size": 2048,
3644           "errors": [],
3645           "shard": 2,
3646           "osd": 0,
3647           "primary": false
3648         },
3649         {
3650           "object_info": {
3651             "oid": {
3652               "oid": "EOBJ1",
3653               "key": "",
3654               "snapid": -2,
3655               "hash": 560836233,
3656               "max": 0,
3657               "pool": 3,
3658               "namespace": ""
3659             },
3660             "version": "27'1",
3661             "prior_version": "0'0",
3662             "last_reqid": "client.4184.0:1",
3663             "user_version": 1,
3664             "size": 7,
3665             "mtime": "",
3666             "local_mtime": "",
3667             "lost": 0,
3668             "flags": [
3669               "dirty",
3670               "data_digest"
3671             ],
3672             "truncate_seq": 0,
3673             "truncate_size": 0,
3674             "data_digest": "0x2ddbf8f5",
3675             "omap_digest": "0xffffffff",
3676             "expected_object_size": 0,
3677             "expected_write_size": 0,
3678             "alloc_hint_flags": 0,
3679             "manifest": {
3680               "type": 0
3681             },
3682             "watchers": {}
3683           },
3684           "size": 9,
3685           "shard": 0,
3686           "errors": [
3687             "size_mismatch_info",
3688             "obj_size_info_mismatch"
3689           ],
3690           "osd": 1,
3691           "primary": true
3692         },
3693         {
3694           "size": 2048,
3695           "shard": 1,
3696           "errors": [],
3697           "osd": 2,
3698           "primary": false
3699         }
3700       ],
3701       "selected_object_info": {
3702         "oid": {
3703           "oid": "EOBJ1",
3704           "key": "",
3705           "snapid": -2,
3706           "hash": 560836233,
3707           "max": 0,
3708           "pool": 3,
3709           "namespace": ""
3710         },
3711         "version": "27'1",
3712         "prior_version": "0'0",
3713         "last_reqid": "client.4184.0:1",
3714         "user_version": 1,
3715         "size": 7,
3716         "mtime": "",
3717         "local_mtime": "",
3718         "lost": 0,
3719         "flags": [
3720           "dirty",
3721           "data_digest"
3722         ],
3723         "truncate_seq": 0,
3724         "truncate_size": 0,
3725         "data_digest": "0x2ddbf8f5",
3726         "omap_digest": "0xffffffff",
3727         "expected_object_size": 0,
3728         "expected_write_size": 0,
3729         "alloc_hint_flags": 0,
3730         "manifest": {
3731           "type": 0
3732         },
3733         "watchers": {}
3734       },
3735       "union_shard_errors": [
3736         "size_mismatch_info",
3737         "obj_size_info_mismatch"
3738       ],
3739       "errors": [
3740         "size_mismatch"
3741       ],
3742       "object": {
3743         "version": 1,
3744         "snap": "head",
3745         "locator": "",
3746         "nspace": "",
3747         "name": "EOBJ1"
3748       }
3749     },
3750     {
3751       "shards": [
3752         {
3753           "size": 2048,
3754           "errors": [],
3755           "shard": 2,
3756           "osd": 0,
3757           "primary": false
3758         },
3759         {
3760           "shard": 0,
3761           "errors": [
3762             "missing"
3763           ],
3764           "osd": 1,
3765           "primary": true
3766         },
3767         {
3768           "size": 2048,
3769           "shard": 1,
3770           "errors": [],
3771           "osd": 2,
3772           "primary": false
3773         }
3774       ],
3775       "selected_object_info": {
3776         "oid": {
3777           "oid": "EOBJ3",
3778           "key": "",
3779           "snapid": -2,
3780           "hash": 3125668237,
3781           "max": 0,
3782           "pool": 3,
3783           "namespace": ""
3784         },
3785         "version": "39'3",
3786         "prior_version": "0'0",
3787         "last_reqid": "client.4252.0:1",
3788         "user_version": 3,
3789         "size": 7,
3790         "mtime": "",
3791         "local_mtime": "",
3792         "lost": 0,
3793         "flags": [
3794           "dirty",
3795           "data_digest"
3796         ],
3797         "truncate_seq": 0,
3798         "truncate_size": 0,
3799         "data_digest": "0x2ddbf8f5",
3800         "omap_digest": "0xffffffff",
3801         "expected_object_size": 0,
3802         "expected_write_size": 0,
3803         "alloc_hint_flags": 0,
3804         "manifest": {
3805           "type": 0
3806         },
3807         "watchers": {}
3808       },
3809       "union_shard_errors": [
3810         "missing"
3811       ],
3812       "errors": [],
3813       "object": {
3814         "version": 3,
3815         "snap": "head",
3816         "locator": "",
3817         "nspace": "",
3818         "name": "EOBJ3"
3819       }
3820     },
3821     {
3822       "shards": [
3823         {
3824           "attrs": [
3825             {
3826               "Base64": false,
3827               "value": "bad-val",
3828               "name": "key1-EOBJ4"
3829             },
3830             {
3831               "Base64": false,
3832               "value": "val2-EOBJ4",
3833               "name": "key2-EOBJ4"
3834             }
3835           ],
3836           "size": 2048,
3837           "errors": [],
3838           "shard": 2,
3839           "osd": 0,
3840           "primary": false
3841         },
3842         {
3843           "osd": 1,
3844           "primary": true,
3845           "shard": 0,
3846           "errors": [],
3847           "size": 2048,
3848           "attrs": [
3849             {
3850               "Base64": false,
3851               "value": "val1-EOBJ4",
3852               "name": "key1-EOBJ4"
3853             },
3854             {
3855               "Base64": false,
3856               "value": "val2-EOBJ4",
3857               "name": "key2-EOBJ4"
3858             }
3859           ]
3860         },
3861         {
3862           "osd": 2,
3863           "primary": false,
3864           "shard": 1,
3865           "errors": [],
3866           "size": 2048,
3867           "attrs": [
3868             {
3869               "Base64": false,
3870               "value": "val1-EOBJ4",
3871               "name": "key1-EOBJ4"
3872             },
3873             {
3874               "Base64": false,
3875               "value": "val3-EOBJ4",
3876               "name": "key3-EOBJ4"
3877             }
3878           ]
3879         }
3880       ],
3881       "selected_object_info": {
3882         "oid": {
3883           "oid": "EOBJ4",
3884           "key": "",
3885           "snapid": -2,
3886           "hash": 1618759290,
3887           "max": 0,
3888           "pool": 3,
3889           "namespace": ""
3890         },
3891         "version": "45'6",
3892         "prior_version": "45'5",
3893         "last_reqid": "client.4294.0:1",
3894         "user_version": 6,
3895         "size": 7,
3896         "mtime": "",
3897         "local_mtime": "",
3898         "lost": 0,
3899         "flags": [
3900           "dirty",
3901           "data_digest"
3902         ],
3903         "truncate_seq": 0,
3904         "truncate_size": 0,
3905         "data_digest": "0x2ddbf8f5",
3906         "omap_digest": "0xffffffff",
3907         "expected_object_size": 0,
3908         "expected_write_size": 0,
3909         "alloc_hint_flags": 0,
3910         "manifest": {
3911           "type": 0
3912         },
3913         "watchers": {}
3914       },
3915       "union_shard_errors": [],
3916       "errors": [
3917         "attr_value_mismatch",
3918         "attr_name_mismatch"
3919       ],
3920       "object": {
3921         "version": 6,
3922         "snap": "head",
3923         "locator": "",
3924         "nspace": "",
3925         "name": "EOBJ4"
3926       }
3927     },
3928     {
3929       "shards": [
3930         {
3931           "size": 2048,
3932           "errors": [],
3933           "shard": 2,
3934           "osd": 0,
3935           "primary": false
3936         },
3937         {
3938           "object_info": {
3939             "oid": {
3940               "oid": "EOBJ5",
3941               "key": "",
3942               "snapid": -2,
3943               "hash": 2918945441,
3944               "max": 0,
3945               "pool": 3,
3946               "namespace": ""
3947             },
3948             "version": "59'7",
3949             "prior_version": "0'0",
3950             "last_reqid": "client.4382.0:1",
3951             "user_version": 7,
3952             "size": 7,
3953             "mtime": "",
3954             "local_mtime": "",
3955             "lost": 0,
3956             "flags": [
3957               "dirty",
3958               "data_digest"
3959             ],
3960             "truncate_seq": 0,
3961             "truncate_size": 0,
3962             "data_digest": "0x2ddbf8f5",
3963             "omap_digest": "0xffffffff",
3964             "expected_object_size": 0,
3965             "expected_write_size": 0,
3966             "alloc_hint_flags": 0,
3967             "manifest": {
3968               "type": 0
3969             },
3970             "watchers": {}
3971           },
3972           "size": 4096,
3973           "shard": 0,
3974           "errors": [
3975             "size_mismatch_info",
3976             "obj_size_info_mismatch"
3977           ],
3978           "osd": 1,
3979           "primary": true
3980         },
3981         {
3982           "size": 2048,
3983           "shard": 1,
3984           "errors": [],
3985           "osd": 2,
3986           "primary": false
3987         }
3988       ],
3989       "selected_object_info": {
3990         "oid": {
3991           "oid": "EOBJ5",
3992           "key": "",
3993           "snapid": -2,
3994           "hash": 2918945441,
3995           "max": 0,
3996           "pool": 3,
3997           "namespace": ""
3998         },
3999         "version": "59'7",
4000         "prior_version": "0'0",
4001         "last_reqid": "client.4382.0:1",
4002         "user_version": 7,
4003         "size": 7,
4004         "mtime": "",
4005         "local_mtime": "",
4006         "lost": 0,
4007         "flags": [
4008           "dirty",
4009           "data_digest"
4010         ],
4011         "truncate_seq": 0,
4012         "truncate_size": 0,
4013         "data_digest": "0x2ddbf8f5",
4014         "omap_digest": "0xffffffff",
4015         "expected_object_size": 0,
4016         "expected_write_size": 0,
4017         "alloc_hint_flags": 0,
4018         "manifest": {
4019           "type": 0
4020         },
4021         "watchers": {}
4022       },
4023       "union_shard_errors": [
4024         "size_mismatch_info",
4025         "obj_size_info_mismatch"
4026       ],
4027       "errors": [
4028         "size_mismatch"
4029       ],
4030       "object": {
4031         "version": 7,
4032         "snap": "head",
4033         "locator": "",
4034         "nspace": "",
4035         "name": "EOBJ5"
4036       }
4037    },
4038    {
4039      "errors": [],
4040      "object": {
4041        "locator": "",
4042        "name": "EOBJ6",
4043        "nspace": "",
4044        "snap": "head",
4045        "version": 8
4046      },
4047      "selected_object_info": {
4048         "oid": {
4049           "oid": "EOBJ6",
4050           "key": "",
4051           "snapid": -2,
4052           "hash": 3050890866,
4053           "max": 0,
4054           "pool": 3,
4055           "namespace": ""
4056         },
4057         "version": "65'8",
4058         "prior_version": "0'0",
4059         "last_reqid": "client.4418.0:1",
4060         "user_version": 8,
4061         "size": 7,
4062         "mtime": "",
4063         "local_mtime": "",
4064         "lost": 0,
4065         "flags": [
4066           "dirty",
4067           "data_digest"
4068         ],
4069         "truncate_seq": 0,
4070         "truncate_size": 0,
4071         "data_digest": "0x2ddbf8f5",
4072         "omap_digest": "0xffffffff",
4073         "expected_object_size": 0,
4074         "expected_write_size": 0,
4075         "alloc_hint_flags": 0,
4076         "manifest": {
4077           "type": 0
4078         },
4079         "watchers": {}
4080      },
4081      "shards": [
4082        {
4083          "errors": [
4084            "hinfo_missing"
4085          ],
4086          "osd": 0,
4087          "primary": false,
4088          "shard": 2,
4089          "size": 2048
4090        },
4091        {
4092          "errors": [
4093            "hinfo_corrupted"
4094          ],
4095          "osd": 1,
4096          "primary": true,
4097          "shard": 0,
4098          "hashinfo": "bad-val",
4099          "size": 2048
4100        },
4101        {
4102          "errors": [],
4103          "osd": 2,
4104          "primary": false,
4105          "shard": 1,
4106          "size": 2048,
4107          "hashinfo": {
4108            "cumulative_shard_hashes": [
4109             {
4110               "hash": 80717615,
4111               "shard": 0
4112             },
4113             {
4114               "hash": 1534491824,
4115               "shard": 1
4116             },
4117             {
4118               "hash": 80717615,
4119               "shard": 2
4120             }
4121            ],
4122            "total_chunk_size": 2048
4123          }
4124        }
4125      ],
4126      "union_shard_errors": [
4127        "hinfo_missing",
4128        "hinfo_corrupted"
4129      ]
4130    },
4131    {
4132      "errors": [
4133        "hinfo_inconsistency"
4134      ],
4135      "object": {
4136        "locator": "",
4137        "name": "EOBJ7",
4138        "nspace": "",
4139        "snap": "head",
4140        "version": 10
4141      },
4142      "selected_object_info": {
4143         "oid": {
4144           "oid": "EOBJ7",
4145           "key": "",
4146           "snapid": -2,
4147           "hash": 3258066308,
4148           "max": 0,
4149           "pool": 3,
4150           "namespace": ""
4151         },
4152         "version": "75'10",
4153         "prior_version": "75'9",
4154         "last_reqid": "client.4482.0:1",
4155         "user_version": 10,
4156         "size": 34,
4157         "mtime": "",
4158         "local_mtime": "",
4159         "lost": 0,
4160         "flags": [
4161           "dirty",
4162           "data_digest"
4163         ],
4164         "truncate_seq": 0,
4165         "truncate_size": 0,
4166         "data_digest": "0x136e4e27",
4167         "omap_digest": "0xffffffff",
4168         "expected_object_size": 0,
4169         "expected_write_size": 0,
4170         "alloc_hint_flags": 0,
4171         "manifest": {
4172           "type": 0
4173         },
4174         "watchers": {}
4175      },
4176      "shards": [
4177        {
4178          "hashinfo": {
4179            "cumulative_shard_hashes": [
4180             {
4181               "hash": 80717615,
4182               "shard": 0
4183             },
4184             {
4185               "hash": 1534491824,
4186               "shard": 1
4187             },
4188             {
4189               "hash": 80717615,
4190               "shard": 2
4191             }
4192            ],
4193            "total_chunk_size": 2048
4194          },
4195          "errors": [],
4196          "osd": 0,
4197          "primary": false,
4198          "shard": 2,
4199          "size": 2048
4200        },
4201        {
4202          "hashinfo": {
4203            "cumulative_shard_hashes": [
4204             {
4205               "hash": 1534350760,
4206               "shard": 0
4207             },
4208             {
4209               "hash": 1534491824,
4210               "shard": 1
4211             },
4212             {
4213               "hash": 1534350760,
4214               "shard": 2
4215             }
4216            ],
4217            "total_chunk_size": 2048
4218          },
4219          "errors": [],
4220          "osd": 1,
4221          "primary": true,
4222          "shard": 0,
4223          "size": 2048
4224        },
4225        {
4226          "hashinfo": {
4227            "cumulative_shard_hashes": [
4228             {
4229               "hash": 1534350760,
4230               "shard": 0
4231             },
4232             {
4233               "hash": 1534491824,
4234               "shard": 1
4235             },
4236             {
4237               "hash": 1534350760,
4238               "shard": 2
4239             }
4240            ],
4241            "total_chunk_size": 2048
4242          },
4243          "errors": [],
4244          "osd": 2,
4245          "primary": false,
4246          "shard": 1,
4247          "size": 2048
4248        }
4249      ],
4250      "union_shard_errors": []
4251     }
4252   ],
4253   "epoch": 0
4254 }
4255 EOF
4256
4257     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
4258     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
4259     if test $getjson = "yes"
4260     then
4261         jq '.' $dir/json > save3.json
4262     fi
4263
4264     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
4265     then
4266       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
4267     fi
4268
4269     pg_deep_scrub $pg
4270
4271     rados list-inconsistent-pg $poolname > $dir/json || return 1
4272     # Check pg count
4273     test $(jq '. | length' $dir/json) = "1" || return 1
4274     # Check pgid
4275     test $(jq -r '.[0]' $dir/json) = $pg || return 1
4276
4277     rados list-inconsistent-obj $pg > $dir/json || return 1
4278     # Get epoch for repair-get requests
4279     epoch=$(jq .epoch $dir/json)
4280
4281     if [ "$allow_overwrites" = "true" ]
4282     then
4283       jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
4284 {
4285   "inconsistents": [
4286     {
4287       "shards": [
4288         {
4289           "data_digest": "0x00000000",
4290           "omap_digest": "0xffffffff",
4291           "size": 2048,
4292           "errors": [],
4293           "shard": 2,
4294           "osd": 0,
4295           "primary": false
4296         },
4297         {
4298           "object_info": {
4299             "oid": {
4300               "oid": "EOBJ1",
4301               "key": "",
4302               "snapid": -2,
4303               "hash": 560836233,
4304               "max": 0,
4305               "pool": 3,
4306               "namespace": ""
4307             },
4308             "version": "27'1",
4309             "prior_version": "0'0",
4310             "last_reqid": "client.4184.0:1",
4311             "user_version": 1,
4312             "size": 7,
4313             "mtime": "2018-04-05 14:31:33.837147",
4314             "local_mtime": "2018-04-05 14:31:33.840763",
4315             "lost": 0,
4316             "flags": [
4317               "dirty",
4318               "data_digest"
4319             ],
4320             "truncate_seq": 0,
4321             "truncate_size": 0,
4322             "data_digest": "0x2ddbf8f5",
4323             "omap_digest": "0xffffffff",
4324             "expected_object_size": 0,
4325             "expected_write_size": 0,
4326             "alloc_hint_flags": 0,
4327             "manifest": {
4328               "type": 0
4329             },
4330             "watchers": {}
4331           },
4332           "size": 9,
4333           "shard": 0,
4334           "errors": [
4335             "read_error",
4336             "size_mismatch_info",
4337             "obj_size_info_mismatch"
4338           ],
4339           "osd": 1,
4340           "primary": true
4341         },
4342         {
4343           "data_digest": "0x00000000",
4344           "omap_digest": "0xffffffff",
4345           "size": 2048,
4346           "shard": 1,
4347           "errors": [],
4348           "osd": 2,
4349           "primary": false
4350         }
4351       ],
4352       "selected_object_info": {
4353         "oid": {
4354           "oid": "EOBJ1",
4355           "key": "",
4356           "snapid": -2,
4357           "hash": 560836233,
4358           "max": 0,
4359           "pool": 3,
4360           "namespace": ""
4361         },
4362         "version": "27'1",
4363         "prior_version": "0'0",
4364         "last_reqid": "client.4184.0:1",
4365         "user_version": 1,
4366         "size": 7,
4367         "mtime": "2018-04-05 14:31:33.837147",
4368         "local_mtime": "2018-04-05 14:31:33.840763",
4369         "lost": 0,
4370         "flags": [
4371           "dirty",
4372           "data_digest"
4373         ],
4374         "truncate_seq": 0,
4375         "truncate_size": 0,
4376         "data_digest": "0x2ddbf8f5",
4377         "omap_digest": "0xffffffff",
4378         "expected_object_size": 0,
4379         "expected_write_size": 0,
4380         "alloc_hint_flags": 0,
4381         "manifest": {
4382           "type": 0
4383         },
4384         "watchers": {}
4385       },
4386       "union_shard_errors": [
4387         "read_error",
4388         "size_mismatch_info",
4389         "obj_size_info_mismatch"
4390       ],
4391       "errors": [
4392         "size_mismatch"
4393       ],
4394       "object": {
4395         "version": 1,
4396         "snap": "head",
4397         "locator": "",
4398         "nspace": "",
4399         "name": "EOBJ1"
4400       }
4401     },
4402     {
4403       "shards": [
4404         {
4405           "data_digest": "0x00000000",
4406           "omap_digest": "0xffffffff",
4407           "size": 2048,
4408           "errors": [],
4409           "shard": 2,
4410           "osd": 0,
4411           "primary": false
4412         },
4413         {
4414           "shard": 0,
4415           "errors": [
4416             "missing"
4417           ],
4418           "osd": 1,
4419           "primary": true
4420         },
4421         {
4422           "data_digest": "0x00000000",
4423           "omap_digest": "0xffffffff",
4424           "size": 2048,
4425           "shard": 1,
4426           "errors": [],
4427           "osd": 2,
4428           "primary": false
4429         }
4430       ],
4431       "selected_object_info": {
4432         "oid": {
4433           "oid": "EOBJ3",
4434           "key": "",
4435           "snapid": -2,
4436           "hash": 3125668237,
4437           "max": 0,
4438           "pool": 3,
4439           "namespace": ""
4440         },
4441         "version": "39'3",
4442         "prior_version": "0'0",
4443         "last_reqid": "client.4252.0:1",
4444         "user_version": 3,
4445         "size": 7,
4446         "mtime": "2018-04-05 14:31:46.841145",
4447         "local_mtime": "2018-04-05 14:31:46.844996",
4448         "lost": 0,
4449         "flags": [
4450           "dirty",
4451           "data_digest"
4452         ],
4453         "truncate_seq": 0,
4454         "truncate_size": 0,
4455         "data_digest": "0x2ddbf8f5",
4456         "omap_digest": "0xffffffff",
4457         "expected_object_size": 0,
4458         "expected_write_size": 0,
4459         "alloc_hint_flags": 0,
4460         "manifest": {
4461           "type": 0
4462         },
4463         "watchers": {}
4464       },
4465       "union_shard_errors": [
4466         "missing"
4467       ],
4468       "errors": [],
4469       "object": {
4470         "version": 3,
4471         "snap": "head",
4472         "locator": "",
4473         "nspace": "",
4474         "name": "EOBJ3"
4475       }
4476     },
4477     {
4478       "shards": [
4479         {
4480           "attrs": [
4481             {
4482               "Base64": false,
4483               "value": "bad-val",
4484               "name": "key1-EOBJ4"
4485             },
4486             {
4487               "Base64": false,
4488               "value": "val2-EOBJ4",
4489               "name": "key2-EOBJ4"
4490             }
4491           ],
4492           "data_digest": "0x00000000",
4493           "omap_digest": "0xffffffff",
4494           "size": 2048,
4495           "errors": [],
4496           "shard": 2,
4497           "osd": 0,
4498           "primary": false
4499         },
4500         {
4501           "attrs": [
4502             {
4503               "Base64": false,
4504               "value": "val1-EOBJ4",
4505               "name": "key1-EOBJ4"
4506             },
4507             {
4508               "Base64": false,
4509               "value": "val2-EOBJ4",
4510               "name": "key2-EOBJ4"
4511             }
4512           ],
4513           "data_digest": "0x00000000",
4514           "omap_digest": "0xffffffff",
4515           "size": 2048,
4516           "errors": [],
4517           "shard": 0,
4518           "osd": 1,
4519           "primary": true
4520         },
4521         {
4522           "attrs": [
4523             {
4524               "Base64": false,
4525               "value": "val1-EOBJ4",
4526               "name": "key1-EOBJ4"
4527             },
4528             {
4529               "Base64": false,
4530               "value": "val3-EOBJ4",
4531               "name": "key3-EOBJ4"
4532             }
4533           ],
4534           "data_digest": "0x00000000",
4535           "omap_digest": "0xffffffff",
4536           "size": 2048,
4537           "errors": [],
4538           "shard": 1,
4539           "osd": 2,
4540           "primary": false
4541         }
4542       ],
4543       "selected_object_info": {
4544         "oid": {
4545           "oid": "EOBJ4",
4546           "key": "",
4547           "snapid": -2,
4548           "hash": 1618759290,
4549           "max": 0,
4550           "pool": 3,
4551           "namespace": ""
4552         },
4553         "version": "45'6",
4554         "prior_version": "45'5",
4555         "last_reqid": "client.4294.0:1",
4556         "user_version": 6,
4557         "size": 7,
4558         "mtime": "2018-04-05 14:31:54.663622",
4559         "local_mtime": "2018-04-05 14:31:54.664527",
4560         "lost": 0,
4561         "flags": [
4562           "dirty",
4563           "data_digest"
4564         ],
4565         "truncate_seq": 0,
4566         "truncate_size": 0,
4567         "data_digest": "0x2ddbf8f5",
4568         "omap_digest": "0xffffffff",
4569         "expected_object_size": 0,
4570         "expected_write_size": 0,
4571         "alloc_hint_flags": 0,
4572         "manifest": {
4573           "type": 0
4574         },
4575         "watchers": {}
4576       },
4577       "union_shard_errors": [],
4578       "errors": [
4579         "attr_value_mismatch",
4580         "attr_name_mismatch"
4581       ],
4582       "object": {
4583         "version": 6,
4584         "snap": "head",
4585         "locator": "",
4586         "nspace": "",
4587         "name": "EOBJ4"
4588       }
4589     },
4590     {
4591       "shards": [
4592         {
4593           "data_digest": "0x00000000",
4594           "omap_digest": "0xffffffff",
4595           "size": 2048,
4596           "errors": [],
4597           "shard": 2,
4598           "osd": 0,
4599           "primary": false
4600         },
4601         {
4602           "data_digest": "0x00000000",
4603           "omap_digest": "0xffffffff",
4604           "object_info": {
4605             "oid": {
4606               "oid": "EOBJ5",
4607               "key": "",
4608               "snapid": -2,
4609               "hash": 2918945441,
4610               "max": 0,
4611               "pool": 3,
4612               "namespace": ""
4613             },
4614             "version": "59'7",
4615             "prior_version": "0'0",
4616             "last_reqid": "client.4382.0:1",
4617             "user_version": 7,
4618             "size": 7,
4619             "mtime": "2018-04-05 14:32:12.929161",
4620             "local_mtime": "2018-04-05 14:32:12.934707",
4621             "lost": 0,
4622             "flags": [
4623               "dirty",
4624               "data_digest"
4625             ],
4626             "truncate_seq": 0,
4627             "truncate_size": 0,
4628             "data_digest": "0x2ddbf8f5",
4629             "omap_digest": "0xffffffff",
4630             "expected_object_size": 0,
4631             "expected_write_size": 0,
4632             "alloc_hint_flags": 0,
4633             "manifest": {
4634               "type": 0
4635             },
4636             "watchers": {}
4637           },
4638           "size": 4096,
4639           "errors": [
4640             "size_mismatch_info",
4641             "obj_size_info_mismatch"
4642           ],
4643           "shard": 0,
4644           "osd": 1,
4645           "primary": true
4646         },
4647         {
4648           "data_digest": "0x00000000",
4649           "omap_digest": "0xffffffff",
4650           "size": 2048,
4651           "errors": [],
4652           "shard": 1,
4653           "osd": 2,
4654           "primary": false
4655         }
4656       ],
4657       "selected_object_info": {
4658         "oid": {
4659           "oid": "EOBJ5",
4660           "key": "",
4661           "snapid": -2,
4662           "hash": 2918945441,
4663           "max": 0,
4664           "pool": 3,
4665           "namespace": ""
4666         },
4667         "version": "59'7",
4668         "prior_version": "0'0",
4669         "last_reqid": "client.4382.0:1",
4670         "user_version": 7,
4671         "size": 7,
4672         "mtime": "2018-04-05 14:32:12.929161",
4673         "local_mtime": "2018-04-05 14:32:12.934707",
4674         "lost": 0,
4675         "flags": [
4676           "dirty",
4677           "data_digest"
4678         ],
4679         "truncate_seq": 0,
4680         "truncate_size": 0,
4681         "data_digest": "0x2ddbf8f5",
4682         "omap_digest": "0xffffffff",
4683         "expected_object_size": 0,
4684         "expected_write_size": 0,
4685         "alloc_hint_flags": 0,
4686         "manifest": {
4687           "type": 0
4688         },
4689         "watchers": {}
4690       },
4691       "union_shard_errors": [
4692         "size_mismatch_info",
4693         "obj_size_info_mismatch"
4694       ],
4695       "errors": [
4696         "size_mismatch"
4697       ],
4698       "object": {
4699         "version": 7,
4700         "snap": "head",
4701         "locator": "",
4702         "nspace": "",
4703         "name": "EOBJ5"
4704       }
4705     },
4706     {
4707       "object": {
4708         "name": "EOBJ6",
4709         "nspace": "",
4710         "locator": "",
4711         "snap": "head",
4712         "version": 8
4713       },
4714       "errors": [],
4715       "union_shard_errors": [
4716         "read_error",
4717         "hinfo_missing",
4718         "hinfo_corrupted"
4719       ],
4720       "selected_object_info": {
4721         "oid": {
4722           "oid": "EOBJ6",
4723           "key": "",
4724           "snapid": -2,
4725           "hash": 3050890866,
4726           "max": 0,
4727           "pool": 3,
4728           "namespace": ""
4729         },
4730         "version": "65'8",
4731         "prior_version": "0'0",
4732         "last_reqid": "client.4418.0:1",
4733         "user_version": 8,
4734         "size": 7,
4735         "mtime": "2018-04-05 14:32:20.634116",
4736         "local_mtime": "2018-04-05 14:32:20.637999",
4737         "lost": 0,
4738         "flags": [
4739           "dirty",
4740           "data_digest"
4741         ],
4742         "truncate_seq": 0,
4743         "truncate_size": 0,
4744         "data_digest": "0x2ddbf8f5",
4745         "omap_digest": "0xffffffff",
4746         "expected_object_size": 0,
4747         "expected_write_size": 0,
4748         "alloc_hint_flags": 0,
4749         "manifest": {
4750           "type": 0
4751         },
4752         "watchers": {}
4753       },
4754       "shards": [
4755         {
4756           "osd": 0,
4757           "primary": false,
4758           "shard": 2,
4759           "errors": [
4760             "read_error",
4761             "hinfo_missing"
4762           ],
4763           "size": 2048
4764         },
4765         {
4766           "osd": 1,
4767           "primary": true,
4768           "shard": 0,
4769           "errors": [
4770             "read_error",
4771             "hinfo_corrupted"
4772           ],
4773           "size": 2048,
4774           "hashinfo": "bad-val"
4775         },
4776         {
4777           "osd": 2,
4778           "primary": false,
4779           "shard": 1,
4780           "errors": [],
4781           "size": 2048,
4782           "omap_digest": "0xffffffff",
4783           "data_digest": "0x00000000",
4784           "hashinfo": {
4785             "cumulative_shard_hashes": [
4786             {
4787               "hash": 80717615,
4788               "shard": 0
4789             },
4790             {
4791               "hash": 1534491824,
4792               "shard": 1
4793             },
4794             {
4795               "hash": 80717615,
4796               "shard": 2
4797             }
4798            ],
4799            "total_chunk_size": 2048
4800          }
4801         }
4802       ]
4803     },
4804     {
4805       "object": {
4806         "name": "EOBJ7",
4807         "nspace": "",
4808         "locator": "",
4809         "snap": "head",
4810         "version": 10
4811       },
4812       "errors": [
4813         "hinfo_inconsistency"
4814       ],
4815       "union_shard_errors": [],
4816       "selected_object_info": {
4817         "oid": {
4818           "oid": "EOBJ7",
4819           "key": "",
4820           "snapid": -2,
4821           "hash": 3258066308,
4822           "max": 0,
4823           "pool": 3,
4824           "namespace": ""
4825         },
4826         "version": "75'10",
4827         "prior_version": "75'9",
4828         "last_reqid": "client.4482.0:1",
4829         "user_version": 10,
4830         "size": 34,
4831         "mtime": "2018-04-05 14:32:33.058782",
4832         "local_mtime": "2018-04-05 14:32:33.059679",
4833         "lost": 0,
4834         "flags": [
4835           "dirty",
4836           "data_digest"
4837         ],
4838         "truncate_seq": 0,
4839         "truncate_size": 0,
4840         "data_digest": "0x136e4e27",
4841         "omap_digest": "0xffffffff",
4842         "expected_object_size": 0,
4843         "expected_write_size": 0,
4844         "alloc_hint_flags": 0,
4845         "manifest": {
4846           "type": 0
4847         },
4848         "watchers": {}
4849       },
4850       "shards": [
4851         {
4852           "osd": 0,
4853           "primary": false,
4854           "shard": 2,
4855           "errors": [],
4856           "size": 2048,
4857           "omap_digest": "0xffffffff",
4858           "data_digest": "0x00000000",
4859           "hashinfo": {
4860            "cumulative_shard_hashes": [
4861             {
4862               "hash": 80717615,
4863               "shard": 0
4864             },
4865             {
4866               "hash": 1534491824,
4867               "shard": 1
4868             },
4869             {
4870               "hash": 80717615,
4871               "shard": 2
4872             }
4873            ],
4874            "total_chunk_size": 2048
4875           }
4876         },
4877         {
4878           "osd": 1,
4879           "primary": true,
4880           "shard": 0,
4881           "errors": [],
4882           "size": 2048,
4883           "omap_digest": "0xffffffff",
4884           "data_digest": "0x00000000",
4885           "hashinfo": {
4886            "cumulative_shard_hashes": [
4887             {
4888               "hash": 1534350760,
4889               "shard": 0
4890             },
4891             {
4892               "hash": 1534491824,
4893               "shard": 1
4894             },
4895             {
4896               "hash": 1534350760,
4897               "shard": 2
4898             }
4899            ],
4900            "total_chunk_size": 2048
4901           }
4902         },
4903         {
4904           "osd": 2,
4905           "primary": false,
4906           "shard": 1,
4907           "errors": [],
4908           "size": 2048,
4909           "omap_digest": "0xffffffff",
4910           "data_digest": "0x00000000",
4911           "hashinfo": {
4912            "cumulative_shard_hashes": [
4913             {
4914               "hash": 1534350760,
4915               "shard": 0
4916             },
4917             {
4918               "hash": 1534491824,
4919               "shard": 1
4920             },
4921             {
4922               "hash": 1534350760,
4923               "shard": 2
4924             }
4925            ],
4926            "total_chunk_size": 2048
4927           }
4928         }
4929       ]
4930     }
4931   ],
4932   "epoch": 0
4933 }
4934 EOF
4935
4936     else
4937
4938       jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
4939 {
4940   "inconsistents": [
4941     {
4942       "shards": [
4943         {
4944           "data_digest": "0x04cfa72f",
4945           "omap_digest": "0xffffffff",
4946           "size": 2048,
4947           "errors": [],
4948           "shard": 2,
4949           "osd": 0,
4950           "primary": false
4951         },
4952         {
4953           "object_info": {
4954             "oid": {
4955               "oid": "EOBJ1",
4956               "key": "",
4957               "snapid": -2,
4958               "hash": 560836233,
4959               "max": 0,
4960               "pool": 3,
4961               "namespace": ""
4962             },
4963             "version": "27'1",
4964             "prior_version": "0'0",
4965             "last_reqid": "client.4192.0:1",
4966             "user_version": 1,
4967             "size": 7,
4968             "mtime": "2018-04-05 14:30:10.688009",
4969             "local_mtime": "2018-04-05 14:30:10.691774",
4970             "lost": 0,
4971             "flags": [
4972               "dirty",
4973               "data_digest"
4974             ],
4975             "truncate_seq": 0,
4976             "truncate_size": 0,
4977             "data_digest": "0x2ddbf8f5",
4978             "omap_digest": "0xffffffff",
4979             "expected_object_size": 0,
4980             "expected_write_size": 0,
4981             "alloc_hint_flags": 0,
4982             "manifest": {
4983               "type": 0
4984             },
4985             "watchers": {}
4986           },
4987           "size": 9,
4988           "shard": 0,
4989           "errors": [
4990             "read_error",
4991             "size_mismatch_info",
4992             "obj_size_info_mismatch"
4993           ],
4994           "osd": 1,
4995           "primary": true
4996         },
4997         {
4998           "data_digest": "0x04cfa72f",
4999           "omap_digest": "0xffffffff",
5000           "size": 2048,
5001           "shard": 1,
5002           "errors": [],
5003           "osd": 2,
5004           "primary": false
5005         }
5006       ],
5007       "selected_object_info": {
5008         "oid": {
5009           "oid": "EOBJ1",
5010           "key": "",
5011           "snapid": -2,
5012           "hash": 560836233,
5013           "max": 0,
5014           "pool": 3,
5015           "namespace": ""
5016         },
5017         "version": "27'1",
5018         "prior_version": "0'0",
5019         "last_reqid": "client.4192.0:1",
5020         "user_version": 1,
5021         "size": 7,
5022         "mtime": "2018-04-05 14:30:10.688009",
5023         "local_mtime": "2018-04-05 14:30:10.691774",
5024         "lost": 0,
5025         "flags": [
5026           "dirty",
5027           "data_digest"
5028         ],
5029         "truncate_seq": 0,
5030         "truncate_size": 0,
5031         "data_digest": "0x2ddbf8f5",
5032         "omap_digest": "0xffffffff",
5033         "expected_object_size": 0,
5034         "expected_write_size": 0,
5035         "alloc_hint_flags": 0,
5036         "manifest": {
5037           "type": 0
5038         },
5039         "watchers": {}
5040       },
5041       "union_shard_errors": [
5042         "read_error",
5043         "size_mismatch_info",
5044         "obj_size_info_mismatch"
5045       ],
5046       "errors": [
5047         "size_mismatch"
5048       ],
5049       "object": {
5050         "version": 1,
5051         "snap": "head",
5052         "locator": "",
5053         "nspace": "",
5054         "name": "EOBJ1"
5055       }
5056     },
5057     {
5058       "shards": [
5059         {
5060           "size": 2048,
5061           "errors": [
5062             "ec_hash_error"
5063           ],
5064           "shard": 2,
5065           "osd": 0,
5066           "primary": false
5067         },
5068         {
5069           "data_digest": "0x04cfa72f",
5070           "omap_digest": "0xffffffff",
5071           "size": 2048,
5072           "errors": [],
5073           "shard": 0,
5074           "osd": 1,
5075           "primary": true
5076         },
5077         {
5078           "data_digest": "0x04cfa72f",
5079           "omap_digest": "0xffffffff",
5080           "size": 2048,
5081           "errors": [],
5082           "shard": 1,
5083           "osd": 2,
5084           "primary": false
5085         }
5086       ],
5087       "selected_object_info": {
5088         "oid": {
5089           "oid": "EOBJ2",
5090           "key": "",
5091           "snapid": -2,
5092           "hash": 562812377,
5093           "max": 0,
5094           "pool": 3,
5095           "namespace": ""
5096         },
5097         "version": "33'2",
5098         "prior_version": "0'0",
5099         "last_reqid": "client.4224.0:1",
5100         "user_version": 2,
5101         "size": 7,
5102         "mtime": "2018-04-05 14:30:14.152945",
5103         "local_mtime": "2018-04-05 14:30:14.154014",
5104         "lost": 0,
5105         "flags": [
5106           "dirty",
5107           "data_digest"
5108         ],
5109         "truncate_seq": 0,
5110         "truncate_size": 0,
5111         "data_digest": "0x2ddbf8f5",
5112         "omap_digest": "0xffffffff",
5113         "expected_object_size": 0,
5114         "expected_write_size": 0,
5115         "alloc_hint_flags": 0,
5116         "manifest": {
5117           "type": 0
5118         },
5119         "watchers": {}
5120       },
5121       "union_shard_errors": [
5122         "ec_hash_error"
5123       ],
5124       "errors": [],
5125       "object": {
5126         "version": 2,
5127         "snap": "head",
5128         "locator": "",
5129         "nspace": "",
5130         "name": "EOBJ2"
5131       }
5132     },
5133     {
5134       "shards": [
5135         {
5136           "data_digest": "0x04cfa72f",
5137           "omap_digest": "0xffffffff",
5138           "size": 2048,
5139           "errors": [],
5140           "shard": 2,
5141           "osd": 0,
5142           "primary": false
5143         },
5144         {
5145           "osd": 1,
5146           "primary": true,
5147           "shard": 0,
5148           "errors": [
5149             "missing"
5150           ]
5151         },
5152         {
5153           "data_digest": "0x04cfa72f",
5154           "omap_digest": "0xffffffff",
5155           "size": 2048,
5156           "shard": 1,
5157           "errors": [],
5158           "osd": 2,
5159           "primary": false
5160         }
5161       ],
5162       "selected_object_info": {
5163         "oid": {
5164           "oid": "EOBJ3",
5165           "key": "",
5166           "snapid": -2,
5167           "hash": 3125668237,
5168           "max": 0,
5169           "pool": 3,
5170           "namespace": ""
5171         },
5172         "version": "39'3",
5173         "prior_version": "0'0",
5174         "last_reqid": "client.4258.0:1",
5175         "user_version": 3,
5176         "size": 7,
5177         "mtime": "2018-04-05 14:30:18.875544",
5178         "local_mtime": "2018-04-05 14:30:18.880153",
5179         "lost": 0,
5180         "flags": [
5181           "dirty",
5182           "data_digest"
5183         ],
5184         "truncate_seq": 0,
5185         "truncate_size": 0,
5186         "data_digest": "0x2ddbf8f5",
5187         "omap_digest": "0xffffffff",
5188         "expected_object_size": 0,
5189         "expected_write_size": 0,
5190         "alloc_hint_flags": 0,
5191         "manifest": {
5192           "type": 0
5193         },
5194         "watchers": {}
5195       },
5196       "union_shard_errors": [
5197         "missing"
5198       ],
5199       "errors": [],
5200       "object": {
5201         "version": 3,
5202         "snap": "head",
5203         "locator": "",
5204         "nspace": "",
5205         "name": "EOBJ3"
5206       }
5207     },
5208     {
5209       "shards": [
5210         {
5211           "attrs": [
5212             {
5213               "Base64": false,
5214               "value": "bad-val",
5215               "name": "key1-EOBJ4"
5216             },
5217             {
5218               "Base64": false,
5219               "value": "val2-EOBJ4",
5220               "name": "key2-EOBJ4"
5221             }
5222           ],
5223           "data_digest": "0x04cfa72f",
5224           "omap_digest": "0xffffffff",
5225           "size": 2048,
5226           "errors": [],
5227           "shard": 2,
5228           "osd": 0,
5229           "primary": false
5230         },
5231         {
5232           "osd": 1,
5233           "primary": true,
5234           "shard": 0,
5235           "errors": [],
5236           "size": 2048,
5237           "omap_digest": "0xffffffff",
5238           "data_digest": "0x04cfa72f",
5239           "attrs": [
5240             {
5241               "Base64": false,
5242               "value": "val1-EOBJ4",
5243               "name": "key1-EOBJ4"
5244             },
5245             {
5246               "Base64": false,
5247               "value": "val2-EOBJ4",
5248               "name": "key2-EOBJ4"
5249             }
5250           ]
5251         },
5252         {
5253           "osd": 2,
5254           "primary": false,
5255           "shard": 1,
5256           "errors": [],
5257           "size": 2048,
5258           "omap_digest": "0xffffffff",
5259           "data_digest": "0x04cfa72f",
5260           "attrs": [
5261             {
5262               "Base64": false,
5263               "value": "val1-EOBJ4",
5264               "name": "key1-EOBJ4"
5265             },
5266             {
5267               "Base64": false,
5268               "value": "val3-EOBJ4",
5269               "name": "key3-EOBJ4"
5270             }
5271           ]
5272         }
5273       ],
5274       "selected_object_info": {
5275         "oid": {
5276           "oid": "EOBJ4",
5277           "key": "",
5278           "snapid": -2,
5279           "hash": 1618759290,
5280           "max": 0,
5281           "pool": 3,
5282           "namespace": ""
5283         },
5284         "version": "45'6",
5285         "prior_version": "45'5",
5286         "last_reqid": "client.4296.0:1",
5287         "user_version": 6,
5288         "size": 7,
5289         "mtime": "2018-04-05 14:30:22.271983",
5290         "local_mtime": "2018-04-05 14:30:22.272840",
5291         "lost": 0,
5292         "flags": [
5293           "dirty",
5294           "data_digest"
5295         ],
5296         "truncate_seq": 0,
5297         "truncate_size": 0,
5298         "data_digest": "0x2ddbf8f5",
5299         "omap_digest": "0xffffffff",
5300         "expected_object_size": 0,
5301         "expected_write_size": 0,
5302         "alloc_hint_flags": 0,
5303         "manifest": {
5304           "type": 0
5305         },
5306         "watchers": {}
5307       },
5308       "union_shard_errors": [],
5309       "errors": [
5310         "attr_value_mismatch",
5311         "attr_name_mismatch"
5312       ],
5313       "object": {
5314         "version": 6,
5315         "snap": "head",
5316         "locator": "",
5317         "nspace": "",
5318         "name": "EOBJ4"
5319       }
5320     },
5321     {
5322       "shards": [
5323         {
5324           "data_digest": "0x04cfa72f",
5325           "omap_digest": "0xffffffff",
5326           "size": 2048,
5327           "errors": [],
5328           "shard": 2,
5329           "osd": 0,
5330           "primary": false
5331         },
5332         {
5333           "object_info": {
5334             "oid": {
5335               "oid": "EOBJ5",
5336               "key": "",
5337               "snapid": -2,
5338               "hash": 2918945441,
5339               "max": 0,
5340               "pool": 3,
5341               "namespace": ""
5342             },
5343             "version": "59'7",
5344             "prior_version": "0'0",
5345             "last_reqid": "client.4384.0:1",
5346             "user_version": 7,
5347             "size": 7,
5348             "mtime": "2018-04-05 14:30:35.162395",
5349             "local_mtime": "2018-04-05 14:30:35.166390",
5350             "lost": 0,
5351             "flags": [
5352               "dirty",
5353               "data_digest"
5354             ],
5355             "truncate_seq": 0,
5356             "truncate_size": 0,
5357             "data_digest": "0x2ddbf8f5",
5358             "omap_digest": "0xffffffff",
5359             "expected_object_size": 0,
5360             "expected_write_size": 0,
5361             "alloc_hint_flags": 0,
5362             "manifest": {
5363               "type": 0
5364             },
5365             "watchers": {}
5366           },
5367           "size": 4096,
5368           "shard": 0,
5369           "errors": [
5370             "size_mismatch_info",
5371             "ec_size_error",
5372             "obj_size_info_mismatch"
5373           ],
5374           "osd": 1,
5375           "primary": true
5376         },
5377         {
5378           "data_digest": "0x04cfa72f",
5379           "omap_digest": "0xffffffff",
5380           "size": 2048,
5381           "shard": 1,
5382           "errors": [],
5383           "osd": 2,
5384           "primary": false
5385         }
5386       ],
5387       "selected_object_info": {
5388         "oid": {
5389           "oid": "EOBJ5",
5390           "key": "",
5391           "snapid": -2,
5392           "hash": 2918945441,
5393           "max": 0,
5394           "pool": 3,
5395           "namespace": ""
5396         },
5397         "version": "59'7",
5398         "prior_version": "0'0",
5399         "last_reqid": "client.4384.0:1",
5400         "user_version": 7,
5401         "size": 7,
5402         "mtime": "2018-04-05 14:30:35.162395",
5403         "local_mtime": "2018-04-05 14:30:35.166390",
5404         "lost": 0,
5405         "flags": [
5406           "dirty",
5407           "data_digest"
5408         ],
5409         "truncate_seq": 0,
5410         "truncate_size": 0,
5411         "data_digest": "0x2ddbf8f5",
5412         "omap_digest": "0xffffffff",
5413         "expected_object_size": 0,
5414         "expected_write_size": 0,
5415         "alloc_hint_flags": 0,
5416         "manifest": {
5417           "type": 0
5418         },
5419         "watchers": {}
5420       },
5421       "union_shard_errors": [
5422         "size_mismatch_info",
5423         "ec_size_error",
5424         "obj_size_info_mismatch"
5425       ],
5426       "errors": [
5427         "size_mismatch"
5428       ],
5429       "object": {
5430         "version": 7,
5431         "snap": "head",
5432         "locator": "",
5433         "nspace": "",
5434         "name": "EOBJ5"
5435       }
5436     },
5437     {
5438       "object": {
5439         "name": "EOBJ6",
5440         "nspace": "",
5441         "locator": "",
5442         "snap": "head",
5443         "version": 8
5444       },
5445       "errors": [],
5446       "union_shard_errors": [
5447         "read_error",
5448         "hinfo_missing",
5449         "hinfo_corrupted"
5450       ],
5451       "selected_object_info": {
5452         "oid": {
5453           "oid": "EOBJ6",
5454           "key": "",
5455           "snapid": -2,
5456           "hash": 3050890866,
5457           "max": 0,
5458           "pool": 3,
5459           "namespace": ""
5460         },
5461         "version": "65'8",
5462         "prior_version": "0'0",
5463         "last_reqid": "client.4420.0:1",
5464         "user_version": 8,
5465         "size": 7,
5466         "mtime": "2018-04-05 14:30:40.914673",
5467         "local_mtime": "2018-04-05 14:30:40.917705",
5468         "lost": 0,
5469         "flags": [
5470           "dirty",
5471           "data_digest"
5472         ],
5473         "truncate_seq": 0,
5474         "truncate_size": 0,
5475         "data_digest": "0x2ddbf8f5",
5476         "omap_digest": "0xffffffff",
5477         "expected_object_size": 0,
5478         "expected_write_size": 0,
5479         "alloc_hint_flags": 0,
5480         "manifest": {
5481           "type": 0
5482         },
5483         "watchers": {}
5484       },
5485       "shards": [
5486         {
5487           "osd": 0,
5488           "primary": false,
5489           "shard": 2,
5490           "errors": [
5491             "read_error",
5492             "hinfo_missing"
5493           ],
5494           "size": 2048
5495         },
5496         {
5497           "osd": 1,
5498           "primary": true,
5499           "shard": 0,
5500           "errors": [
5501             "read_error",
5502             "hinfo_corrupted"
5503           ],
5504           "size": 2048,
5505           "hashinfo": "bad-val"
5506         },
5507         {
5508           "osd": 2,
5509           "primary": false,
5510           "shard": 1,
5511           "errors": [],
5512           "size": 2048,
5513           "omap_digest": "0xffffffff",
5514           "data_digest": "0x04cfa72f",
5515           "hashinfo": {
5516            "cumulative_shard_hashes": [
5517             {
5518               "hash": 80717615,
5519               "shard": 0
5520             },
5521             {
5522               "hash": 1534491824,
5523               "shard": 1
5524             },
5525             {
5526               "hash": 80717615,
5527               "shard": 2
5528             }
5529            ],
5530            "total_chunk_size": 2048
5531           }
5532         }
5533       ]
5534     },
5535     {
5536       "object": {
5537         "name": "EOBJ7",
5538         "nspace": "",
5539         "locator": "",
5540         "snap": "head",
5541         "version": 10
5542       },
5543       "errors": [
5544         "hinfo_inconsistency"
5545       ],
5546       "union_shard_errors": [
5547         "ec_hash_error"
5548       ],
5549       "selected_object_info": {
5550         "oid": {
5551           "oid": "EOBJ7",
5552           "key": "",
5553           "snapid": -2,
5554           "hash": 3258066308,
5555           "max": 0,
5556           "pool": 3,
5557           "namespace": ""
5558         },
5559         "version": "75'10",
5560         "prior_version": "75'9",
5561         "last_reqid": "client.4486.0:1",
5562         "user_version": 10,
5563         "size": 34,
5564         "mtime": "2018-04-05 14:30:50.995009",
5565         "local_mtime": "2018-04-05 14:30:50.996112",
5566         "lost": 0,
5567         "flags": [
5568           "dirty",
5569           "data_digest"
5570         ],
5571         "truncate_seq": 0,
5572         "truncate_size": 0,
5573         "data_digest": "0x136e4e27",
5574         "omap_digest": "0xffffffff",
5575         "expected_object_size": 0,
5576         "expected_write_size": 0,
5577         "alloc_hint_flags": 0,
5578         "manifest": {
5579           "type": 0
5580         },
5581         "watchers": {}
5582       },
5583       "shards": [
5584         {
5585           "osd": 0,
5586           "primary": false,
5587           "shard": 2,
5588           "errors": [
5589             "ec_hash_error"
5590           ],
5591           "size": 2048,
5592           "hashinfo": {
5593            "cumulative_shard_hashes": [
5594             {
5595               "hash": 80717615,
5596               "shard": 0
5597             },
5598             {
5599               "hash": 1534491824,
5600               "shard": 1
5601             },
5602             {
5603               "hash": 80717615,
5604               "shard": 2
5605             }
5606            ],
5607            "total_chunk_size": 2048
5608          }
5609         },
5610         {
5611           "osd": 1,
5612           "primary": true,
5613           "shard": 0,
5614           "errors": [],
5615           "size": 2048,
5616           "omap_digest": "0xffffffff",
5617           "data_digest": "0x5b7455a8",
5618           "hashinfo": {
5619            "cumulative_shard_hashes": [
5620             {
5621               "hash": 1534350760,
5622               "shard": 0
5623             },
5624             {
5625               "hash": 1534491824,
5626               "shard": 1
5627             },
5628             {
5629               "hash": 1534350760,
5630               "shard": 2
5631             }
5632            ],
5633            "total_chunk_size": 2048
5634           }
5635         },
5636         {
5637           "osd": 2,
5638           "primary": false,
5639           "shard": 1,
5640           "errors": [],
5641           "size": 2048,
5642           "omap_digest": "0xffffffff",
5643           "data_digest": "0x5b7455a8",
5644           "hashinfo": {
5645            "cumulative_shard_hashes": [
5646             {
5647               "hash": 1534350760,
5648               "shard": 0
5649             },
5650             {
5651               "hash": 1534491824,
5652               "shard": 1
5653             },
5654             {
5655               "hash": 1534350760,
5656               "shard": 2
5657             }
5658            ],
5659            "total_chunk_size": 2048
5660           }
5661         }
5662       ]
5663     }
5664   ],
5665   "epoch": 0
5666 }
5667 EOF
5668
5669     fi
5670
5671     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
5672     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
5673     if test $getjson = "yes"
5674     then
5675       if [ "$allow_overwrites" = "true" ]
5676       then
5677         num=4
5678       else
5679         num=5
5680       fi
5681       jq '.' $dir/json > save${num}.json
5682     fi
5683
5684     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
5685     then
5686       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
5687     fi
5688
5689     ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it
5690     teardown $dir || return 1
5691 }
5692
5693 function TEST_corrupt_scrub_erasure_appends() {
5694     corrupt_scrub_erasure $1 false
5695 }
5696
5697 function TEST_corrupt_scrub_erasure_overwrites() {
5698     if [ "$use_ec_overwrite" = "true" ]; then
5699         corrupt_scrub_erasure $1 true
5700     fi
5701 }
5702
5703 #
5704 # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost
5705 #
5706 function TEST_periodic_scrub_replicated() {
5707     local dir=$1
5708     local poolname=psr_pool
5709     local objname=POBJ
5710
5711     setup $dir || return 1
5712     run_mon $dir a --osd_pool_default_size=2 || return 1
5713     run_mgr $dir x || return 1
5714     local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
5715     ceph_osd_args+="--osd_scrub_backoff_ratio=0"
5716     run_osd $dir 0 $ceph_osd_args || return 1
5717     run_osd $dir 1 $ceph_osd_args || return 1
5718     create_rbd_pool || return 1
5719     wait_for_clean || return 1
5720
5721     create_pool $poolname 1 1 || return 1
5722     wait_for_clean || return 1
5723
5724     local osd=0
5725     add_something $dir $poolname $objname scrub || return 1
5726     local primary=$(get_primary $poolname $objname)
5727     local pg=$(get_pg $poolname $objname)
5728
5729     # Add deep-scrub only error
5730     local payload=UVWXYZ
5731     echo $payload > $dir/CORRUPT
5732     # Uses $ceph_osd_args for osd restart
5733     objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
5734
5735     # No scrub information available, so expect failure
5736     set -o pipefail
5737     !  rados list-inconsistent-obj $pg | jq '.' || return 1
5738     set +o pipefail
5739
5740     pg_deep_scrub $pg || return 1
5741
5742     # Make sure bad object found
5743     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5744
5745     flush_pg_stats
5746     local last_scrub=$(get_last_scrub_stamp $pg)
5747     # Fake a schedule scrub
5748     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
5749              trigger_scrub $pg || return 1
5750     # Wait for schedule regular scrub
5751     wait_for_scrub $pg "$last_scrub"
5752
5753     # It needed to be upgraded
5754     grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1
5755
5756     # Bad object still known
5757     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5758
5759     # Can't upgrade with this set
5760     ceph osd set nodeep-scrub
5761     # Let map change propagate to OSDs
5762     flush_pg_stats
5763     sleep 5
5764
5765     # Fake a schedule scrub
5766     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
5767              trigger_scrub $pg || return 1
5768     # Wait for schedule regular scrub
5769     # to notice scrub and skip it
5770     local found=false
5771     for i in $(seq 14 -1 0)
5772     do
5773       sleep 1
5774       ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; }
5775       echo Time left: $i seconds
5776     done
5777     test $found = "true" || return 1
5778
5779     # Bad object still known
5780     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5781
5782     flush_pg_stats
5783     # Request a regular scrub and it will be done
5784     pg_scrub $pg
5785     grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1
5786
5787     # deep-scrub error is no longer present
5788     rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1
5789 }
5790
5791 function TEST_scrub_warning() {
5792     local dir=$1
5793     local poolname=psr_pool
5794     local objname=POBJ
5795     local scrubs=5
5796     local deep_scrubs=5
5797     local i1_day=86400
5798     local i7_days=$(calc $i1_day \* 7)
5799     local i14_days=$(calc $i1_day \* 14)
5800     local overdue=0.5
5801     local conf_overdue_seconds=$(calc $i7_days + $i1_day + \( $i7_days \* $overdue \) )
5802     local pool_overdue_seconds=$(calc $i14_days + $i1_day + \( $i14_days \* $overdue \) )
5803
5804     setup $dir || return 1
5805     run_mon $dir a --osd_pool_default_size=1 || return 1
5806     run_mgr $dir x --mon_warn_pg_not_scrubbed_ratio=${overdue} --mon_warn_pg_not_deep_scrubbed_ratio=${overdue} || return 1
5807     run_osd $dir 0 $ceph_osd_args --osd_scrub_backoff_ratio=0 || return 1
5808
5809     for i in $(seq 1 $(expr $scrubs + $deep_scrubs))
5810     do
5811       create_pool $poolname-$i 1 1 || return 1
5812       wait_for_clean || return 1
5813       if [ $i = "1" ];
5814       then
5815         ceph osd pool set $poolname-$i scrub_max_interval $i14_days
5816       fi
5817       if [ $i = $(expr $scrubs + 1) ];
5818       then
5819         ceph osd pool set $poolname-$i deep_scrub_interval $i14_days
5820       fi
5821     done
5822
5823     # Only 1 osd
5824     local primary=0
5825
5826     ceph osd set noscrub || return 1
5827     ceph osd set nodeep-scrub || return 1
5828     ceph config set global osd_scrub_interval_randomize_ratio 0
5829     ceph config set global osd_deep_scrub_randomize_ratio 0
5830     ceph config set global osd_scrub_max_interval ${i7_days}
5831     ceph config set global osd_deep_scrub_interval ${i7_days}
5832
5833     # Fake schedule scrubs
5834     for i in $(seq 1 $scrubs)
5835     do
5836       if [ $i = "1" ];
5837       then
5838         overdue_seconds=$pool_overdue_seconds
5839       else
5840         overdue_seconds=$conf_overdue_seconds
5841       fi
5842       CEPH_ARGS='' ceph daemon $(get_asok_path osd.${primary}) \
5843              trigger_scrub ${i}.0 $(expr ${overdue_seconds} + ${i}00) || return 1
5844     done
5845     # Fake schedule deep scrubs
5846     for i in $(seq $(expr $scrubs + 1) $(expr $scrubs + $deep_scrubs))
5847     do
5848       if [ $i = "$(expr $scrubs + 1)" ];
5849       then
5850         overdue_seconds=$pool_overdue_seconds
5851       else
5852         overdue_seconds=$conf_overdue_seconds
5853       fi
5854       CEPH_ARGS='' ceph daemon $(get_asok_path osd.${primary}) \
5855              trigger_deep_scrub ${i}.0 $(expr ${overdue_seconds} + ${i}00) || return 1
5856     done
5857     flush_pg_stats
5858
5859     ceph health
5860     ceph health detail
5861     ceph health | grep -q "$deep_scrubs pgs not deep-scrubbed in time" || return 1
5862     ceph health | grep -q "$scrubs pgs not scrubbed in time" || return 1
5863     COUNT=$(ceph health detail | grep "not scrubbed since" | wc -l)
5864     if [ "$COUNT" != $scrubs ]; then
5865       ceph health detail | grep "not scrubbed since"
5866       return 1
5867     fi
5868     COUNT=$(ceph health detail | grep "not deep-scrubbed since" | wc -l)
5869     if [ "$COUNT" != $deep_scrubs ]; then
5870       ceph health detail | grep "not deep-scrubbed since"
5871       return 1
5872     fi
5873     return 0
5874 }
5875
5876 #
5877 # Corrupt snapset in replicated pool
5878 #
5879 function TEST_corrupt_snapset_scrub_rep() {
5880     local dir=$1
5881     local poolname=csr_pool
5882     local total_objs=2
5883
5884     setup $dir || return 1
5885     run_mon $dir a --osd_pool_default_size=2 || return 1
5886     run_mgr $dir x || return 1
5887     run_osd $dir 0 || return 1
5888     run_osd $dir 1 || return 1
5889     create_rbd_pool || return 1
5890     wait_for_clean || return 1
5891
5892     create_pool foo 1 || return 1
5893     create_pool $poolname 1 1 || return 1
5894     wait_for_clean || return 1
5895
5896     for i in $(seq 1 $total_objs) ; do
5897         objname=ROBJ${i}
5898         add_something $dir $poolname $objname || return 1
5899
5900         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
5901         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
5902     done
5903
5904     local pg=$(get_pg $poolname ROBJ0)
5905     local primary=$(get_primary $poolname ROBJ0)
5906
5907     rados -p $poolname mksnap snap1
5908     echo -n head_of_snapshot_data > $dir/change
5909
5910     for i in $(seq 1 $total_objs) ; do
5911         objname=ROBJ${i}
5912
5913         # Alternate corruption between osd.0 and osd.1
5914         local osd=$(expr $i % 2)
5915
5916         case $i in
5917         1)
5918           rados --pool $poolname put $objname $dir/change
5919           objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
5920           ;;
5921
5922         2)
5923           rados --pool $poolname put $objname $dir/change
5924           objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
5925           ;;
5926
5927         esac
5928     done
5929     rm $dir/change
5930
5931     pg_scrub $pg
5932
5933     rados list-inconsistent-pg $poolname > $dir/json || return 1
5934     # Check pg count
5935     test $(jq '. | length' $dir/json) = "1" || return 1
5936     # Check pgid
5937     test $(jq -r '.[0]' $dir/json) = $pg || return 1
5938
5939     rados list-inconsistent-obj $pg > $dir/json || return 1
5940
5941     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
5942 {
5943   "epoch": 34,
5944   "inconsistents": [
5945     {
5946       "object": {
5947         "name": "ROBJ1",
5948         "nspace": "",
5949         "locator": "",
5950         "snap": "head",
5951         "version": 8
5952       },
5953       "errors": [
5954         "snapset_inconsistency"
5955       ],
5956       "union_shard_errors": [],
5957       "selected_object_info": {
5958         "oid": {
5959           "oid": "ROBJ1",
5960           "key": "",
5961           "snapid": -2,
5962           "hash": 1454963827,
5963           "max": 0,
5964           "pool": 3,
5965           "namespace": ""
5966         },
5967         "version": "24'8",
5968         "prior_version": "21'3",
5969         "last_reqid": "client.4195.0:1",
5970         "user_version": 8,
5971         "size": 21,
5972         "mtime": "2018-04-05 14:35:43.286117",
5973         "local_mtime": "2018-04-05 14:35:43.288990",
5974         "lost": 0,
5975         "flags": [
5976           "dirty",
5977           "omap",
5978           "data_digest"
5979         ],
5980         "truncate_seq": 0,
5981         "truncate_size": 0,
5982         "data_digest": "0x53acb008",
5983         "omap_digest": "0xffffffff",
5984         "expected_object_size": 0,
5985         "expected_write_size": 0,
5986         "alloc_hint_flags": 0,
5987         "manifest": {
5988           "type": 0
5989         },
5990         "watchers": {}
5991       },
5992       "shards": [
5993         {
5994           "osd": 0,
5995           "primary": false,
5996           "errors": [],
5997           "size": 21,
5998           "snapset": {
5999             "clones": [
6000               {
6001                 "overlap": "[]",
6002                 "size": 7,
6003                 "snap": 1,
6004                 "snaps": [
6005                   1
6006                 ]
6007               }
6008             ],
6009             "snap_context": {
6010               "seq": 1,
6011               "snaps": [
6012                 1
6013               ]
6014             }
6015           }
6016         },
6017         {
6018           "osd": 1,
6019           "primary": true,
6020           "errors": [],
6021           "size": 21,
6022           "snapset": {
6023             "clones": [],
6024             "snap_context": {
6025               "seq": 0,
6026               "snaps": []
6027             }
6028           }
6029         }
6030       ]
6031     },
6032     {
6033       "object": {
6034         "name": "ROBJ2",
6035         "nspace": "",
6036         "locator": "",
6037         "snap": "head",
6038         "version": 10
6039       },
6040       "errors": [
6041         "snapset_inconsistency"
6042       ],
6043       "union_shard_errors": [],
6044       "selected_object_info": {
6045         "oid": {
6046           "oid": "ROBJ2",
6047           "key": "",
6048           "snapid": -2,
6049           "hash": 2026323607,
6050           "max": 0,
6051           "pool": 3,
6052           "namespace": ""
6053         },
6054         "version": "28'10",
6055         "prior_version": "23'6",
6056         "last_reqid": "client.4223.0:1",
6057         "user_version": 10,
6058         "size": 21,
6059         "mtime": "2018-04-05 14:35:48.326856",
6060         "local_mtime": "2018-04-05 14:35:48.328097",
6061         "lost": 0,
6062         "flags": [
6063           "dirty",
6064           "omap",
6065           "data_digest"
6066         ],
6067         "truncate_seq": 0,
6068         "truncate_size": 0,
6069         "data_digest": "0x53acb008",
6070         "omap_digest": "0xffffffff",
6071         "expected_object_size": 0,
6072         "expected_write_size": 0,
6073         "alloc_hint_flags": 0,
6074         "manifest": {
6075           "type": 0
6076         },
6077         "watchers": {}
6078       },
6079       "shards": [
6080         {
6081           "osd": 0,
6082           "primary": false,
6083           "errors": [],
6084           "size": 21,
6085           "snapset": {
6086             "clones": [],
6087             "snap_context": {
6088               "seq": 0,
6089               "snaps": []
6090             }
6091           }
6092         },
6093         {
6094           "osd": 1,
6095           "primary": true,
6096           "errors": [],
6097           "size": 21,
6098           "snapset": {
6099             "clones": [
6100               {
6101                 "overlap": "[]",
6102                 "size": 7,
6103                 "snap": 1,
6104                 "snaps": [
6105                   1
6106                 ]
6107               }
6108             ],
6109             "snap_context": {
6110               "seq": 1,
6111               "snaps": [
6112                 1
6113               ]
6114             }
6115           }
6116         }
6117       ]
6118     }
6119   ]
6120 }
6121 EOF
6122
6123     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
6124     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
6125     if test $getjson = "yes"
6126     then
6127         jq '.' $dir/json > save6.json
6128     fi
6129
6130     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
6131     then
6132       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
6133     fi
6134
6135     ERRORS=0
6136     declare -a err_strings
6137     err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid [0-9]*:.*:::ROBJ1:head : snapset inconsistent"
6138     err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid [0-9]*:.*:::ROBJ2:head : snapset inconsistent"
6139     err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 [0-9]*:.*:::ROBJ1:1 : is an unexpected clone"
6140     err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub : stat mismatch, got 3/4 objects, 1/2 clones, 3/4 dirty, 3/4 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 49/56 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
6141     err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 0 missing, 2 inconsistent objects"
6142     err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 4 errors"
6143
6144     for err_string in "${err_strings[@]}"
6145     do
6146         if ! grep -q "$err_string" $dir/osd.${primary}.log
6147         then
6148             echo "Missing log message '$err_string'"
6149             ERRORS=$(expr $ERRORS + 1)
6150         fi
6151     done
6152
6153     if [ $ERRORS != "0" ];
6154     then
6155         echo "TEST FAILED WITH $ERRORS ERRORS"
6156         return 1
6157     fi
6158
6159     ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it
6160     teardown $dir || return 1
6161 }
6162
6163 function TEST_request_scrub_priority() {
6164     local dir=$1
6165     local poolname=psr_pool
6166     local objname=POBJ
6167     local OBJECTS=64
6168     local PGS=8
6169
6170     setup $dir || return 1
6171     run_mon $dir a --osd_pool_default_size=1 || return 1
6172     run_mgr $dir x || return 1
6173     local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
6174     ceph_osd_args+="--osd_scrub_backoff_ratio=0"
6175     run_osd $dir 0 $ceph_osd_args || return 1
6176
6177     create_pool $poolname $PGS $PGS || return 1
6178     wait_for_clean || return 1
6179
6180     local osd=0
6181     add_something $dir $poolname $objname noscrub || return 1
6182     local primary=$(get_primary $poolname $objname)
6183     local pg=$(get_pg $poolname $objname)
6184     poolid=$(ceph osd dump | grep "^pool.*[']${poolname}[']" | awk '{ print $2 }')
6185
6186     local otherpgs
6187     for i in $(seq 0 $(expr $PGS - 1))
6188     do
6189         opg="${poolid}.${i}"
6190         if [ "$opg" = "$pg" ]; then
6191           continue
6192         fi
6193         otherpgs="${otherpgs}${opg} "
6194         local other_last_scrub=$(get_last_scrub_stamp $pg)
6195         # Fake a schedule scrub
6196         CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
6197              trigger_scrub $opg || return 1
6198     done
6199
6200     sleep 15
6201     flush_pg_stats
6202
6203     # Request a regular scrub and it will be done
6204     local last_scrub=$(get_last_scrub_stamp $pg)
6205     ceph pg scrub $pg
6206
6207     ceph osd unset noscrub || return 1
6208     ceph osd unset nodeep-scrub || return 1
6209
6210     wait_for_scrub $pg "$last_scrub"
6211
6212     for opg in $otherpgs $pg
6213     do
6214         wait_for_scrub $opg "$other_last_scrub"
6215     done
6216
6217     # Verify that the requested scrub ran first
6218     grep "log_channel.*scrub ok" $dir/osd.${primary}.log | head -1 | sed 's/.*[[]DBG[]]//' | grep -q $pg || return 1
6219
6220     return 0
6221 }
6222
6223
6224 main osd-scrub-repair "$@"
6225
6226 # Local Variables:
6227 # compile-command: "cd build ; make -j4 && \
6228 #    ../qa/run-standalone.sh osd-scrub-repair.sh"
6229 # End: