ceph/qa/standalone/scrub/osd-scrub-repair.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
   4 #
   5 # Author: Loic Dachary <loic@dachary.org>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17 set -x
  18 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  19
  20 if [ `uname` = FreeBSD ]; then
  21     # erasure coding overwrites are only tested on Bluestore
  22     # erasure coding on filestore is unsafe
  23     # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites
  24     use_ec_overwrite=false
  25 else
  26     use_ec_overwrite=true
  27 fi
  28
  29 # Test development and debugging
  30 # Set to "yes" in order to ignore diff errors and save results to update test
  31 getjson="no"
  32
  33 # Filter out mtime and local_mtime dates, version, prior_version and last_reqid (client) from any object_info.
  34 jqfilter='def walk(f):
  35   . as $in
  36   | if type == "object" then
  37       reduce keys[] as $key
  38         ( {}; . + { ($key):  ($in[$key] | walk(f)) } ) | f
  39     elif type == "array" then map( walk(f) ) | f
  40     else f
  41     end;
  42 walk(if type == "object" then del(.mtime) else . end)
  43 | walk(if type == "object" then del(.local_mtime) else . end)
  44 | walk(if type == "object" then del(.last_reqid) else . end)
  45 | walk(if type == "object" then del(.version) else . end)
  46 | walk(if type == "object" then del(.prior_version) else . end)'
  47
  48 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
  49
  50 function run() {
  51     local dir=$1
  52     shift
  53
  54     export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
  55     export CEPH_ARGS
  56     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  57     CEPH_ARGS+="--mon-host=$CEPH_MON "
  58     CEPH_ARGS+="--osd-skip-data-digest=false "
  59
  60     export -n CEPH_CLI_TEST_DUP_COMMAND
  61     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  62     for func in $funcs ; do
  63         $func $dir || return 1
  64     done
  65 }
  66
  67 function add_something() {
  68     local dir=$1
  69     local poolname=$2
  70     local obj=${3:-SOMETHING}
  71     local scrub=${4:-noscrub}
  72
  73     if [ "$scrub" = "noscrub" ];
  74     then
  75         ceph osd set noscrub || return 1
  76         ceph osd set nodeep-scrub || return 1
  77     else
  78         ceph osd unset noscrub || return 1
  79         ceph osd unset nodeep-scrub || return 1
  80     fi
  81
  82     local payload=ABCDEF
  83     echo $payload > $dir/ORIGINAL
  84     rados --pool $poolname put $obj $dir/ORIGINAL || return 1
  85 }
  86
  87 #
  88 # Corrupt one copy of a replicated pool
  89 #
  90 function TEST_corrupt_and_repair_replicated() {
  91     local dir=$1
  92     local poolname=rbd
  93
  94     setup $dir || return 1
  95     run_mon $dir a --osd_pool_default_size=2 || return 1
  96     run_mgr $dir x || return 1
  97     run_osd $dir 0 || return 1
  98     run_osd $dir 1 || return 1
  99     create_rbd_pool || return 1
 100     wait_for_clean || return 1
 101
 102     add_something $dir $poolname || return 1
 103     corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
 104     # Reproduces http://tracker.ceph.com/issues/8914
 105     corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1
 106
 107     teardown $dir || return 1
 108 }
 109
 110 #
 111 # Allow repair to be scheduled when some recovering is still undergoing on the same OSD
 112 #
 113 function TEST_allow_repair_during_recovery() {
 114     local dir=$1
 115     local poolname=rbd
 116
 117     setup $dir || return 1
 118     run_mon $dir a --osd_pool_default_size=2 || return 1
 119     run_mgr $dir x || return 1
 120     run_osd $dir 0 --osd_scrub_during_recovery=false \
 121                    --osd_repair_during_recovery=true \
 122                    --osd_debug_pretend_recovery_active=true || return 1
 123     run_osd $dir 1 --osd_scrub_during_recovery=false \
 124                    --osd_repair_during_recovery=true \
 125                    --osd_debug_pretend_recovery_active=true || return 1
 126     create_rbd_pool || return 1
 127     wait_for_clean || return 1
 128
 129     add_something $dir $poolname || return 1
 130     corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
 131
 132     teardown $dir || return 1
 133 }
 134
 135 #
 136 # Skip non-repair scrub correctly during recovery
 137 #
 138 function TEST_skip_non_repair_during_recovery() {
 139     local dir=$1
 140     local poolname=rbd
 141
 142     setup $dir || return 1
 143     run_mon $dir a --osd_pool_default_size=2 || return 1
 144     run_mgr $dir x || return 1
 145     run_osd $dir 0 --osd_scrub_during_recovery=false \
 146                    --osd_repair_during_recovery=true \
 147                    --osd_debug_pretend_recovery_active=true || return 1
 148     run_osd $dir 1 --osd_scrub_during_recovery=false \
 149                    --osd_repair_during_recovery=true \
 150                    --osd_debug_pretend_recovery_active=true || return 1
 151     create_rbd_pool || return 1
 152     wait_for_clean || return 1
 153
 154     add_something $dir $poolname || return 1
 155     scrub_and_not_schedule $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
 156
 157     teardown $dir || return 1
 158 }
 159
 160 function scrub_and_not_schedule() {
 161     local dir=$1
 162     local poolname=$2
 163     local osd=$3
 164
 165     #
 166     # 1) start a non-repair scrub
 167     #
 168     local pg=$(get_pg $poolname SOMETHING)
 169     local last_scrub=$(get_last_scrub_stamp $pg)
 170     ceph pg scrub $pg
 171
 172     #
 173     # 2) Assure the scrub is not scheduled
 174     #
 175     for ((i=0; i < 3; i++)); do
 176         if test "$(get_last_scrub_stamp $pg)" '>' "$last_scrub" ; then
 177             return 1
 178         fi
 179         sleep 1
 180     done
 181
 182     #
 183     # 3) Access to the file must OK
 184     #
 185     objectstore_tool $dir $osd SOMETHING list-attrs || return 1
 186     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 187     diff $dir/ORIGINAL $dir/COPY || return 1
 188 }
 189
 190 function corrupt_and_repair_two() {
 191     local dir=$1
 192     local poolname=$2
 193     local first=$3
 194     local second=$4
 195
 196     #
 197     # 1) remove the corresponding file from the OSDs
 198     #
 199     pids=""
 200     run_in_background pids objectstore_tool $dir $first SOMETHING remove
 201     run_in_background pids objectstore_tool $dir $second SOMETHING remove
 202     wait_background pids
 203     return_code=$?
 204     if [ $return_code -ne 0 ]; then return $return_code; fi
 205
 206     #
 207     # 2) repair the PG
 208     #
 209     local pg=$(get_pg $poolname SOMETHING)
 210     repair $pg
 211     #
 212     # 3) The files must be back
 213     #
 214     pids=""
 215     run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs
 216     run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs
 217     wait_background pids
 218     return_code=$?
 219     if [ $return_code -ne 0 ]; then return $return_code; fi
 220
 221     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 222     diff $dir/ORIGINAL $dir/COPY || return 1
 223 }
 224
 225 #
 226 # 1) add an object
 227 # 2) remove the corresponding file from a designated OSD
 228 # 3) repair the PG
 229 # 4) check that the file has been restored in the designated OSD
 230 #
 231 function corrupt_and_repair_one() {
 232     local dir=$1
 233     local poolname=$2
 234     local osd=$3
 235
 236     #
 237     # 1) remove the corresponding file from the OSD
 238     #
 239     objectstore_tool $dir $osd SOMETHING remove || return 1
 240     #
 241     # 2) repair the PG
 242     #
 243     local pg=$(get_pg $poolname SOMETHING)
 244     repair $pg
 245     #
 246     # 3) The file must be back
 247     #
 248     objectstore_tool $dir $osd SOMETHING list-attrs || return 1
 249     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 250     diff $dir/ORIGINAL $dir/COPY || return 1
 251 }
 252
 253 function corrupt_and_repair_erasure_coded() {
 254     local dir=$1
 255     local poolname=$2
 256
 257     add_something $dir $poolname || return 1
 258
 259     local primary=$(get_primary $poolname SOMETHING)
 260     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 261     local not_primary_first=${osds[0]}
 262     local not_primary_second=${osds[1]}
 263
 264     # Reproduces http://tracker.ceph.com/issues/10017
 265     corrupt_and_repair_one $dir $poolname $primary  || return 1
 266     # Reproduces http://tracker.ceph.com/issues/10409
 267     corrupt_and_repair_one $dir $poolname $not_primary_first || return 1
 268     corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1
 269     corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1
 270
 271 }
 272
 273 function auto_repair_erasure_coded() {
 274     local dir=$1
 275     local allow_overwrites=$2
 276     local poolname=ecpool
 277
 278     # Launch a cluster with 5 seconds scrub interval
 279     setup $dir || return 1
 280     run_mon $dir a || return 1
 281     run_mgr $dir x || return 1
 282     local ceph_osd_args="--osd-scrub-auto-repair=true \
 283             --osd-deep-scrub-interval=5 \
 284             --osd-scrub-max-interval=5 \
 285             --osd-scrub-min-interval=5 \
 286             --osd-scrub-interval-randomize-ratio=0"
 287     for id in $(seq 0 2) ; do
 288         if [ "$allow_overwrites" = "true" ]; then
 289             run_osd $dir $id $ceph_osd_args || return 1
 290         else
 291             run_osd_filestore $dir $id $ceph_osd_args || return 1
 292         fi
 293     done
 294     create_rbd_pool || return 1
 295     wait_for_clean || return 1
 296
 297     # Create an EC pool
 298     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 299
 300     # Put an object
 301     local payload=ABCDEF
 302     echo $payload > $dir/ORIGINAL
 303     rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
 304
 305     # Remove the object from one shard physically
 306     # Restarted osd get $ceph_osd_args passed
 307     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
 308     # Wait for auto repair
 309     local pgid=$(get_pg $poolname SOMETHING)
 310     wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)"
 311     wait_for_clean || return 1
 312     # Verify - the file should be back
 313     # Restarted osd get $ceph_osd_args passed
 314     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
 315     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 316     diff $dir/ORIGINAL $dir/COPY || return 1
 317
 318     # Tear down
 319     teardown $dir || return 1
 320 }
 321
 322 function TEST_auto_repair_erasure_coded_appends() {
 323     auto_repair_erasure_coded $1 false
 324 }
 325
 326 function TEST_auto_repair_erasure_coded_overwrites() {
 327     if [ "$use_ec_overwrite" = "true" ]; then
 328         auto_repair_erasure_coded $1 true
 329     fi
 330 }
 331
 332 function TEST_auto_repair_bluestore_basic() {
 333     local dir=$1
 334     local poolname=testpool
 335
 336     # Launch a cluster with 5 seconds scrub interval
 337     setup $dir || return 1
 338     run_mon $dir a || return 1
 339     run_mgr $dir x || return 1
 340     local ceph_osd_args="--osd-scrub-auto-repair=true \
 341             --osd_deep_scrub_randomize_ratio=0 \
 342             --osd-scrub-interval-randomize-ratio=0"
 343     for id in $(seq 0 2) ; do
 344         run_osd $dir $id $ceph_osd_args || return 1
 345     done
 346
 347     create_pool $poolname 1 1 || return 1
 348     ceph osd pool set $poolname size 2
 349     wait_for_clean || return 1
 350
 351     # Put an object
 352     local payload=ABCDEF
 353     echo $payload > $dir/ORIGINAL
 354     rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
 355
 356     # Remove the object from one shard physically
 357     # Restarted osd get $ceph_osd_args passed
 358     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
 359
 360     local pgid=$(get_pg $poolname SOMETHING)
 361     local primary=$(get_primary $poolname SOMETHING)
 362     local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
 363     ceph tell $pgid deep_scrub
 364     ceph tell $pgid scrub
 365
 366     # Wait for auto repair
 367     wait_for_scrub $pgid "$last_scrub_stamp" || return 1
 368     wait_for_clean || return 1
 369     ceph pg dump pgs
 370     # Verify - the file should be back
 371     # Restarted osd get $ceph_osd_args passed
 372     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
 373     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING get-bytes $dir/COPY || return 1
 374     diff $dir/ORIGINAL $dir/COPY || return 1
 375     grep scrub_finish $dir/osd.${primary}.log
 376
 377     # Tear down
 378     teardown $dir || return 1
 379 }
 380
 381 function TEST_auto_repair_bluestore_scrub() {
 382     local dir=$1
 383     local poolname=testpool
 384
 385     # Launch a cluster with 5 seconds scrub interval
 386     setup $dir || return 1
 387     run_mon $dir a || return 1
 388     run_mgr $dir x || return 1
 389     local ceph_osd_args="--osd-scrub-auto-repair=true \
 390             --osd_deep_scrub_randomize_ratio=0 \
 391             --osd-scrub-interval-randomize-ratio=0"
 392     for id in $(seq 0 2) ; do
 393         run_osd $dir $id $ceph_osd_args || return 1
 394     done
 395
 396     create_pool $poolname 1 1 || return 1
 397     ceph osd pool set $poolname size 2
 398     wait_for_clean || return 1
 399
 400     # Put an object
 401     local payload=ABCDEF
 402     echo $payload > $dir/ORIGINAL
 403     rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
 404
 405     # Remove the object from one shard physically
 406     # Restarted osd get $ceph_osd_args passed
 407     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
 408
 409     local pgid=$(get_pg $poolname SOMETHING)
 410     local primary=$(get_primary $poolname SOMETHING)
 411     local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
 412     ceph tell $pgid scrub
 413
 414     # Wait for scrub -> auto repair
 415     wait_for_scrub $pgid "$last_scrub_stamp" || return 1
 416     ceph pg dump pgs
 417     # Actually this causes 2 scrubs, so we better wait a little longer
 418     sleep 5
 419     wait_for_clean || return 1
 420     ceph pg dump pgs
 421     # Verify - the file should be back
 422     # Restarted osd get $ceph_osd_args passed
 423     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
 424     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 425     diff $dir/ORIGINAL $dir/COPY || return 1
 426     grep scrub_finish $dir/osd.${primary}.log
 427
 428     # This should have caused 1 object to be repaired
 429     COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
 430     test "$COUNT" = "1" || return 1
 431
 432     # Tear down
 433     teardown $dir || return 1
 434 }
 435
 436 function TEST_auto_repair_bluestore_failed() {
 437     local dir=$1
 438     local poolname=testpool
 439
 440     # Launch a cluster with 5 seconds scrub interval
 441     setup $dir || return 1
 442     run_mon $dir a || return 1
 443     run_mgr $dir x || return 1
 444     local ceph_osd_args="--osd-scrub-auto-repair=true \
 445             --osd_deep_scrub_randomize_ratio=0 \
 446             --osd-scrub-interval-randomize-ratio=0"
 447     for id in $(seq 0 2) ; do
 448         run_osd $dir $id $ceph_osd_args || return 1
 449     done
 450
 451     create_pool $poolname 1 1 || return 1
 452     ceph osd pool set $poolname size 2
 453     wait_for_clean || return 1
 454
 455     # Put an object
 456     local payload=ABCDEF
 457     echo $payload > $dir/ORIGINAL
 458     for i in $(seq 1 10)
 459     do
 460       rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
 461     done
 462
 463     # Remove the object from one shard physically
 464     # Restarted osd get $ceph_osd_args passed
 465     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj1 remove || return 1
 466     # obj2 can't be repaired
 467     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj2 remove || return 1
 468     objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 rm-attr _ || return 1
 469
 470     local pgid=$(get_pg $poolname obj1)
 471     local primary=$(get_primary $poolname obj1)
 472     local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
 473     ceph tell $pgid deep_scrub
 474     ceph tell $pgid scrub
 475
 476     # Wait for auto repair
 477     wait_for_scrub $pgid "$last_scrub_stamp" || return 1
 478     wait_for_clean || return 1
 479     flush_pg_stats
 480     grep scrub_finish $dir/osd.${primary}.log
 481     grep -q "scrub_finish.*still present after re-scrub" $dir/osd.${primary}.log || return 1
 482     ceph pg dump pgs
 483     ceph pg dump pgs | grep -q "^$(pgid).*+failed_repair" || return 1
 484
 485     # Verify - obj1 should be back
 486     # Restarted osd get $ceph_osd_args passed
 487     objectstore_tool $dir $(get_not_primary $poolname obj1) obj1 list-attrs || return 1
 488     rados --pool $poolname get obj1 $dir/COPY || return 1
 489     diff $dir/ORIGINAL $dir/COPY || return 1
 490     grep scrub_finish $dir/osd.${primary}.log
 491
 492     # Make it repairable
 493     objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 remove || return 1
 494     repair $pgid
 495     sleep 2
 496
 497     ceph pg dump pgs
 498     ceph pg dump pgs | grep -q "^$(pgid).* active+clean " || return 1
 499     grep scrub_finish $dir/osd.${primary}.log
 500
 501     # Tear down
 502     teardown $dir || return 1
 503 }
 504
 505 function TEST_auto_repair_bluestore_failed_norecov() {
 506     local dir=$1
 507     local poolname=testpool
 508
 509     # Launch a cluster with 5 seconds scrub interval
 510     setup $dir || return 1
 511     run_mon $dir a || return 1
 512     run_mgr $dir x || return 1
 513     local ceph_osd_args="--osd-scrub-auto-repair=true \
 514             --osd_deep_scrub_randomize_ratio=0 \
 515             --osd-scrub-interval-randomize-ratio=0"
 516     for id in $(seq 0 2) ; do
 517         run_osd $dir $id $ceph_osd_args || return 1
 518     done
 519
 520     create_pool $poolname 1 1 || return 1
 521     ceph osd pool set $poolname size 2
 522     wait_for_clean || return 1
 523
 524     # Put an object
 525     local payload=ABCDEF
 526     echo $payload > $dir/ORIGINAL
 527     for i in $(seq 1 10)
 528     do
 529       rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
 530     done
 531
 532     # Remove the object from one shard physically
 533     # Restarted osd get $ceph_osd_args passed
 534     # obj1 can't be repaired
 535     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj1 remove || return 1
 536     objectstore_tool $dir $(get_primary $poolname SOMETHING) obj1 rm-attr _ || return 1
 537     # obj2 can't be repaired
 538     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj2 remove || return 1
 539     objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 rm-attr _ || return 1
 540
 541     local pgid=$(get_pg $poolname obj1)
 542     local primary=$(get_primary $poolname obj1)
 543     local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
 544     ceph tell $pgid deep_scrub
 545     ceph tell $pgid scrub
 546
 547     # Wait for auto repair
 548     wait_for_scrub $pgid "$last_scrub_stamp" || return 1
 549     wait_for_clean || return 1
 550     flush_pg_stats
 551     grep -q "scrub_finish.*present with no repair possible" $dir/osd.${primary}.log || return 1
 552     ceph pg dump pgs
 553     ceph pg dump pgs | grep -q "^$(pgid).*+failed_repair" || return 1
 554
 555     # Tear down
 556     teardown $dir || return 1
 557 }
 558
 559 function TEST_repair_stats() {
 560     local dir=$1
 561     local poolname=testpool
 562     local OSDS=2
 563     local OBJS=30
 564     # This need to be an even number
 565     local REPAIRS=20
 566
 567     # Launch a cluster with 5 seconds scrub interval
 568     setup $dir || return 1
 569     run_mon $dir a || return 1
 570     run_mgr $dir x || return 1
 571     local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \
 572             --osd-scrub-interval-randomize-ratio=0"
 573     for id in $(seq 0 $(expr $OSDS - 1)) ; do
 574         run_osd $dir $id $ceph_osd_args || return 1
 575     done
 576
 577     create_pool $poolname 1 1 || return 1
 578     ceph osd pool set $poolname size 2
 579     wait_for_clean || return 1
 580
 581     # Put an object
 582     local payload=ABCDEF
 583     echo $payload > $dir/ORIGINAL
 584     for i in $(seq 1 $OBJS)
 585     do
 586       rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
 587     done
 588
 589     # Remove the object from one shard physically
 590     # Restarted osd get $ceph_osd_args passed
 591     local other=$(get_not_primary $poolname obj1)
 592     local pgid=$(get_pg $poolname obj1)
 593     local primary=$(get_primary $poolname obj1)
 594
 595     kill_daemons $dir TERM osd.$other >&2 < /dev/null || return 1
 596     kill_daemons $dir TERM osd.$primary >&2 < /dev/null || return 1
 597     for i in $(seq 1 $REPAIRS)
 598     do
 599       # Remove from both osd.0 and osd.1
 600       OSD=$(expr $i % 2)
 601       _objectstore_tool_nodown $dir $OSD obj$i remove || return 1
 602     done
 603     activate_osd $dir $primary $ceph_osd_args || return 1
 604     activate_osd $dir $other $ceph_osd_args || return 1
 605     wait_for_clean || return 1
 606
 607     repair $pgid
 608     wait_for_clean || return 1
 609     ceph pg dump pgs
 610     flush_pg_stats
 611
 612     # This should have caused 1 object to be repaired
 613     ceph pg $pgid query | jq '.info.stats.stat_sum'
 614     COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
 615     test "$COUNT" = "$REPAIRS" || return 1
 616
 617     ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $primary )"
 618     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $primary ).num_shards_repaired")
 619     test "$COUNT" = "$(expr $REPAIRS / 2)" || return 1
 620
 621     ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $other )"
 622     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $other ).num_shards_repaired")
 623     test "$COUNT" = "$(expr $REPAIRS / 2)" || return 1
 624
 625     ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum"
 626     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
 627     test "$COUNT" = "$REPAIRS" || return 1
 628
 629     # Tear down
 630     teardown $dir || return 1
 631 }
 632
 633 function TEST_repair_stats_ec() {
 634     local dir=$1
 635     local poolname=testpool
 636     local OSDS=3
 637     local OBJS=30
 638     # This need to be an even number
 639     local REPAIRS=26
 640     local allow_overwrites=false
 641
 642     # Launch a cluster with 5 seconds scrub interval
 643     setup $dir || return 1
 644     run_mon $dir a || return 1
 645     run_mgr $dir x || return 1
 646     local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \
 647             --osd-scrub-interval-randomize-ratio=0"
 648     for id in $(seq 0 $(expr $OSDS - 1)) ; do
 649         run_osd $dir $id $ceph_osd_args || return 1
 650     done
 651
 652     # Create an EC pool
 653     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 654
 655     # Put an object
 656     local payload=ABCDEF
 657     echo $payload > $dir/ORIGINAL
 658     for i in $(seq 1 $OBJS)
 659     do
 660       rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
 661     done
 662
 663     # Remove the object from one shard physically
 664     # Restarted osd get $ceph_osd_args passed
 665     local other=$(get_not_primary $poolname obj1)
 666     local pgid=$(get_pg $poolname obj1)
 667     local primary=$(get_primary $poolname obj1)
 668
 669     kill_daemons $dir TERM osd.$other >&2 < /dev/null || return 1
 670     kill_daemons $dir TERM osd.$primary >&2 < /dev/null || return 1
 671     for i in $(seq 1 $REPAIRS)
 672     do
 673       # Remove from both osd.0 and osd.1
 674       OSD=$(expr $i % 2)
 675       _objectstore_tool_nodown $dir $OSD obj$i remove || return 1
 676     done
 677     activate_osd $dir $primary $ceph_osd_args || return 1
 678     activate_osd $dir $other $ceph_osd_args || return 1
 679     wait_for_clean || return 1
 680
 681     repair $pgid
 682     wait_for_clean || return 1
 683     ceph pg dump pgs
 684     flush_pg_stats
 685
 686     # This should have caused 1 object to be repaired
 687     ceph pg $pgid query | jq '.info.stats.stat_sum'
 688     COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
 689     test "$COUNT" = "$REPAIRS" || return 1
 690
 691     for osd in $(seq 0 $(expr $OSDS - 1)) ; do
 692       if [ $osd = $other -o $osd = $primary ]; then
 693         repair=$(expr $REPAIRS / 2)
 694       else
 695         repair="0"
 696       fi
 697
 698       ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $osd )"
 699       COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $osd ).num_shards_repaired")
 700       test "$COUNT" = "$repair" || return 1
 701     done
 702
 703     ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum"
 704     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
 705     test "$COUNT" = "$REPAIRS" || return 1
 706
 707     # Tear down
 708     teardown $dir || return 1
 709 }
 710
 711 function corrupt_and_repair_jerasure() {
 712     local dir=$1
 713     local allow_overwrites=$2
 714     local poolname=ecpool
 715
 716     setup $dir || return 1
 717     run_mon $dir a || return 1
 718     run_mgr $dir x || return 1
 719     for id in $(seq 0 3) ; do
 720         if [ "$allow_overwrites" = "true" ]; then
 721             run_osd $dir $id || return 1
 722         else
 723             run_osd_filestore $dir $id || return 1
 724         fi
 725     done
 726     create_rbd_pool || return 1
 727     wait_for_clean || return 1
 728
 729     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 730     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 731
 732     teardown $dir || return 1
 733 }
 734
 735 function TEST_corrupt_and_repair_jerasure_appends() {
 736     corrupt_and_repair_jerasure $1 false
 737 }
 738
 739 function TEST_corrupt_and_repair_jerasure_overwrites() {
 740     if [ "$use_ec_overwrite" = "true" ]; then
 741         corrupt_and_repair_jerasure $1 true
 742     fi
 743 }
 744
 745 function corrupt_and_repair_lrc() {
 746     local dir=$1
 747     local allow_overwrites=$2
 748     local poolname=ecpool
 749
 750     setup $dir || return 1
 751     run_mon $dir a || return 1
 752     run_mgr $dir x || return 1
 753     for id in $(seq 0 9) ; do
 754         if [ "$allow_overwrites" = "true" ]; then
 755             run_osd $dir $id || return 1
 756         else
 757             run_osd_filestore $dir $id || return 1
 758         fi
 759     done
 760     create_rbd_pool || return 1
 761     wait_for_clean || return 1
 762
 763     create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1
 764     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 765
 766     teardown $dir || return 1
 767 }
 768
 769 function TEST_corrupt_and_repair_lrc_appends() {
 770     corrupt_and_repair_lrc $1 false
 771 }
 772
 773 function TEST_corrupt_and_repair_lrc_overwrites() {
 774     if [ "$use_ec_overwrite" = "true" ]; then
 775         corrupt_and_repair_lrc $1 true
 776     fi
 777 }
 778
 779 function unfound_erasure_coded() {
 780     local dir=$1
 781     local allow_overwrites=$2
 782     local poolname=ecpool
 783     local payload=ABCDEF
 784
 785     setup $dir || return 1
 786     run_mon $dir a || return 1
 787     run_mgr $dir x || return 1
 788     for id in $(seq 0 3) ; do
 789         if [ "$allow_overwrites" = "true" ]; then
 790             run_osd $dir $id || return 1
 791         else
 792             run_osd_filestore $dir $id || return 1
 793         fi
 794     done
 795
 796     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 797
 798     add_something $dir $poolname || return 1
 799
 800     local primary=$(get_primary $poolname SOMETHING)
 801     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 802     local not_primary_first=${osds[0]}
 803     local not_primary_second=${osds[1]}
 804     local not_primary_third=${osds[2]}
 805
 806     #
 807     # 1) remove the corresponding file from the OSDs
 808     #
 809     pids=""
 810     run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove
 811     run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove
 812     run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove
 813     wait_background pids
 814     return_code=$?
 815     if [ $return_code -ne 0 ]; then return $return_code; fi
 816
 817     #
 818     # 2) repair the PG
 819     #
 820     local pg=$(get_pg $poolname SOMETHING)
 821     repair $pg
 822     #
 823     # 3) check pg state
 824     #
 825     # it may take a bit to appear due to mon/mgr asynchrony
 826     for f in `seq 1 60`; do
 827         ceph -s | grep "1/1 objects unfound" && break
 828         sleep 1
 829     done
 830     ceph -s|grep "4 up" || return 1
 831     ceph -s|grep "4 in" || return 1
 832     ceph -s|grep "1/1 objects unfound" || return 1
 833
 834     teardown $dir || return 1
 835 }
 836
 837 function TEST_unfound_erasure_coded_appends() {
 838     unfound_erasure_coded $1 false
 839 }
 840
 841 function TEST_unfound_erasure_coded_overwrites() {
 842     if [ "$use_ec_overwrite" = "true" ]; then
 843         unfound_erasure_coded $1 true
 844     fi
 845 }
 846
 847 #
 848 # list_missing for EC pool
 849 #
 850 function list_missing_erasure_coded() {
 851     local dir=$1
 852     local allow_overwrites=$2
 853     local poolname=ecpool
 854
 855     setup $dir || return 1
 856     run_mon $dir a || return 1
 857     run_mgr $dir x || return 1
 858     for id in $(seq 0 2) ; do
 859         if [ "$allow_overwrites" = "true" ]; then
 860             run_osd $dir $id || return 1
 861         else
 862             run_osd_filestore $dir $id || return 1
 863         fi
 864     done
 865     create_rbd_pool || return 1
 866     wait_for_clean || return 1
 867
 868     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 869
 870     # Put an object and remove the two shards (including primary)
 871     add_something $dir $poolname MOBJ0 || return 1
 872     local -a osds0=($(get_osds $poolname MOBJ0))
 873
 874     # Put another object and remove two shards (excluding primary)
 875     add_something $dir $poolname MOBJ1 || return 1
 876     local -a osds1=($(get_osds $poolname MOBJ1))
 877
 878     # Stop all osd daemons
 879     for id in $(seq 0 2) ; do
 880         kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1
 881     done
 882
 883     id=${osds0[0]}
 884     ceph-objectstore-tool --data-path $dir/$id \
 885         MOBJ0 remove || return 1
 886     id=${osds0[1]}
 887     ceph-objectstore-tool --data-path $dir/$id \
 888         MOBJ0 remove || return 1
 889
 890     id=${osds1[1]}
 891     ceph-objectstore-tool --data-path $dir/$id \
 892         MOBJ1 remove || return 1
 893     id=${osds1[2]}
 894     ceph-objectstore-tool --data-path $dir/$id \
 895         MOBJ1 remove || return 1
 896
 897     for id in $(seq 0 2) ; do
 898         activate_osd $dir $id >&2 || return 1
 899     done
 900     create_rbd_pool || return 1
 901     wait_for_clean || return 1
 902
 903     # Get get - both objects should in the same PG
 904     local pg=$(get_pg $poolname MOBJ0)
 905
 906     # Repair the PG, which triggers the recovering,
 907     # and should mark the object as unfound
 908     repair $pg
 909
 910     for i in $(seq 0 120) ; do
 911         [ $i -lt 60 ] || return 1
 912         matches=$(ceph pg $pg list_unfound | egrep "MOBJ0|MOBJ1" | wc -l)
 913         [ $matches -eq 2 ] && break
 914     done
 915
 916     teardown $dir || return 1
 917 }
 918
 919 function TEST_list_missing_erasure_coded_appends() {
 920     list_missing_erasure_coded $1 false
 921 }
 922
 923 function TEST_list_missing_erasure_coded_overwrites() {
 924     if [ "$use_ec_overwrite" = "true" ]; then
 925         list_missing_erasure_coded $1 true
 926     fi
 927 }
 928
 929 #
 930 # Corrupt one copy of a replicated pool
 931 #
 932 function TEST_corrupt_scrub_replicated() {
 933     local dir=$1
 934     local poolname=csr_pool
 935     local total_objs=19
 936
 937     setup $dir || return 1
 938     run_mon $dir a --osd_pool_default_size=2 || return 1
 939     run_mgr $dir x || return 1
 940     run_osd $dir 0 || return 1
 941     run_osd $dir 1 || return 1
 942     create_rbd_pool || return 1
 943     wait_for_clean || return 1
 944
 945     create_pool foo 1 || return 1
 946     create_pool $poolname 1 1 || return 1
 947     wait_for_clean || return 1
 948
 949     for i in $(seq 1 $total_objs) ; do
 950         objname=ROBJ${i}
 951         add_something $dir $poolname $objname || return 1
 952
 953         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
 954         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
 955     done
 956
 957     # Increase file 1 MB + 1KB
 958     dd if=/dev/zero of=$dir/new.ROBJ19 bs=1024 count=1025
 959     rados --pool $poolname put $objname $dir/new.ROBJ19 || return 1
 960     rm -f $dir/new.ROBJ19
 961
 962     local pg=$(get_pg $poolname ROBJ0)
 963     local primary=$(get_primary $poolname ROBJ0)
 964
 965     # Compute an old omap digest and save oi
 966     CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \
 967         config set osd_deep_scrub_update_digest_min_age 0
 968     CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \
 969         config set osd_deep_scrub_update_digest_min_age 0
 970     pg_deep_scrub $pg
 971
 972     for i in $(seq 1 $total_objs) ; do
 973         objname=ROBJ${i}
 974
 975         # Alternate corruption between osd.0 and osd.1
 976         local osd=$(expr $i % 2)
 977
 978         case $i in
 979         1)
 980             # Size (deep scrub data_digest too)
 981             local payload=UVWXYZZZ
 982             echo $payload > $dir/CORRUPT
 983             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 984             ;;
 985
 986         2)
 987             # digest (deep scrub only)
 988             local payload=UVWXYZ
 989             echo $payload > $dir/CORRUPT
 990             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 991             ;;
 992
 993         3)
 994              # missing
 995              objectstore_tool $dir $osd $objname remove || return 1
 996              ;;
 997
 998          4)
 999              # Modify omap value (deep scrub only)
1000              objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1
1001              ;;
1002
1003          5)
1004             # Delete omap key (deep scrub only)
1005             objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1
1006             ;;
1007
1008          6)
1009             # Add extra omap key (deep scrub only)
1010             echo extra > $dir/extra-val
1011             objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1
1012             rm $dir/extra-val
1013             ;;
1014
1015          7)
1016             # Modify omap header (deep scrub only)
1017             echo -n newheader > $dir/hdr
1018             objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1
1019             rm $dir/hdr
1020             ;;
1021
1022          8)
1023             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
1024             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
1025
1026             # Break xattrs
1027             echo -n bad-val > $dir/bad-val
1028             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
1029             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
1030             echo -n val3-$objname > $dir/newval
1031             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
1032             rm $dir/bad-val $dir/newval
1033             ;;
1034
1035         9)
1036             objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi
1037             echo -n D > $dir/change
1038             rados --pool $poolname put $objname $dir/change
1039             objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi
1040             rm $dir/oi $dir/change
1041             ;;
1042
1043           # ROBJ10 must be handled after digests are re-computed by a deep scrub below
1044           # ROBJ11 must be handled with config change before deep scrub
1045           # ROBJ12 must be handled with config change before scrubs
1046           # ROBJ13 must be handled before scrubs
1047
1048         14)
1049             echo -n bad-val > $dir/bad-val
1050             objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1
1051             objectstore_tool $dir 1 $objname rm-attr _ || return 1
1052             rm $dir/bad-val
1053             ;;
1054
1055         15)
1056             objectstore_tool $dir $osd $objname rm-attr _ || return 1
1057             ;;
1058
1059         16)
1060             objectstore_tool $dir 0 $objname rm-attr snapset || return 1
1061             echo -n bad-val > $dir/bad-val
1062             objectstore_tool $dir 1 $objname set-attr snapset $dir/bad-val || return 1
1063             ;;
1064
1065         17)
1066             # Deep-scrub only (all replicas are diffent than the object info
1067            local payload=ROBJ17
1068            echo $payload > $dir/new.ROBJ17
1069            objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ17 || return 1
1070            objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ17 || return 1
1071            ;;
1072
1073         18)
1074             # Deep-scrub only (all replicas are diffent than the object info
1075            local payload=ROBJ18
1076            echo $payload > $dir/new.ROBJ18
1077            objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ18 || return 1
1078            objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ18 || return 1
1079            # Make one replica have a different object info, so a full repair must happen too
1080            objectstore_tool $dir $osd $objname corrupt-info || return 1
1081            ;;
1082
1083         19)
1084            # Set osd-max-object-size smaller than this object's size
1085
1086         esac
1087     done
1088
1089     local pg=$(get_pg $poolname ROBJ0)
1090
1091     ceph tell osd.\* injectargs -- --osd-max-object-size=1048576
1092
1093     inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1094     inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1095     inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1096     inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1097
1098     pg_scrub $pg
1099
1100     ERRORS=0
1101     declare -a s_err_strings
1102     err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:30259878:::ROBJ15:head : candidate had a missing info key"
1103     err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:33aca486:::ROBJ18:head : object info inconsistent "
1104     err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:5c7b2c47:::ROBJ16:head : candidate had a corrupt snapset"
1105     err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:5c7b2c47:::ROBJ16:head : candidate had a missing snapset key"
1106     err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:5c7b2c47:::ROBJ16:head : failed to pick suitable object info"
1107     err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:86586531:::ROBJ8:head : attr value mismatch '_key1-ROBJ8', attr name mismatch '_key3-ROBJ8', attr name mismatch '_key2-ROBJ8'"
1108     err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:bc819597:::ROBJ12:head : candidate had a stat error"
1109     err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:c0c86b1d:::ROBJ14:head : candidate had a missing info key"
1110     err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:c0c86b1d:::ROBJ14:head : candidate had a corrupt info"
1111     err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:c0c86b1d:::ROBJ14:head : failed to pick suitable object info"
1112     err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : candidate size 9 info size 7 mismatch"
1113     err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : size 9 != size 7 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from shard 0"
1114     err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:d60617f9:::ROBJ13:head : candidate had a stat error"
1115     err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 3:f2a5b2a4:::ROBJ3:head : missing"
1116     err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ffdb2004:::ROBJ9:head : candidate size 1 info size 7 mismatch"
1117     err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ffdb2004:::ROBJ9:head : object info inconsistent "
1118     err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 3:c0c86b1d:::ROBJ14:head : no '_' attr"
1119     err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 3:5c7b2c47:::ROBJ16:head : can't decode 'snapset' attr buffer::malformed_input: .* no longer understand old encoding version 3 < 97"
1120     err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub : stat mismatch, got 19/19 objects, 0/0 clones, 18/19 dirty, 18/19 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 1049713/1049720 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
1121     err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 1 missing, 8 inconsistent objects"
1122     err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 18 errors"
1123     err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:123a5f55:::ROBJ19:head : size 1049600 > 1048576 is too large"
1124
1125     for err_string in "${err_strings[@]}"
1126     do
1127         if ! grep -q "$err_string" $dir/osd.${primary}.log
1128         then
1129             echo "Missing log message '$err_string'"
1130             ERRORS=$(expr $ERRORS + 1)
1131         fi
1132     done
1133
1134     rados list-inconsistent-pg $poolname > $dir/json || return 1
1135     # Check pg count
1136     test $(jq '. | length' $dir/json) = "1" || return 1
1137     # Check pgid
1138     test $(jq -r '.[0]' $dir/json) = $pg || return 1
1139
1140     rados list-inconsistent-obj $pg > $dir/json || return 1
1141     # Get epoch for repair-get requests
1142     epoch=$(jq .epoch $dir/json)
1143
1144     jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
1145 {
1146   "inconsistents": [
1147     {
1148       "shards": [
1149         {
1150           "size": 7,
1151           "errors": [],
1152           "osd": 0,
1153           "primary": false
1154         },
1155         {
1156           "object_info": {
1157             "oid": {
1158               "oid": "ROBJ1",
1159               "key": "",
1160               "snapid": -2,
1161               "hash": 1454963827,
1162               "max": 0,
1163               "pool": 3,
1164               "namespace": ""
1165             },
1166             "version": "51'58",
1167             "prior_version": "21'3",
1168             "last_reqid": "osd.1.0:57",
1169             "user_version": 3,
1170             "size": 7,
1171             "mtime": "",
1172             "local_mtime": "",
1173             "lost": 0,
1174             "flags": [
1175               "dirty",
1176               "omap",
1177               "data_digest",
1178               "omap_digest"
1179             ],
1180             "truncate_seq": 0,
1181             "truncate_size": 0,
1182             "data_digest": "0x2ddbf8f5",
1183             "omap_digest": "0xf5fba2c6",
1184             "expected_object_size": 0,
1185             "expected_write_size": 0,
1186             "alloc_hint_flags": 0,
1187             "manifest": {
1188               "type": 0
1189             },
1190             "watchers": {}
1191           },
1192           "size": 9,
1193           "errors": [
1194             "size_mismatch_info",
1195             "obj_size_info_mismatch"
1196           ],
1197           "osd": 1,
1198           "primary": true
1199         }
1200       ],
1201       "selected_object_info": {
1202         "oid": {
1203           "oid": "ROBJ1",
1204           "key": "",
1205           "snapid": -2,
1206           "hash": 1454963827,
1207           "max": 0,
1208           "pool": 3,
1209           "namespace": ""
1210         },
1211         "version": "51'58",
1212         "prior_version": "21'3",
1213         "last_reqid": "osd.1.0:57",
1214         "user_version": 3,
1215         "size": 7,
1216         "mtime": "2018-04-05 14:33:19.804040",
1217         "local_mtime": "2018-04-05 14:33:19.804839",
1218         "lost": 0,
1219         "flags": [
1220           "dirty",
1221           "omap",
1222           "data_digest",
1223           "omap_digest"
1224         ],
1225         "truncate_seq": 0,
1226         "truncate_size": 0,
1227         "data_digest": "0x2ddbf8f5",
1228         "omap_digest": "0xf5fba2c6",
1229         "expected_object_size": 0,
1230         "expected_write_size": 0,
1231         "alloc_hint_flags": 0,
1232         "manifest": {
1233           "type": 0
1234         },
1235         "watchers": {}
1236       },
1237       "union_shard_errors": [
1238         "size_mismatch_info",
1239         "obj_size_info_mismatch"
1240       ],
1241       "errors": [
1242         "size_mismatch"
1243       ],
1244       "object": {
1245         "version": 3,
1246         "snap": "head",
1247         "locator": "",
1248         "nspace": "",
1249         "name": "ROBJ1"
1250       }
1251     },
1252     {
1253       "shards": [
1254         {
1255           "errors": [
1256             "stat_error"
1257           ],
1258           "osd": 0,
1259           "primary": false
1260         },
1261         {
1262           "size": 7,
1263           "errors": [],
1264           "osd": 1,
1265           "primary": true
1266         }
1267       ],
1268       "selected_object_info": {
1269         "oid": {
1270           "oid": "ROBJ12",
1271           "key": "",
1272           "snapid": -2,
1273           "hash": 3920199997,
1274           "max": 0,
1275           "pool": 3,
1276           "namespace": ""
1277         },
1278         "version": "51'56",
1279         "prior_version": "43'36",
1280         "last_reqid": "osd.1.0:55",
1281         "user_version": 36,
1282         "size": 7,
1283         "mtime": "",
1284         "local_mtime": "",
1285         "lost": 0,
1286         "flags": [
1287           "dirty",
1288           "omap",
1289           "data_digest",
1290           "omap_digest"
1291         ],
1292         "truncate_seq": 0,
1293         "truncate_size": 0,
1294         "data_digest": "0x2ddbf8f5",
1295         "omap_digest": "0x067f306a",
1296         "expected_object_size": 0,
1297         "expected_write_size": 0,
1298         "alloc_hint_flags": 0,
1299         "manifest": {
1300           "type": 0
1301         },
1302         "watchers": {}
1303       },
1304       "union_shard_errors": [
1305         "stat_error"
1306       ],
1307       "errors": [],
1308       "object": {
1309         "version": 36,
1310         "snap": "head",
1311         "locator": "",
1312         "nspace": "",
1313         "name": "ROBJ12"
1314       }
1315     },
1316     {
1317       "shards": [
1318         {
1319           "errors": [
1320             "stat_error"
1321           ],
1322           "osd": 0,
1323           "primary": false
1324         },
1325         {
1326           "size": 7,
1327           "errors": [],
1328           "osd": 1,
1329           "primary": true
1330         }
1331       ],
1332       "selected_object_info": {
1333         "oid": {
1334           "oid": "ROBJ13",
1335           "key": "",
1336           "snapid": -2,
1337           "hash": 2682806379,
1338           "max": 0,
1339           "pool": 3,
1340           "namespace": ""
1341         },
1342         "version": "51'59",
1343         "prior_version": "45'39",
1344         "last_reqid": "osd.1.0:58",
1345         "user_version": 39,
1346         "size": 7,
1347         "mtime": "",
1348         "local_mtime": "",
1349         "lost": 0,
1350         "flags": [
1351           "dirty",
1352           "omap",
1353           "data_digest",
1354           "omap_digest"
1355         ],
1356         "truncate_seq": 0,
1357         "truncate_size": 0,
1358         "data_digest": "0x2ddbf8f5",
1359         "omap_digest": "0x6441854d",
1360         "expected_object_size": 0,
1361         "expected_write_size": 0,
1362         "alloc_hint_flags": 0,
1363         "manifest": {
1364           "type": 0
1365         },
1366         "watchers": {}
1367       },
1368       "union_shard_errors": [
1369         "stat_error"
1370       ],
1371       "errors": [],
1372       "object": {
1373         "version": 39,
1374         "snap": "head",
1375         "locator": "",
1376         "nspace": "",
1377         "name": "ROBJ13"
1378       }
1379     },
1380     {
1381       "shards": [
1382         {
1383           "object_info": "bad-val",
1384           "size": 7,
1385           "errors": [
1386             "info_corrupted"
1387           ],
1388           "osd": 0,
1389           "primary": false
1390         },
1391         {
1392           "size": 7,
1393           "errors": [
1394             "info_missing"
1395           ],
1396           "osd": 1,
1397           "primary": true
1398         }
1399       ],
1400       "union_shard_errors": [
1401         "info_missing",
1402         "info_corrupted"
1403       ],
1404       "errors": [],
1405       "object": {
1406         "version": 0,
1407         "snap": "head",
1408         "locator": "",
1409         "nspace": "",
1410         "name": "ROBJ14"
1411       }
1412     },
1413     {
1414       "shards": [
1415         {
1416           "object_info": {
1417             "oid": {
1418               "oid": "ROBJ15",
1419               "key": "",
1420               "snapid": -2,
1421               "hash": 504996876,
1422               "max": 0,
1423               "pool": 3,
1424               "namespace": ""
1425             },
1426             "version": "51'49",
1427             "prior_version": "49'45",
1428             "last_reqid": "osd.1.0:48",
1429             "user_version": 45,
1430             "size": 7,
1431             "mtime": "2018-04-05 14:33:29.498969",
1432             "local_mtime": "2018-04-05 14:33:29.499890",
1433             "lost": 0,
1434             "flags": [
1435               "dirty",
1436               "omap",
1437               "data_digest",
1438               "omap_digest"
1439             ],
1440             "truncate_seq": 0,
1441             "truncate_size": 0,
1442             "data_digest": "0x2ddbf8f5",
1443             "omap_digest": "0x2d2a4d6e",
1444             "expected_object_size": 0,
1445             "expected_write_size": 0,
1446             "alloc_hint_flags": 0,
1447             "manifest": {
1448               "type": 0
1449             },
1450             "watchers": {}
1451           },
1452           "size": 7,
1453           "errors": [],
1454           "osd": 0,
1455           "primary": false
1456         },
1457         {
1458           "size": 7,
1459           "errors": [
1460             "info_missing"
1461           ],
1462           "osd": 1,
1463           "primary": true
1464         }
1465       ],
1466       "selected_object_info": {
1467         "oid": {
1468           "oid": "ROBJ15",
1469           "key": "",
1470           "snapid": -2,
1471           "hash": 504996876,
1472           "max": 0,
1473           "pool": 3,
1474           "namespace": ""
1475         },
1476         "version": "51'49",
1477         "prior_version": "49'45",
1478         "last_reqid": "osd.1.0:48",
1479         "user_version": 45,
1480         "size": 7,
1481         "mtime": "",
1482         "local_mtime": "",
1483         "lost": 0,
1484         "flags": [
1485           "dirty",
1486           "omap",
1487           "data_digest",
1488           "omap_digest"
1489         ],
1490         "truncate_seq": 0,
1491         "truncate_size": 0,
1492         "data_digest": "0x2ddbf8f5",
1493         "omap_digest": "0x2d2a4d6e",
1494         "expected_object_size": 0,
1495         "expected_write_size": 0,
1496         "alloc_hint_flags": 0,
1497         "manifest": {
1498           "type": 0
1499         },
1500         "watchers": {}
1501       },
1502       "union_shard_errors": [
1503         "info_missing"
1504       ],
1505       "errors": [],
1506       "object": {
1507         "version": 45,
1508         "snap": "head",
1509         "locator": "",
1510         "nspace": "",
1511         "name": "ROBJ15"
1512       }
1513     },
1514     {
1515       "errors": [],
1516       "object": {
1517       "locator": "",
1518       "name": "ROBJ16",
1519       "nspace": "",
1520       "snap": "head",
1521       "version": 0
1522        },
1523         "shards": [
1524       {
1525         "errors": [
1526           "snapset_missing"
1527         ],
1528         "osd": 0,
1529         "primary": false,
1530         "size": 7
1531       },
1532       {
1533         "errors": [
1534           "snapset_corrupted"
1535         ],
1536         "osd": 1,
1537         "primary": true,
1538         "snapset": "bad-val",
1539         "size": 7
1540       }
1541       ],
1542       "union_shard_errors": [
1543         "snapset_missing",
1544         "snapset_corrupted"
1545       ]
1546     },
1547     {
1548      "errors": [
1549        "object_info_inconsistency"
1550      ],
1551      "object": {
1552        "locator": "",
1553        "name": "ROBJ18",
1554        "nspace": "",
1555        "snap": "head"
1556      },
1557      "selected_object_info": {
1558        "alloc_hint_flags": 255,
1559        "data_digest": "0x2ddbf8f5",
1560        "expected_object_size": 0,
1561        "expected_write_size": 0,
1562        "flags": [
1563          "dirty",
1564          "omap",
1565          "data_digest",
1566          "omap_digest"
1567        ],
1568        "lost": 0,
1569        "manifest": {
1570          "type": 0
1571        },
1572        "oid": {
1573          "hash": 1629828556,
1574          "key": "",
1575          "max": 0,
1576          "namespace": "",
1577          "oid": "ROBJ18",
1578          "pool": 3,
1579          "snapid": -2
1580        },
1581        "omap_digest": "0xddc3680f",
1582        "size": 7,
1583        "truncate_seq": 0,
1584        "truncate_size": 0,
1585        "user_version": 54,
1586        "watchers": {}
1587      },
1588      "shards": [
1589        {
1590          "errors": [],
1591          "object_info": {
1592            "alloc_hint_flags": 0,
1593            "data_digest": "0x2ddbf8f5",
1594            "expected_object_size": 0,
1595            "expected_write_size": 0,
1596            "flags": [
1597              "dirty",
1598              "omap",
1599              "data_digest",
1600              "omap_digest"
1601            ],
1602            "lost": 0,
1603            "manifest": {
1604              "type": 0
1605            },
1606            "oid": {
1607              "hash": 1629828556,
1608              "key": "",
1609              "max": 0,
1610              "namespace": "",
1611              "oid": "ROBJ18",
1612              "pool": 3,
1613              "snapid": -2
1614            },
1615            "omap_digest": "0xddc3680f",
1616            "size": 7,
1617            "truncate_seq": 0,
1618            "truncate_size": 0,
1619            "user_version": 54,
1620            "watchers": {}
1621          },
1622          "osd": 0,
1623          "primary": false,
1624          "size": 7
1625        },
1626        {
1627          "errors": [],
1628          "object_info": {
1629            "alloc_hint_flags": 255,
1630            "data_digest": "0x2ddbf8f5",
1631            "expected_object_size": 0,
1632            "expected_write_size": 0,
1633            "flags": [
1634              "dirty",
1635              "omap",
1636              "data_digest",
1637              "omap_digest"
1638            ],
1639            "lost": 0,
1640            "manifest": {
1641              "type": 0
1642            },
1643            "oid": {
1644              "hash": 1629828556,
1645              "key": "",
1646              "max": 0,
1647              "namespace": "",
1648              "oid": "ROBJ18",
1649              "pool": 3,
1650              "snapid": -2
1651            },
1652            "omap_digest": "0xddc3680f",
1653            "size": 7,
1654            "truncate_seq": 0,
1655            "truncate_size": 0,
1656            "user_version": 54,
1657            "watchers": {}
1658          },
1659          "osd": 1,
1660          "primary": true,
1661          "size": 7
1662        }
1663      ],
1664      "union_shard_errors": []
1665    },
1666    {
1667       "object": {
1668         "name": "ROBJ19",
1669         "nspace": "",
1670         "locator": "",
1671         "snap": "head",
1672         "version": 58
1673       },
1674       "errors": [
1675         "size_too_large"
1676       ],
1677       "union_shard_errors": [],
1678       "selected_object_info": {
1679         "oid": {
1680           "oid": "ROBJ19",
1681           "key": "",
1682           "snapid": -2,
1683           "hash": 2868534344,
1684           "max": 0,
1685           "pool": 3,
1686           "namespace": ""
1687         },
1688         "version": "63'59",
1689         "prior_version": "63'58",
1690         "last_reqid": "osd.1.0:58",
1691         "user_version": 58,
1692         "size": 1049600,
1693         "mtime": "2019-08-09T23:33:58.340709+0000",
1694         "local_mtime": "2019-08-09T23:33:58.345676+0000",
1695         "lost": 0,
1696         "flags": [
1697           "dirty",
1698           "omap",
1699           "data_digest",
1700           "omap_digest"
1701         ],
1702         "truncate_seq": 0,
1703         "truncate_size": 0,
1704         "data_digest": "0x3dde0ef3",
1705         "omap_digest": "0xbffddd28",
1706         "expected_object_size": 0,
1707         "expected_write_size": 0,
1708         "alloc_hint_flags": 0,
1709         "manifest": {
1710           "type": 0
1711         },
1712         "watchers": {}
1713       },
1714       "shards": [
1715         {
1716           "osd": 0,
1717           "primary": false,
1718           "errors": [],
1719           "size": 1049600
1720         },
1721         {
1722           "osd": 1,
1723           "primary": true,
1724           "errors": [],
1725           "size": 1049600
1726         }
1727       ]
1728    },
1729    {
1730       "shards": [
1731         {
1732           "size": 7,
1733           "errors": [],
1734           "osd": 0,
1735           "primary": false
1736         },
1737         {
1738           "errors": [
1739             "missing"
1740           ],
1741           "osd": 1,
1742           "primary": true
1743         }
1744       ],
1745       "selected_object_info": {
1746         "oid": {
1747           "oid": "ROBJ3",
1748           "key": "",
1749           "snapid": -2,
1750           "hash": 625845583,
1751           "max": 0,
1752           "pool": 3,
1753           "namespace": ""
1754         },
1755         "version": "51'61",
1756         "prior_version": "25'9",
1757         "last_reqid": "osd.1.0:60",
1758         "user_version": 9,
1759         "size": 7,
1760         "mtime": "",
1761         "local_mtime": "",
1762         "lost": 0,
1763         "flags": [
1764           "dirty",
1765           "omap",
1766           "data_digest",
1767           "omap_digest"
1768         ],
1769         "truncate_seq": 0,
1770         "truncate_size": 0,
1771         "data_digest": "0x2ddbf8f5",
1772         "omap_digest": "0x00b35dfd",
1773         "expected_object_size": 0,
1774         "expected_write_size": 0,
1775         "alloc_hint_flags": 0,
1776         "manifest": {
1777           "type": 0
1778         },
1779         "watchers": {}
1780       },
1781       "union_shard_errors": [
1782         "missing"
1783       ],
1784       "errors": [],
1785       "object": {
1786         "version": 9,
1787         "snap": "head",
1788         "locator": "",
1789         "nspace": "",
1790         "name": "ROBJ3"
1791       }
1792     },
1793     {
1794       "shards": [
1795         {
1796           "attrs": [
1797             {
1798               "Base64": false,
1799               "value": "bad-val",
1800               "name": "key1-ROBJ8"
1801             },
1802             {
1803               "Base64": false,
1804               "value": "val2-ROBJ8",
1805               "name": "key2-ROBJ8"
1806             }
1807           ],
1808           "size": 7,
1809           "errors": [],
1810           "osd": 0,
1811           "primary": false
1812         },
1813         {
1814           "attrs": [
1815             {
1816               "Base64": false,
1817               "value": "val1-ROBJ8",
1818               "name": "key1-ROBJ8"
1819             },
1820             {
1821               "Base64": false,
1822               "value": "val3-ROBJ8",
1823               "name": "key3-ROBJ8"
1824             }
1825           ],
1826           "size": 7,
1827           "errors": [],
1828           "osd": 1,
1829           "primary": true
1830         }
1831       ],
1832       "selected_object_info": {
1833         "oid": {
1834           "oid": "ROBJ8",
1835           "key": "",
1836           "snapid": -2,
1837           "hash": 2359695969,
1838           "max": 0,
1839           "pool": 3,
1840           "namespace": ""
1841         },
1842         "version": "79'66",
1843         "prior_version": "79'65",
1844         "last_reqid": "client.4554.0:1",
1845         "user_version": 79,
1846         "size": 7,
1847         "mtime": "",
1848         "local_mtime": "",
1849         "lost": 0,
1850         "flags": [
1851           "dirty",
1852           "omap",
1853           "data_digest",
1854           "omap_digest"
1855         ],
1856         "truncate_seq": 0,
1857         "truncate_size": 0,
1858         "data_digest": "0x2ddbf8f5",
1859         "omap_digest": "0xd6be81dc",
1860         "expected_object_size": 0,
1861         "expected_write_size": 0,
1862         "alloc_hint_flags": 0,
1863         "manifest": {
1864           "type": 0
1865         },
1866         "watchers": {}
1867       },
1868       "union_shard_errors": [],
1869       "errors": [
1870         "attr_value_mismatch",
1871         "attr_name_mismatch"
1872       ],
1873       "object": {
1874         "version": 66,
1875         "snap": "head",
1876         "locator": "",
1877         "nspace": "",
1878         "name": "ROBJ8"
1879       }
1880     },
1881     {
1882       "shards": [
1883         {
1884           "object_info": {
1885             "oid": {
1886               "oid": "ROBJ9",
1887               "key": "",
1888               "snapid": -2,
1889               "hash": 537189375,
1890               "max": 0,
1891               "pool": 3,
1892               "namespace": ""
1893             },
1894             "version": "95'67",
1895             "prior_version": "51'64",
1896             "last_reqid": "client.4649.0:1",
1897             "user_version": 80,
1898             "size": 1,
1899             "mtime": "",
1900             "local_mtime": "",
1901             "lost": 0,
1902             "flags": [
1903               "dirty",
1904               "omap",
1905               "data_digest",
1906               "omap_digest"
1907             ],
1908             "truncate_seq": 0,
1909             "truncate_size": 0,
1910             "data_digest": "0x2b63260d",
1911             "omap_digest": "0x2eecc539",
1912             "expected_object_size": 0,
1913             "expected_write_size": 0,
1914             "alloc_hint_flags": 0,
1915             "manifest": {
1916               "type": 0
1917             },
1918             "watchers": {}
1919           },
1920           "size": 1,
1921           "errors": [],
1922           "osd": 0,
1923           "primary": false
1924         },
1925         {
1926           "object_info": {
1927             "oid": {
1928               "oid": "ROBJ9",
1929               "key": "",
1930               "snapid": -2,
1931               "hash": 537189375,
1932               "max": 0,
1933               "pool": 3,
1934               "namespace": ""
1935             },
1936             "version": "51'64",
1937             "prior_version": "37'27",
1938             "last_reqid": "osd.1.0:63",
1939             "user_version": 27,
1940             "size": 7,
1941             "mtime": "2018-04-05 14:33:25.352485",
1942             "local_mtime": "2018-04-05 14:33:25.353746",
1943             "lost": 0,
1944             "flags": [
1945               "dirty",
1946               "omap",
1947               "data_digest",
1948               "omap_digest"
1949             ],
1950             "truncate_seq": 0,
1951             "truncate_size": 0,
1952             "data_digest": "0x2ddbf8f5",
1953             "omap_digest": "0x2eecc539",
1954             "expected_object_size": 0,
1955             "expected_write_size": 0,
1956             "alloc_hint_flags": 0,
1957             "manifest": {
1958               "type": 0
1959             },
1960             "watchers": {}
1961           },
1962           "size": 1,
1963           "errors": [
1964             "obj_size_info_mismatch"
1965           ],
1966           "osd": 1,
1967           "primary": true
1968         }
1969       ],
1970       "selected_object_info": {
1971         "oid": {
1972           "oid": "ROBJ9",
1973           "key": "",
1974           "snapid": -2,
1975           "hash": 537189375,
1976           "max": 0,
1977           "pool": 3,
1978           "namespace": ""
1979         },
1980         "version": "95'67",
1981         "prior_version": "51'64",
1982         "last_reqid": "client.4649.0:1",
1983         "user_version": 80,
1984         "size": 1,
1985         "mtime": "",
1986         "local_mtime": "",
1987         "lost": 0,
1988         "flags": [
1989           "dirty",
1990           "omap",
1991           "data_digest",
1992           "omap_digest"
1993         ],
1994         "truncate_seq": 0,
1995         "truncate_size": 0,
1996         "data_digest": "0x2b63260d",
1997         "omap_digest": "0x2eecc539",
1998         "expected_object_size": 0,
1999         "expected_write_size": 0,
2000         "alloc_hint_flags": 0,
2001         "manifest": {
2002           "type": 0
2003         },
2004         "watchers": {}
2005       },
2006       "union_shard_errors": [
2007          "obj_size_info_mismatch"
2008       ],
2009       "errors": [
2010         "object_info_inconsistency"
2011       ],
2012       "object": {
2013         "version": 67,
2014         "snap": "head",
2015         "locator": "",
2016         "nspace": "",
2017         "name": "ROBJ9"
2018       }
2019     }
2020   ],
2021   "epoch": 0
2022 }
2023 EOF
2024
2025     jq "$jqfilter" $dir/json | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/csjson
2026     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
2027     if test $getjson = "yes"
2028     then
2029         jq '.' $dir/json > save1.json
2030     fi
2031
2032     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
2033     then
2034       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
2035     fi
2036
2037     objname=ROBJ9
2038     # Change data and size again because digest was recomputed
2039     echo -n ZZZ > $dir/change
2040     rados --pool $poolname put $objname $dir/change
2041     # Set one to an even older value
2042     objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi
2043     rm $dir/oi $dir/change
2044
2045     objname=ROBJ10
2046     objectstore_tool $dir 1 $objname get-attr _ > $dir/oi
2047     rados --pool $poolname setomapval $objname key2-$objname val2-$objname
2048     objectstore_tool $dir 0 $objname set-attr _ $dir/oi
2049     objectstore_tool $dir 1 $objname set-attr _ $dir/oi
2050     rm $dir/oi
2051
2052     inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
2053     inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
2054     inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
2055     inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
2056
2057     # ROBJ19 won't error this time
2058     ceph tell osd.\* injectargs -- --osd-max-object-size=134217728
2059
2060     pg_deep_scrub $pg
2061
2062     err_strings=()
2063     err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:30259878:::ROBJ15:head : candidate had a missing info key"
2064     err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:33aca486:::ROBJ18:head : data_digest 0xbd89c912 != data_digest 0x2ddbf8f5 from auth oi 3:33aca486:::ROBJ18:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 54 dd 2ddbf8f5 od ddc3680f alloc_hint [[]0 0 255[]][)], object info inconsistent "
2065     err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:33aca486:::ROBJ18:head : data_digest 0xbd89c912 != data_digest 0x2ddbf8f5 from auth oi 3:33aca486:::ROBJ18:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 54 dd 2ddbf8f5 od ddc3680f alloc_hint [[]0 0 255[]][)]"
2066     err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:33aca486:::ROBJ18:head : failed to pick suitable auth object"
2067     err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:5c7b2c47:::ROBJ16:head : candidate had a corrupt snapset"
2068     err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:5c7b2c47:::ROBJ16:head : candidate had a missing snapset key"
2069     err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:5c7b2c47:::ROBJ16:head : failed to pick suitable object info"
2070     err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:86586531:::ROBJ8:head : attr value mismatch '_key1-ROBJ8', attr name mismatch '_key3-ROBJ8', attr name mismatch '_key2-ROBJ8'"
2071     err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:87abbf36:::ROBJ11:head : candidate had a read error"
2072     err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:8aa5320e:::ROBJ17:head : data_digest 0x5af0c3ef != data_digest 0x2ddbf8f5 from auth oi 3:8aa5320e:::ROBJ17:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 51 dd 2ddbf8f5 od e9572720 alloc_hint [[]0 0 0[]][)]"
2073     err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:8aa5320e:::ROBJ17:head : data_digest 0x5af0c3ef != data_digest 0x2ddbf8f5 from auth oi 3:8aa5320e:::ROBJ17:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 51 dd 2ddbf8f5 od e9572720 alloc_hint [[]0 0 0[]][)]"
2074     err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:8aa5320e:::ROBJ17:head : failed to pick suitable auth object"
2075     err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:8b55fa4b:::ROBJ7:head : omap_digest 0xefced57a != omap_digest 0x6a73cc07 from shard 1"
2076     err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:8b55fa4b:::ROBJ7:head : omap_digest 0x6a73cc07 != omap_digest 0xefced57a from auth oi 3:8b55fa4b:::ROBJ7:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [[]0 0 0[]][)]"
2077     err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:a53c12e8:::ROBJ6:head : omap_digest 0x689ee887 != omap_digest 0x179c919f from shard 1, omap_digest 0x689ee887 != omap_digest 0x179c919f from auth oi 3:a53c12e8:::ROBJ6:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [[]0 0 0[]][)]"
2078     err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:b1f19cbd:::ROBJ10:head : omap_digest 0xa8dd5adc != omap_digest 0xc2025a24 from auth oi 3:b1f19cbd:::ROBJ10:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [[]0 0 0[]][)]"
2079     err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:b1f19cbd:::ROBJ10:head : omap_digest 0xa8dd5adc != omap_digest 0xc2025a24 from auth oi 3:b1f19cbd:::ROBJ10:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [[]0 0 0[]][)]"
2080     err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:b1f19cbd:::ROBJ10:head : failed to pick suitable auth object"
2081     err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:bc819597:::ROBJ12:head : candidate had a stat error"
2082     err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:c0c86b1d:::ROBJ14:head : candidate had a missing info key"
2083     err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:c0c86b1d:::ROBJ14:head : candidate had a corrupt info"
2084     err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:c0c86b1d:::ROBJ14:head : failed to pick suitable object info"
2085     err_strings[22]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : candidate size 9 info size 7 mismatch"
2086     err_strings[23]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : data_digest 0x2d4a11c2 != data_digest 0x2ddbf8f5 from shard 0, data_digest 0x2d4a11c2 != data_digest 0x2ddbf8f5 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from shard 0"
2087     err_strings[24]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:d60617f9:::ROBJ13:head : candidate had a read error"
2088     err_strings[25]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:d60617f9:::ROBJ13:head : candidate had a stat error"
2089     err_strings[26]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:d60617f9:::ROBJ13:head : failed to pick suitable object info"
2090     err_strings[27]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:e97ce31e:::ROBJ2:head : data_digest 0x578a4830 != data_digest 0x2ddbf8f5 from shard 1, data_digest 0x578a4830 != data_digest 0x2ddbf8f5 from auth oi 3:e97ce31e:::ROBJ2:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [[]0 0 0[]][)]"
2091     err_strings[28]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 3:f2a5b2a4:::ROBJ3:head : missing"
2092     err_strings[29]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:f4981d31:::ROBJ4:head : omap_digest 0xd7178dfe != omap_digest 0xe2d46ea4 from shard 1, omap_digest 0xd7178dfe != omap_digest 0xe2d46ea4 from auth oi 3:f4981d31:::ROBJ4:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [[]0 0 0[]][)]"
2093     err_strings[30]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:f4bfd4d1:::ROBJ5:head : omap_digest 0x1a862a41 != omap_digest 0x6cac8f6 from shard 1"
2094     err_strings[31]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:f4bfd4d1:::ROBJ5:head : omap_digest 0x6cac8f6 != omap_digest 0x1a862a41 from auth oi 3:f4bfd4d1:::ROBJ5:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [[]0 0 0[]][)]"
2095     err_strings[32]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:ffdb2004:::ROBJ9:head : candidate size 3 info size 7 mismatch"
2096     err_strings[33]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:ffdb2004:::ROBJ9:head : object info inconsistent "
2097     err_strings[34]="log_channel[(]cluster[)] log [[]ERR[]] : deep-scrub [0-9]*[.]0 3:c0c86b1d:::ROBJ14:head : no '_' attr"
2098     err_strings[35]="log_channel[(]cluster[)] log [[]ERR[]] : deep-scrub [0-9]*[.]0 3:5c7b2c47:::ROBJ16:head : can't decode 'snapset' attr buffer::malformed_input: .* no longer understand old encoding version 3 < 97"
2099     err_strings[36]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub : stat mismatch, got 19/19 objects, 0/0 clones, 18/19 dirty, 18/19 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 1049715/1049716 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
2100     err_strings[37]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub 1 missing, 11 inconsistent objects"
2101     err_strings[38]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub 35 errors"
2102
2103     for err_string in "${err_strings[@]}"
2104     do
2105         if ! grep -q "$err_string" $dir/osd.${primary}.log
2106         then
2107             echo "Missing log message '$err_string'"
2108             ERRORS=$(expr $ERRORS + 1)
2109         fi
2110     done
2111
2112     rados list-inconsistent-pg $poolname > $dir/json || return 1
2113     # Check pg count
2114     test $(jq '. | length' $dir/json) = "1" || return 1
2115     # Check pgid
2116     test $(jq -r '.[0]' $dir/json) = $pg || return 1
2117
2118     rados list-inconsistent-obj $pg > $dir/json || return 1
2119     # Get epoch for repair-get requests
2120     epoch=$(jq .epoch $dir/json)
2121
2122     jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
2123 {
2124   "inconsistents": [
2125     {
2126       "shards": [
2127         {
2128           "data_digest": "0x2ddbf8f5",
2129           "omap_digest": "0xf5fba2c6",
2130           "size": 7,
2131           "errors": [],
2132           "osd": 0,
2133           "primary": false
2134         },
2135         {
2136           "object_info": {
2137             "oid": {
2138               "oid": "ROBJ1",
2139               "key": "",
2140               "snapid": -2,
2141               "hash": 1454963827,
2142               "max": 0,
2143               "pool": 3,
2144               "namespace": ""
2145             },
2146             "version": "51'58",
2147             "prior_version": "21'3",
2148             "last_reqid": "osd.1.0:57",
2149             "user_version": 3,
2150             "size": 7,
2151             "mtime": "2018-04-05 14:33:19.804040",
2152             "local_mtime": "2018-04-05 14:33:19.804839",
2153             "lost": 0,
2154             "flags": [
2155               "dirty",
2156               "omap",
2157               "data_digest",
2158               "omap_digest"
2159             ],
2160             "truncate_seq": 0,
2161             "truncate_size": 0,
2162             "data_digest": "0x2ddbf8f5",
2163             "omap_digest": "0xf5fba2c6",
2164             "expected_object_size": 0,
2165             "expected_write_size": 0,
2166             "alloc_hint_flags": 0,
2167             "manifest": {
2168               "type": 0
2169             },
2170             "watchers": {}
2171           },
2172           "data_digest": "0x2d4a11c2",
2173           "omap_digest": "0xf5fba2c6",
2174           "size": 9,
2175           "errors": [
2176             "data_digest_mismatch_info",
2177             "size_mismatch_info",
2178             "obj_size_info_mismatch"
2179           ],
2180           "osd": 1,
2181           "primary": true
2182         }
2183       ],
2184       "selected_object_info": {
2185         "oid": {
2186           "oid": "ROBJ1",
2187           "key": "",
2188           "snapid": -2,
2189           "hash": 1454963827,
2190           "max": 0,
2191           "pool": 3,
2192           "namespace": ""
2193         },
2194         "version": "51'58",
2195         "prior_version": "21'3",
2196         "last_reqid": "osd.1.0:57",
2197         "user_version": 3,
2198         "size": 7,
2199         "mtime": "2018-04-05 14:33:19.804040",
2200         "local_mtime": "2018-04-05 14:33:19.804839",
2201         "lost": 0,
2202         "flags": [
2203           "dirty",
2204           "omap",
2205           "data_digest",
2206           "omap_digest"
2207         ],
2208         "truncate_seq": 0,
2209         "truncate_size": 0,
2210         "data_digest": "0x2ddbf8f5",
2211         "omap_digest": "0xf5fba2c6",
2212         "expected_object_size": 0,
2213         "expected_write_size": 0,
2214         "alloc_hint_flags": 0,
2215         "manifest": {
2216           "type": 0
2217         },
2218         "watchers": {}
2219       },
2220       "union_shard_errors": [
2221         "data_digest_mismatch_info",
2222         "size_mismatch_info",
2223         "obj_size_info_mismatch"
2224       ],
2225       "errors": [
2226         "data_digest_mismatch",
2227         "size_mismatch"
2228       ],
2229       "object": {
2230         "version": 3,
2231         "snap": "head",
2232         "locator": "",
2233         "nspace": "",
2234         "name": "ROBJ1"
2235       }
2236     },
2237     {
2238       "shards": [
2239         {
2240           "data_digest": "0x2ddbf8f5",
2241           "omap_digest": "0xa8dd5adc",
2242           "size": 7,
2243           "errors": [
2244             "omap_digest_mismatch_info"
2245           ],
2246           "osd": 0,
2247           "primary": false
2248         },
2249         {
2250           "data_digest": "0x2ddbf8f5",
2251           "omap_digest": "0xa8dd5adc",
2252           "size": 7,
2253           "errors": [
2254             "omap_digest_mismatch_info"
2255           ],
2256           "osd": 1,
2257           "primary": true
2258         }
2259       ],
2260       "selected_object_info": {
2261         "alloc_hint_flags": 0,
2262         "data_digest": "0x2ddbf8f5",
2263         "expected_object_size": 0,
2264         "expected_write_size": 0,
2265         "flags": [
2266           "dirty",
2267           "omap",
2268           "data_digest",
2269           "omap_digest"
2270         ],
2271         "lost": 0,
2272         "manifest": {
2273           "type": 0
2274         },
2275         "oid": {
2276           "hash": 3174666125,
2277           "key": "",
2278           "max": 0,
2279           "namespace": "",
2280           "oid": "ROBJ10",
2281           "pool": 3,
2282           "snapid": -2
2283         },
2284         "omap_digest": "0xc2025a24",
2285         "size": 7,
2286         "truncate_seq": 0,
2287         "truncate_size": 0,
2288         "user_version": 30,
2289         "watchers": {}
2290       },
2291       "union_shard_errors": [
2292         "omap_digest_mismatch_info"
2293       ],
2294       "errors": [],
2295       "object": {
2296         "version": 30,
2297         "snap": "head",
2298         "locator": "",
2299         "nspace": "",
2300         "name": "ROBJ10"
2301       }
2302     },
2303     {
2304       "shards": [
2305         {
2306           "data_digest": "0x2ddbf8f5",
2307           "omap_digest": "0xa03cef03",
2308           "size": 7,
2309           "errors": [],
2310           "osd": 0,
2311           "primary": false
2312         },
2313         {
2314           "size": 7,
2315           "errors": [
2316             "read_error"
2317           ],
2318           "osd": 1,
2319           "primary": true
2320         }
2321       ],
2322       "selected_object_info": {
2323         "oid": {
2324           "oid": "ROBJ11",
2325           "key": "",
2326           "snapid": -2,
2327           "hash": 1828574689,
2328           "max": 0,
2329           "pool": 3,
2330           "namespace": ""
2331         },
2332         "version": "51'52",
2333         "prior_version": "41'33",
2334         "last_reqid": "osd.1.0:51",
2335         "user_version": 33,
2336         "size": 7,
2337         "mtime": "2018-04-05 14:33:26.761286",
2338         "local_mtime": "2018-04-05 14:33:26.762368",
2339         "lost": 0,
2340         "flags": [
2341           "dirty",
2342           "omap",
2343           "data_digest",
2344           "omap_digest"
2345         ],
2346         "truncate_seq": 0,
2347         "truncate_size": 0,
2348         "data_digest": "0x2ddbf8f5",
2349         "omap_digest": "0xa03cef03",
2350         "expected_object_size": 0,
2351         "expected_write_size": 0,
2352         "alloc_hint_flags": 0,
2353         "manifest": {
2354           "type": 0
2355         },
2356         "watchers": {}
2357       },
2358       "union_shard_errors": [
2359         "read_error"
2360       ],
2361       "errors": [],
2362       "object": {
2363         "version": 33,
2364         "snap": "head",
2365         "locator": "",
2366         "nspace": "",
2367         "name": "ROBJ11"
2368       }
2369     },
2370     {
2371       "shards": [
2372         {
2373           "errors": [
2374             "stat_error"
2375           ],
2376           "osd": 0,
2377           "primary": false
2378         },
2379         {
2380           "data_digest": "0x2ddbf8f5",
2381           "omap_digest": "0x067f306a",
2382           "size": 7,
2383           "errors": [],
2384           "osd": 1,
2385           "primary": true
2386         }
2387       ],
2388       "selected_object_info": {
2389         "oid": {
2390           "oid": "ROBJ12",
2391           "key": "",
2392           "snapid": -2,
2393           "hash": 3920199997,
2394           "max": 0,
2395           "pool": 3,
2396           "namespace": ""
2397         },
2398         "version": "51'56",
2399         "prior_version": "43'36",
2400         "last_reqid": "osd.1.0:55",
2401         "user_version": 36,
2402         "size": 7,
2403         "mtime": "2018-04-05 14:33:27.460958",
2404         "local_mtime": "2018-04-05 14:33:27.462109",
2405         "lost": 0,
2406         "flags": [
2407           "dirty",
2408           "omap",
2409           "data_digest",
2410           "omap_digest"
2411         ],
2412         "truncate_seq": 0,
2413         "truncate_size": 0,
2414         "data_digest": "0x2ddbf8f5",
2415         "omap_digest": "0x067f306a",
2416         "expected_object_size": 0,
2417         "expected_write_size": 0,
2418         "alloc_hint_flags": 0,
2419         "manifest": {
2420           "type": 0
2421         },
2422         "watchers": {}
2423       },
2424       "union_shard_errors": [
2425         "stat_error"
2426       ],
2427       "errors": [],
2428       "object": {
2429         "version": 36,
2430         "snap": "head",
2431         "locator": "",
2432         "nspace": "",
2433         "name": "ROBJ12"
2434       }
2435     },
2436     {
2437       "shards": [
2438         {
2439           "errors": [
2440             "stat_error"
2441           ],
2442           "osd": 0,
2443           "primary": false
2444         },
2445         {
2446           "size": 7,
2447           "errors": [
2448             "read_error"
2449           ],
2450           "osd": 1,
2451           "primary": true
2452         }
2453       ],
2454       "union_shard_errors": [
2455         "stat_error",
2456         "read_error"
2457       ],
2458       "errors": [],
2459       "object": {
2460         "version": 0,
2461         "snap": "head",
2462         "locator": "",
2463         "nspace": "",
2464         "name": "ROBJ13"
2465       }
2466     },
2467     {
2468       "shards": [
2469         {
2470           "object_info": "bad-val",
2471           "data_digest": "0x2ddbf8f5",
2472           "omap_digest": "0x4f14f849",
2473           "size": 7,
2474           "errors": [
2475             "info_corrupted"
2476           ],
2477           "osd": 0,
2478           "primary": false
2479         },
2480         {
2481           "data_digest": "0x2ddbf8f5",
2482           "omap_digest": "0x4f14f849",
2483           "size": 7,
2484           "errors": [
2485             "info_missing"
2486           ],
2487           "osd": 1,
2488           "primary": true
2489         }
2490       ],
2491       "union_shard_errors": [
2492         "info_missing",
2493         "info_corrupted"
2494       ],
2495       "errors": [],
2496       "object": {
2497         "version": 0,
2498         "snap": "head",
2499         "locator": "",
2500         "nspace": "",
2501         "name": "ROBJ14"
2502       }
2503     },
2504     {
2505       "shards": [
2506         {
2507           "object_info": {
2508             "oid": {
2509               "oid": "ROBJ15",
2510               "key": "",
2511               "snapid": -2,
2512               "hash": 504996876,
2513               "max": 0,
2514               "pool": 3,
2515               "namespace": ""
2516             },
2517             "version": "51'49",
2518             "prior_version": "49'45",
2519             "last_reqid": "osd.1.0:48",
2520             "user_version": 45,
2521             "size": 7,
2522             "mtime": "2018-04-05 14:33:29.498969",
2523             "local_mtime": "2018-04-05 14:33:29.499890",
2524             "lost": 0,
2525             "flags": [
2526               "dirty",
2527               "omap",
2528               "data_digest",
2529               "omap_digest"
2530             ],
2531             "truncate_seq": 0,
2532             "truncate_size": 0,
2533             "data_digest": "0x2ddbf8f5",
2534             "omap_digest": "0x2d2a4d6e",
2535             "expected_object_size": 0,
2536             "expected_write_size": 0,
2537             "alloc_hint_flags": 0,
2538             "manifest": {
2539               "type": 0
2540             },
2541             "watchers": {}
2542           },
2543           "data_digest": "0x2ddbf8f5",
2544           "omap_digest": "0x2d2a4d6e",
2545           "size": 7,
2546           "errors": [],
2547           "osd": 0,
2548           "primary": false
2549         },
2550         {
2551           "data_digest": "0x2ddbf8f5",
2552           "omap_digest": "0x2d2a4d6e",
2553           "size": 7,
2554           "errors": [
2555             "info_missing"
2556           ],
2557           "osd": 1,
2558           "primary": true
2559         }
2560       ],
2561       "selected_object_info": {
2562         "oid": {
2563           "oid": "ROBJ15",
2564           "key": "",
2565           "snapid": -2,
2566           "hash": 504996876,
2567           "max": 0,
2568           "pool": 3,
2569           "namespace": ""
2570         },
2571         "version": "51'49",
2572         "prior_version": "49'45",
2573         "last_reqid": "osd.1.0:48",
2574         "user_version": 45,
2575         "size": 7,
2576         "mtime": "2018-04-05 14:33:29.498969",
2577         "local_mtime": "2018-04-05 14:33:29.499890",
2578         "lost": 0,
2579         "flags": [
2580           "dirty",
2581           "omap",
2582           "data_digest",
2583           "omap_digest"
2584         ],
2585         "truncate_seq": 0,
2586         "truncate_size": 0,
2587         "data_digest": "0x2ddbf8f5",
2588         "omap_digest": "0x2d2a4d6e",
2589         "expected_object_size": 0,
2590         "expected_write_size": 0,
2591         "alloc_hint_flags": 0,
2592         "manifest": {
2593           "type": 0
2594         },
2595         "watchers": {}
2596       },
2597       "union_shard_errors": [
2598         "info_missing"
2599       ],
2600       "errors": [],
2601       "object": {
2602         "version": 45,
2603         "snap": "head",
2604         "locator": "",
2605         "nspace": "",
2606         "name": "ROBJ15"
2607       }
2608     },
2609     {
2610       "errors": [],
2611       "object": {
2612       "locator": "",
2613       "name": "ROBJ16",
2614       "nspace": "",
2615       "snap": "head",
2616       "version": 0
2617        },
2618         "shards": [
2619       {
2620         "data_digest": "0x2ddbf8f5",
2621         "errors": [
2622           "snapset_missing"
2623         ],
2624         "omap_digest": "0x8b699207",
2625         "osd": 0,
2626         "primary": false,
2627         "size": 7
2628       },
2629       {
2630         "snapset": "bad-val",
2631         "data_digest": "0x2ddbf8f5",
2632         "errors": [
2633           "snapset_corrupted"
2634         ],
2635         "omap_digest": "0x8b699207",
2636         "osd": 1,
2637         "primary": true,
2638         "size": 7
2639       }
2640       ],
2641       "union_shard_errors": [
2642         "snapset_missing",
2643         "snapset_corrupted"
2644       ]
2645     },
2646     {
2647      "errors": [],
2648      "object": {
2649        "locator": "",
2650        "name": "ROBJ17",
2651        "nspace": "",
2652        "snap": "head"
2653      },
2654      "selected_object_info": {
2655        "alloc_hint_flags": 0,
2656        "data_digest": "0x2ddbf8f5",
2657        "expected_object_size": 0,
2658        "expected_write_size": 0,
2659        "flags": [
2660          "dirty",
2661          "omap",
2662          "data_digest",
2663          "omap_digest"
2664        ],
2665        "lost": 0,
2666        "manifest": {
2667          "type": 0
2668        },
2669        "oid": {
2670          "hash": 1884071249,
2671          "key": "",
2672          "max": 0,
2673          "namespace": "",
2674          "oid": "ROBJ17",
2675          "pool": 3,
2676          "snapid": -2
2677        },
2678        "omap_digest": "0xe9572720",
2679        "size": 7,
2680        "truncate_seq": 0,
2681        "truncate_size": 0,
2682        "user_version": 51,
2683        "watchers": {}
2684      },
2685      "shards": [
2686        {
2687          "data_digest": "0x5af0c3ef",
2688          "errors": [
2689            "data_digest_mismatch_info"
2690          ],
2691          "omap_digest": "0xe9572720",
2692          "osd": 0,
2693          "primary": false,
2694          "size": 7
2695        },
2696        {
2697          "data_digest": "0x5af0c3ef",
2698          "errors": [
2699            "data_digest_mismatch_info"
2700          ],
2701          "omap_digest": "0xe9572720",
2702          "osd": 1,
2703          "primary": true,
2704          "size": 7
2705        }
2706      ],
2707      "union_shard_errors": [
2708        "data_digest_mismatch_info"
2709      ]
2710    },
2711    {
2712      "errors": [
2713        "object_info_inconsistency"
2714      ],
2715      "object": {
2716        "locator": "",
2717        "name": "ROBJ18",
2718        "nspace": "",
2719        "snap": "head"
2720      },
2721      "selected_object_info": {
2722        "alloc_hint_flags": 255,
2723        "data_digest": "0x2ddbf8f5",
2724        "expected_object_size": 0,
2725        "expected_write_size": 0,
2726        "flags": [
2727          "dirty",
2728          "omap",
2729          "data_digest",
2730          "omap_digest"
2731        ],
2732        "lost": 0,
2733        "manifest": {
2734          "type": 0
2735        },
2736        "oid": {
2737          "hash": 1629828556,
2738          "key": "",
2739          "max": 0,
2740          "namespace": "",
2741          "oid": "ROBJ18",
2742          "pool": 3,
2743          "snapid": -2
2744        },
2745        "omap_digest": "0xddc3680f",
2746        "size": 7,
2747        "truncate_seq": 0,
2748        "truncate_size": 0,
2749        "user_version": 54,
2750        "watchers": {}
2751      },
2752      "shards": [
2753        {
2754          "data_digest": "0xbd89c912",
2755          "errors": [
2756            "data_digest_mismatch_info"
2757          ],
2758          "object_info": {
2759            "alloc_hint_flags": 0,
2760            "data_digest": "0x2ddbf8f5",
2761            "expected_object_size": 0,
2762            "expected_write_size": 0,
2763            "flags": [
2764              "dirty",
2765              "omap",
2766              "data_digest",
2767              "omap_digest"
2768            ],
2769            "lost": 0,
2770            "manifest": {
2771              "type": 0
2772            },
2773            "oid": {
2774              "hash": 1629828556,
2775              "key": "",
2776              "max": 0,
2777              "namespace": "",
2778              "oid": "ROBJ18",
2779              "pool": 3,
2780              "snapid": -2
2781            },
2782            "omap_digest": "0xddc3680f",
2783            "size": 7,
2784            "truncate_seq": 0,
2785            "truncate_size": 0,
2786            "user_version": 54,
2787            "watchers": {}
2788          },
2789          "omap_digest": "0xddc3680f",
2790          "osd": 0,
2791          "primary": false,
2792          "size": 7
2793        },
2794        {
2795          "data_digest": "0xbd89c912",
2796          "errors": [
2797            "data_digest_mismatch_info"
2798          ],
2799          "object_info": {
2800            "alloc_hint_flags": 255,
2801            "data_digest": "0x2ddbf8f5",
2802            "expected_object_size": 0,
2803            "expected_write_size": 0,
2804            "flags": [
2805              "dirty",
2806              "omap",
2807              "data_digest",
2808              "omap_digest"
2809            ],
2810            "lost": 0,
2811            "manifest": {
2812              "type": 0
2813            },
2814            "oid": {
2815              "hash": 1629828556,
2816              "key": "",
2817              "max": 0,
2818              "namespace": "",
2819              "oid": "ROBJ18",
2820              "pool": 3,
2821              "snapid": -2
2822            },
2823            "omap_digest": "0xddc3680f",
2824            "size": 7,
2825            "truncate_seq": 0,
2826            "truncate_size": 0,
2827            "user_version": 54,
2828            "watchers": {}
2829          },
2830          "omap_digest": "0xddc3680f",
2831          "osd": 1,
2832          "primary": true,
2833          "size": 7
2834        }
2835      ],
2836      "union_shard_errors": [
2837        "data_digest_mismatch_info"
2838      ]
2839    },
2840    {
2841      "shards": [
2842         {
2843           "data_digest": "0x578a4830",
2844           "omap_digest": "0xf8e11918",
2845           "size": 7,
2846           "errors": [
2847             "data_digest_mismatch_info"
2848           ],
2849           "osd": 0,
2850           "primary": false
2851         },
2852         {
2853           "data_digest": "0x2ddbf8f5",
2854           "omap_digest": "0xf8e11918",
2855           "size": 7,
2856           "errors": [],
2857           "osd": 1,
2858           "primary": true
2859         }
2860       ],
2861       "selected_object_info": {
2862         "oid": {
2863           "oid": "ROBJ2",
2864           "key": "",
2865           "snapid": -2,
2866           "hash": 2026323607,
2867           "max": 0,
2868           "pool": 3,
2869           "namespace": ""
2870         },
2871         "version": "51'60",
2872         "prior_version": "23'6",
2873         "last_reqid": "osd.1.0:59",
2874         "user_version": 6,
2875         "size": 7,
2876         "mtime": "2018-04-05 14:33:20.498756",
2877         "local_mtime": "2018-04-05 14:33:20.499704",
2878         "lost": 0,
2879         "flags": [
2880           "dirty",
2881           "omap",
2882           "data_digest",
2883           "omap_digest"
2884         ],
2885         "truncate_seq": 0,
2886         "truncate_size": 0,
2887         "data_digest": "0x2ddbf8f5",
2888         "omap_digest": "0xf8e11918",
2889         "expected_object_size": 0,
2890         "expected_write_size": 0,
2891         "alloc_hint_flags": 0,
2892         "manifest": {
2893           "type": 0
2894         },
2895         "watchers": {}
2896       },
2897       "union_shard_errors": [
2898         "data_digest_mismatch_info"
2899       ],
2900       "errors": [
2901         "data_digest_mismatch"
2902       ],
2903       "object": {
2904         "version": 6,
2905         "snap": "head",
2906         "locator": "",
2907         "nspace": "",
2908         "name": "ROBJ2"
2909       }
2910     },
2911     {
2912       "shards": [
2913         {
2914           "data_digest": "0x2ddbf8f5",
2915           "omap_digest": "0x00b35dfd",
2916           "size": 7,
2917           "errors": [],
2918           "osd": 0,
2919           "primary": false
2920         },
2921         {
2922           "errors": [
2923             "missing"
2924           ],
2925           "osd": 1,
2926           "primary": true
2927         }
2928       ],
2929       "selected_object_info": {
2930         "oid": {
2931           "oid": "ROBJ3",
2932           "key": "",
2933           "snapid": -2,
2934           "hash": 625845583,
2935           "max": 0,
2936           "pool": 3,
2937           "namespace": ""
2938         },
2939         "version": "51'61",
2940         "prior_version": "25'9",
2941         "last_reqid": "osd.1.0:60",
2942         "user_version": 9,
2943         "size": 7,
2944         "mtime": "2018-04-05 14:33:21.189382",
2945         "local_mtime": "2018-04-05 14:33:21.190446",
2946         "lost": 0,
2947         "flags": [
2948           "dirty",
2949           "omap",
2950           "data_digest",
2951           "omap_digest"
2952         ],
2953         "truncate_seq": 0,
2954         "truncate_size": 0,
2955         "data_digest": "0x2ddbf8f5",
2956         "omap_digest": "0x00b35dfd",
2957         "expected_object_size": 0,
2958         "expected_write_size": 0,
2959         "alloc_hint_flags": 0,
2960         "manifest": {
2961           "type": 0
2962         },
2963         "watchers": {}
2964       },
2965       "union_shard_errors": [
2966         "missing"
2967       ],
2968       "errors": [],
2969       "object": {
2970         "version": 9,
2971         "snap": "head",
2972         "locator": "",
2973         "nspace": "",
2974         "name": "ROBJ3"
2975       }
2976     },
2977     {
2978       "shards": [
2979         {
2980           "data_digest": "0x2ddbf8f5",
2981           "omap_digest": "0xd7178dfe",
2982           "size": 7,
2983           "errors": [
2984             "omap_digest_mismatch_info"
2985           ],
2986           "osd": 0,
2987           "primary": false
2988         },
2989         {
2990           "data_digest": "0x2ddbf8f5",
2991           "omap_digest": "0xe2d46ea4",
2992           "size": 7,
2993           "errors": [],
2994           "osd": 1,
2995           "primary": true
2996         }
2997       ],
2998       "selected_object_info": {
2999         "oid": {
3000           "oid": "ROBJ4",
3001           "key": "",
3002           "snapid": -2,
3003           "hash": 2360875311,
3004           "max": 0,
3005           "pool": 3,
3006           "namespace": ""
3007         },
3008         "version": "51'62",
3009         "prior_version": "27'12",
3010         "last_reqid": "osd.1.0:61",
3011         "user_version": 12,
3012         "size": 7,
3013         "mtime": "2018-04-05 14:33:21.862313",
3014         "local_mtime": "2018-04-05 14:33:21.863261",
3015         "lost": 0,
3016         "flags": [
3017           "dirty",
3018           "omap",
3019           "data_digest",
3020           "omap_digest"
3021         ],
3022         "truncate_seq": 0,
3023         "truncate_size": 0,
3024         "data_digest": "0x2ddbf8f5",
3025         "omap_digest": "0xe2d46ea4",
3026         "expected_object_size": 0,
3027         "expected_write_size": 0,
3028         "alloc_hint_flags": 0,
3029         "manifest": {
3030           "type": 0
3031         },
3032         "watchers": {}
3033       },
3034       "union_shard_errors": [
3035         "omap_digest_mismatch_info"
3036       ],
3037       "errors": [
3038         "omap_digest_mismatch"
3039       ],
3040       "object": {
3041         "version": 12,
3042         "snap": "head",
3043         "locator": "",
3044         "nspace": "",
3045         "name": "ROBJ4"
3046       }
3047     },
3048     {
3049       "shards": [
3050         {
3051           "data_digest": "0x2ddbf8f5",
3052           "omap_digest": "0x1a862a41",
3053           "size": 7,
3054           "errors": [],
3055           "osd": 0,
3056           "primary": false
3057         },
3058         {
3059           "data_digest": "0x2ddbf8f5",
3060           "omap_digest": "0x06cac8f6",
3061           "size": 7,
3062           "errors": [
3063             "omap_digest_mismatch_info"
3064           ],
3065           "osd": 1,
3066           "primary": true
3067         }
3068       ],
3069       "selected_object_info": {
3070         "oid": {
3071           "oid": "ROBJ5",
3072           "key": "",
3073           "snapid": -2,
3074           "hash": 2334915887,
3075           "max": 0,
3076           "pool": 3,
3077           "namespace": ""
3078         },
3079         "version": "51'63",
3080         "prior_version": "29'15",
3081         "last_reqid": "osd.1.0:62",
3082         "user_version": 15,
3083         "size": 7,
3084         "mtime": "2018-04-05 14:33:22.589300",
3085         "local_mtime": "2018-04-05 14:33:22.590376",
3086         "lost": 0,
3087         "flags": [
3088           "dirty",
3089           "omap",
3090           "data_digest",
3091           "omap_digest"
3092         ],
3093         "truncate_seq": 0,
3094         "truncate_size": 0,
3095         "data_digest": "0x2ddbf8f5",
3096         "omap_digest": "0x1a862a41",
3097         "expected_object_size": 0,
3098         "expected_write_size": 0,
3099         "alloc_hint_flags": 0,
3100         "manifest": {
3101           "type": 0
3102         },
3103         "watchers": {}
3104       },
3105       "union_shard_errors": [
3106         "omap_digest_mismatch_info"
3107       ],
3108       "errors": [
3109         "omap_digest_mismatch"
3110       ],
3111       "object": {
3112         "version": 15,
3113         "snap": "head",
3114         "locator": "",
3115         "nspace": "",
3116         "name": "ROBJ5"
3117       }
3118     },
3119     {
3120       "shards": [
3121         {
3122           "data_digest": "0x2ddbf8f5",
3123           "omap_digest": "0x689ee887",
3124           "size": 7,
3125           "errors": [
3126             "omap_digest_mismatch_info"
3127           ],
3128           "osd": 0,
3129           "primary": false
3130         },
3131         {
3132           "data_digest": "0x2ddbf8f5",
3133           "omap_digest": "0x179c919f",
3134           "size": 7,
3135           "errors": [],
3136           "osd": 1,
3137           "primary": true
3138         }
3139       ],
3140       "selected_object_info": {
3141         "oid": {
3142           "oid": "ROBJ6",
3143           "key": "",
3144           "snapid": -2,
3145           "hash": 390610085,
3146           "max": 0,
3147           "pool": 3,
3148           "namespace": ""
3149         },
3150         "version": "51'54",
3151         "prior_version": "31'18",
3152         "last_reqid": "osd.1.0:53",
3153         "user_version": 18,
3154         "size": 7,
3155         "mtime": "2018-04-05 14:33:23.289188",
3156         "local_mtime": "2018-04-05 14:33:23.290130",
3157         "lost": 0,
3158         "flags": [
3159           "dirty",
3160           "omap",
3161           "data_digest",
3162           "omap_digest"
3163         ],
3164         "truncate_seq": 0,
3165         "truncate_size": 0,
3166         "data_digest": "0x2ddbf8f5",
3167         "omap_digest": "0x179c919f",
3168         "expected_object_size": 0,
3169         "expected_write_size": 0,
3170         "alloc_hint_flags": 0,
3171         "manifest": {
3172           "type": 0
3173         },
3174         "watchers": {}
3175       },
3176       "union_shard_errors": [
3177         "omap_digest_mismatch_info"
3178       ],
3179       "errors": [
3180         "omap_digest_mismatch"
3181       ],
3182       "object": {
3183         "version": 18,
3184         "snap": "head",
3185         "locator": "",
3186         "nspace": "",
3187         "name": "ROBJ6"
3188       }
3189     },
3190     {
3191       "shards": [
3192         {
3193           "data_digest": "0x2ddbf8f5",
3194           "omap_digest": "0xefced57a",
3195           "size": 7,
3196           "errors": [],
3197           "osd": 0,
3198           "primary": false
3199         },
3200         {
3201           "data_digest": "0x2ddbf8f5",
3202           "omap_digest": "0x6a73cc07",
3203           "size": 7,
3204           "errors": [
3205             "omap_digest_mismatch_info"
3206           ],
3207           "osd": 1,
3208           "primary": true
3209         }
3210       ],
3211       "selected_object_info": {
3212         "oid": {
3213           "oid": "ROBJ7",
3214           "key": "",
3215           "snapid": -2,
3216           "hash": 3529485009,
3217           "max": 0,
3218           "pool": 3,
3219           "namespace": ""
3220         },
3221         "version": "51'53",
3222         "prior_version": "33'21",
3223         "last_reqid": "osd.1.0:52",
3224         "user_version": 21,
3225         "size": 7,
3226         "mtime": "2018-04-05 14:33:23.979658",
3227         "local_mtime": "2018-04-05 14:33:23.980731",
3228         "lost": 0,
3229         "flags": [
3230           "dirty",
3231           "omap",
3232           "data_digest",
3233           "omap_digest"
3234         ],
3235         "truncate_seq": 0,
3236         "truncate_size": 0,
3237         "data_digest": "0x2ddbf8f5",
3238         "omap_digest": "0xefced57a",
3239         "expected_object_size": 0,
3240         "expected_write_size": 0,
3241         "alloc_hint_flags": 0,
3242         "manifest": {
3243           "type": 0
3244         },
3245         "watchers": {}
3246       },
3247       "union_shard_errors": [
3248         "omap_digest_mismatch_info"
3249       ],
3250       "errors": [
3251         "omap_digest_mismatch"
3252       ],
3253       "object": {
3254         "version": 21,
3255         "snap": "head",
3256         "locator": "",
3257         "nspace": "",
3258         "name": "ROBJ7"
3259       }
3260     },
3261     {
3262       "shards": [
3263         {
3264           "attrs": [
3265             {
3266               "Base64": false,
3267               "value": "bad-val",
3268               "name": "key1-ROBJ8"
3269             },
3270             {
3271               "Base64": false,
3272               "value": "val2-ROBJ8",
3273               "name": "key2-ROBJ8"
3274             }
3275           ],
3276           "data_digest": "0x2ddbf8f5",
3277           "omap_digest": "0xd6be81dc",
3278           "size": 7,
3279           "errors": [],
3280           "osd": 0,
3281           "primary": false
3282         },
3283         {
3284           "attrs": [
3285             {
3286               "Base64": false,
3287               "value": "val1-ROBJ8",
3288               "name": "key1-ROBJ8"
3289             },
3290             {
3291               "Base64": false,
3292               "value": "val3-ROBJ8",
3293               "name": "key3-ROBJ8"
3294             }
3295           ],
3296           "data_digest": "0x2ddbf8f5",
3297           "omap_digest": "0xd6be81dc",
3298           "size": 7,
3299           "errors": [],
3300           "osd": 1,
3301           "primary": true
3302         }
3303       ],
3304       "selected_object_info": {
3305         "oid": {
3306           "oid": "ROBJ8",
3307           "key": "",
3308           "snapid": -2,
3309           "hash": 2359695969,
3310           "max": 0,
3311           "pool": 3,
3312           "namespace": ""
3313         },
3314         "version": "79'66",
3315         "prior_version": "79'65",
3316         "last_reqid": "client.4554.0:1",
3317         "user_version": 79,
3318         "size": 7,
3319         "mtime": "2018-04-05 14:34:05.598688",
3320         "local_mtime": "2018-04-05 14:34:05.599698",
3321         "lost": 0,
3322         "flags": [
3323           "dirty",
3324           "omap",
3325           "data_digest",
3326           "omap_digest"
3327         ],
3328         "truncate_seq": 0,
3329         "truncate_size": 0,
3330         "data_digest": "0x2ddbf8f5",
3331         "omap_digest": "0xd6be81dc",
3332         "expected_object_size": 0,
3333         "expected_write_size": 0,
3334         "alloc_hint_flags": 0,
3335         "manifest": {
3336           "type": 0
3337         },
3338         "watchers": {}
3339       },
3340       "union_shard_errors": [],
3341       "errors": [
3342         "attr_value_mismatch",
3343         "attr_name_mismatch"
3344       ],
3345       "object": {
3346         "version": 66,
3347         "snap": "head",
3348         "locator": "",
3349         "nspace": "",
3350         "name": "ROBJ8"
3351       }
3352     },
3353     {
3354       "shards": [
3355         {
3356           "object_info": {
3357             "oid": {
3358               "oid": "ROBJ9",
3359               "key": "",
3360               "snapid": -2,
3361               "hash": 537189375,
3362               "max": 0,
3363               "pool": 3,
3364               "namespace": ""
3365             },
3366             "version": "51'64",
3367             "prior_version": "37'27",
3368             "last_reqid": "osd.1.0:63",
3369             "user_version": 27,
3370             "size": 7,
3371             "mtime": "2018-04-05 14:33:25.352485",
3372             "local_mtime": "2018-04-05 14:33:25.353746",
3373             "lost": 0,
3374             "flags": [
3375               "dirty",
3376               "omap",
3377               "data_digest",
3378               "omap_digest"
3379             ],
3380             "truncate_seq": 0,
3381             "truncate_size": 0,
3382             "data_digest": "0x2ddbf8f5",
3383             "omap_digest": "0x2eecc539",
3384             "expected_object_size": 0,
3385             "expected_write_size": 0,
3386             "alloc_hint_flags": 0,
3387             "manifest": {
3388               "type": 0
3389             },
3390             "watchers": {}
3391           },
3392           "data_digest": "0x1f26fb26",
3393           "omap_digest": "0x2eecc539",
3394           "size": 3,
3395           "errors": [
3396             "obj_size_info_mismatch"
3397           ],
3398           "osd": 0,
3399           "primary": false
3400         },
3401         {
3402           "object_info": {
3403             "oid": {
3404               "oid": "ROBJ9",
3405               "key": "",
3406               "snapid": -2,
3407               "hash": 537189375,
3408               "max": 0,
3409               "pool": 3,
3410               "namespace": ""
3411             },
3412             "version": "119'68",
3413             "prior_version": "51'64",
3414             "last_reqid": "client.4834.0:1",
3415             "user_version": 81,
3416             "size": 3,
3417             "mtime": "2018-04-05 14:35:01.500659",
3418             "local_mtime": "2018-04-05 14:35:01.502117",
3419             "lost": 0,
3420             "flags": [
3421               "dirty",
3422               "omap",
3423               "data_digest",
3424               "omap_digest"
3425             ],
3426             "truncate_seq": 0,
3427             "truncate_size": 0,
3428             "data_digest": "0x1f26fb26",
3429             "omap_digest": "0x2eecc539",
3430             "expected_object_size": 0,
3431             "expected_write_size": 0,
3432             "alloc_hint_flags": 0,
3433             "manifest": {
3434               "type": 0
3435             },
3436             "watchers": {}
3437           },
3438           "data_digest": "0x1f26fb26",
3439           "omap_digest": "0x2eecc539",
3440           "size": 3,
3441           "errors": [],
3442           "osd": 1,
3443           "primary": true
3444         }
3445       ],
3446       "selected_object_info": {
3447         "oid": {
3448           "oid": "ROBJ9",
3449           "key": "",
3450           "snapid": -2,
3451           "hash": 537189375,
3452           "max": 0,
3453           "pool": 3,
3454           "namespace": ""
3455         },
3456         "version": "119'68",
3457         "prior_version": "51'64",
3458         "last_reqid": "client.4834.0:1",
3459         "user_version": 81,
3460         "size": 3,
3461         "mtime": "2018-04-05 14:35:01.500659",
3462         "local_mtime": "2018-04-05 14:35:01.502117",
3463         "lost": 0,
3464         "flags": [
3465           "dirty",
3466           "omap",
3467           "data_digest",
3468           "omap_digest"
3469         ],
3470         "truncate_seq": 0,
3471         "truncate_size": 0,
3472         "data_digest": "0x1f26fb26",
3473         "omap_digest": "0x2eecc539",
3474         "expected_object_size": 0,
3475         "expected_write_size": 0,
3476         "alloc_hint_flags": 0,
3477         "manifest": {
3478           "type": 0
3479         },
3480         "watchers": {}
3481       },
3482       "union_shard_errors": [
3483         "obj_size_info_mismatch"
3484       ],
3485       "errors": [
3486         "object_info_inconsistency"
3487       ],
3488       "object": {
3489         "version": 68,
3490         "snap": "head",
3491         "locator": "",
3492         "nspace": "",
3493         "name": "ROBJ9"
3494       }
3495     }
3496   ],
3497   "epoch": 0
3498 }
3499 EOF
3500
3501     jq "$jqfilter" $dir/json | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/csjson
3502     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
3503     if test $getjson = "yes"
3504     then
3505         jq '.' $dir/json > save2.json
3506     fi
3507
3508     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
3509     then
3510       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
3511     fi
3512
3513     repair $pg
3514     wait_for_clean
3515
3516     # This hangs if the repair doesn't work
3517     timeout 30 rados -p $poolname get ROBJ17 $dir/robj17.out || return 1
3518     timeout 30 rados -p $poolname get ROBJ18 $dir/robj18.out || return 1
3519     # Even though we couldn't repair all of the introduced errors, we can fix ROBJ17
3520     diff -q $dir/new.ROBJ17 $dir/robj17.out || return 1
3521     rm -f $dir/new.ROBJ17 $dir/robj17.out || return 1
3522     diff -q $dir/new.ROBJ18 $dir/robj18.out || return 1
3523     rm -f $dir/new.ROBJ18 $dir/robj18.out || return 1
3524
3525     if [ $ERRORS != "0" ];
3526     then
3527         echo "TEST FAILED WITH $ERRORS ERRORS"
3528         return 1
3529     fi
3530
3531     ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it
3532     teardown $dir || return 1
3533 }
3534
3535
3536 #
3537 # Test scrub errors for an erasure coded pool
3538 #
3539 function corrupt_scrub_erasure() {
3540     local dir=$1
3541     local allow_overwrites=$2
3542     local poolname=ecpool
3543     local total_objs=7
3544
3545     setup $dir || return 1
3546     run_mon $dir a || return 1
3547     run_mgr $dir x || return 1
3548     for id in $(seq 0 2) ; do
3549         if [ "$allow_overwrites" = "true" ]; then
3550             run_osd $dir $id || return 1
3551         else
3552             run_osd_filestore $dir $id || return 1
3553         fi
3554     done
3555     create_rbd_pool || return 1
3556     create_pool foo 1
3557
3558     create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1
3559     wait_for_clean || return 1
3560
3561     for i in $(seq 1 $total_objs) ; do
3562         objname=EOBJ${i}
3563         add_something $dir $poolname $objname || return 1
3564
3565         local osd=$(expr $i % 2)
3566
3567         case $i in
3568         1)
3569             # Size (deep scrub data_digest too)
3570             local payload=UVWXYZZZ
3571             echo $payload > $dir/CORRUPT
3572             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
3573             ;;
3574
3575         2)
3576             # Corrupt EC shard
3577             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1
3578             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
3579             ;;
3580
3581         3)
3582              # missing
3583              objectstore_tool $dir $osd $objname remove || return 1
3584              ;;
3585
3586         4)
3587             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
3588             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
3589
3590             # Break xattrs
3591             echo -n bad-val > $dir/bad-val
3592             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
3593             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
3594             echo -n val3-$objname > $dir/newval
3595             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
3596             rm $dir/bad-val $dir/newval
3597             ;;
3598
3599         5)
3600             # Corrupt EC shard
3601             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2
3602             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
3603             ;;
3604
3605         6)
3606             objectstore_tool $dir 0 $objname rm-attr hinfo_key || return 1
3607             echo -n bad-val > $dir/bad-val
3608             objectstore_tool $dir 1 $objname set-attr hinfo_key $dir/bad-val || return 1
3609             ;;
3610
3611         7)
3612             local payload=MAKETHISDIFFERENTFROMOTHEROBJECTS
3613             echo $payload > $dir/DIFFERENT
3614             rados --pool $poolname put $objname $dir/DIFFERENT || return 1
3615
3616             # Get hinfo_key from EOBJ1
3617             objectstore_tool $dir 0 EOBJ1 get-attr hinfo_key > $dir/hinfo
3618             objectstore_tool $dir 0 $objname set-attr hinfo_key $dir/hinfo || return 1
3619             rm -f $dir/hinfo
3620             ;;
3621
3622         esac
3623     done
3624
3625     local pg=$(get_pg $poolname EOBJ0)
3626
3627     pg_scrub $pg
3628
3629     rados list-inconsistent-pg $poolname > $dir/json || return 1
3630     # Check pg count
3631     test $(jq '. | length' $dir/json) = "1" || return 1
3632     # Check pgid
3633     test $(jq -r '.[0]' $dir/json) = $pg || return 1
3634
3635     rados list-inconsistent-obj $pg > $dir/json || return 1
3636     # Get epoch for repair-get requests
3637     epoch=$(jq .epoch $dir/json)
3638
3639     jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
3640 {
3641   "inconsistents": [
3642     {
3643       "shards": [
3644         {
3645           "size": 2048,
3646           "errors": [],
3647           "shard": 2,
3648           "osd": 0,
3649           "primary": false
3650         },
3651         {
3652           "object_info": {
3653             "oid": {
3654               "oid": "EOBJ1",
3655               "key": "",
3656               "snapid": -2,
3657               "hash": 560836233,
3658               "max": 0,
3659               "pool": 3,
3660               "namespace": ""
3661             },
3662             "version": "27'1",
3663             "prior_version": "0'0",
3664             "last_reqid": "client.4184.0:1",
3665             "user_version": 1,
3666             "size": 7,
3667             "mtime": "",
3668             "local_mtime": "",
3669             "lost": 0,
3670             "flags": [
3671               "dirty",
3672               "data_digest"
3673             ],
3674             "truncate_seq": 0,
3675             "truncate_size": 0,
3676             "data_digest": "0x2ddbf8f5",
3677             "omap_digest": "0xffffffff",
3678             "expected_object_size": 0,
3679             "expected_write_size": 0,
3680             "alloc_hint_flags": 0,
3681             "manifest": {
3682               "type": 0
3683             },
3684             "watchers": {}
3685           },
3686           "size": 9,
3687           "shard": 0,
3688           "errors": [
3689             "size_mismatch_info",
3690             "obj_size_info_mismatch"
3691           ],
3692           "osd": 1,
3693           "primary": true
3694         },
3695         {
3696           "size": 2048,
3697           "shard": 1,
3698           "errors": [],
3699           "osd": 2,
3700           "primary": false
3701         }
3702       ],
3703       "selected_object_info": {
3704         "oid": {
3705           "oid": "EOBJ1",
3706           "key": "",
3707           "snapid": -2,
3708           "hash": 560836233,
3709           "max": 0,
3710           "pool": 3,
3711           "namespace": ""
3712         },
3713         "version": "27'1",
3714         "prior_version": "0'0",
3715         "last_reqid": "client.4184.0:1",
3716         "user_version": 1,
3717         "size": 7,
3718         "mtime": "",
3719         "local_mtime": "",
3720         "lost": 0,
3721         "flags": [
3722           "dirty",
3723           "data_digest"
3724         ],
3725         "truncate_seq": 0,
3726         "truncate_size": 0,
3727         "data_digest": "0x2ddbf8f5",
3728         "omap_digest": "0xffffffff",
3729         "expected_object_size": 0,
3730         "expected_write_size": 0,
3731         "alloc_hint_flags": 0,
3732         "manifest": {
3733           "type": 0
3734         },
3735         "watchers": {}
3736       },
3737       "union_shard_errors": [
3738         "size_mismatch_info",
3739         "obj_size_info_mismatch"
3740       ],
3741       "errors": [
3742         "size_mismatch"
3743       ],
3744       "object": {
3745         "version": 1,
3746         "snap": "head",
3747         "locator": "",
3748         "nspace": "",
3749         "name": "EOBJ1"
3750       }
3751     },
3752     {
3753       "shards": [
3754         {
3755           "size": 2048,
3756           "errors": [],
3757           "shard": 2,
3758           "osd": 0,
3759           "primary": false
3760         },
3761         {
3762           "shard": 0,
3763           "errors": [
3764             "missing"
3765           ],
3766           "osd": 1,
3767           "primary": true
3768         },
3769         {
3770           "size": 2048,
3771           "shard": 1,
3772           "errors": [],
3773           "osd": 2,
3774           "primary": false
3775         }
3776       ],
3777       "selected_object_info": {
3778         "oid": {
3779           "oid": "EOBJ3",
3780           "key": "",
3781           "snapid": -2,
3782           "hash": 3125668237,
3783           "max": 0,
3784           "pool": 3,
3785           "namespace": ""
3786         },
3787         "version": "39'3",
3788         "prior_version": "0'0",
3789         "last_reqid": "client.4252.0:1",
3790         "user_version": 3,
3791         "size": 7,
3792         "mtime": "",
3793         "local_mtime": "",
3794         "lost": 0,
3795         "flags": [
3796           "dirty",
3797           "data_digest"
3798         ],
3799         "truncate_seq": 0,
3800         "truncate_size": 0,
3801         "data_digest": "0x2ddbf8f5",
3802         "omap_digest": "0xffffffff",
3803         "expected_object_size": 0,
3804         "expected_write_size": 0,
3805         "alloc_hint_flags": 0,
3806         "manifest": {
3807           "type": 0
3808         },
3809         "watchers": {}
3810       },
3811       "union_shard_errors": [
3812         "missing"
3813       ],
3814       "errors": [],
3815       "object": {
3816         "version": 3,
3817         "snap": "head",
3818         "locator": "",
3819         "nspace": "",
3820         "name": "EOBJ3"
3821       }
3822     },
3823     {
3824       "shards": [
3825         {
3826           "attrs": [
3827             {
3828               "Base64": false,
3829               "value": "bad-val",
3830               "name": "key1-EOBJ4"
3831             },
3832             {
3833               "Base64": false,
3834               "value": "val2-EOBJ4",
3835               "name": "key2-EOBJ4"
3836             }
3837           ],
3838           "size": 2048,
3839           "errors": [],
3840           "shard": 2,
3841           "osd": 0,
3842           "primary": false
3843         },
3844         {
3845           "osd": 1,
3846           "primary": true,
3847           "shard": 0,
3848           "errors": [],
3849           "size": 2048,
3850           "attrs": [
3851             {
3852               "Base64": false,
3853               "value": "val1-EOBJ4",
3854               "name": "key1-EOBJ4"
3855             },
3856             {
3857               "Base64": false,
3858               "value": "val2-EOBJ4",
3859               "name": "key2-EOBJ4"
3860             }
3861           ]
3862         },
3863         {
3864           "osd": 2,
3865           "primary": false,
3866           "shard": 1,
3867           "errors": [],
3868           "size": 2048,
3869           "attrs": [
3870             {
3871               "Base64": false,
3872               "value": "val1-EOBJ4",
3873               "name": "key1-EOBJ4"
3874             },
3875             {
3876               "Base64": false,
3877               "value": "val3-EOBJ4",
3878               "name": "key3-EOBJ4"
3879             }
3880           ]
3881         }
3882       ],
3883       "selected_object_info": {
3884         "oid": {
3885           "oid": "EOBJ4",
3886           "key": "",
3887           "snapid": -2,
3888           "hash": 1618759290,
3889           "max": 0,
3890           "pool": 3,
3891           "namespace": ""
3892         },
3893         "version": "45'6",
3894         "prior_version": "45'5",
3895         "last_reqid": "client.4294.0:1",
3896         "user_version": 6,
3897         "size": 7,
3898         "mtime": "",
3899         "local_mtime": "",
3900         "lost": 0,
3901         "flags": [
3902           "dirty",
3903           "data_digest"
3904         ],
3905         "truncate_seq": 0,
3906         "truncate_size": 0,
3907         "data_digest": "0x2ddbf8f5",
3908         "omap_digest": "0xffffffff",
3909         "expected_object_size": 0,
3910         "expected_write_size": 0,
3911         "alloc_hint_flags": 0,
3912         "manifest": {
3913           "type": 0
3914         },
3915         "watchers": {}
3916       },
3917       "union_shard_errors": [],
3918       "errors": [
3919         "attr_value_mismatch",
3920         "attr_name_mismatch"
3921       ],
3922       "object": {
3923         "version": 6,
3924         "snap": "head",
3925         "locator": "",
3926         "nspace": "",
3927         "name": "EOBJ4"
3928       }
3929     },
3930     {
3931       "shards": [
3932         {
3933           "size": 2048,
3934           "errors": [],
3935           "shard": 2,
3936           "osd": 0,
3937           "primary": false
3938         },
3939         {
3940           "object_info": {
3941             "oid": {
3942               "oid": "EOBJ5",
3943               "key": "",
3944               "snapid": -2,
3945               "hash": 2918945441,
3946               "max": 0,
3947               "pool": 3,
3948               "namespace": ""
3949             },
3950             "version": "59'7",
3951             "prior_version": "0'0",
3952             "last_reqid": "client.4382.0:1",
3953             "user_version": 7,
3954             "size": 7,
3955             "mtime": "",
3956             "local_mtime": "",
3957             "lost": 0,
3958             "flags": [
3959               "dirty",
3960               "data_digest"
3961             ],
3962             "truncate_seq": 0,
3963             "truncate_size": 0,
3964             "data_digest": "0x2ddbf8f5",
3965             "omap_digest": "0xffffffff",
3966             "expected_object_size": 0,
3967             "expected_write_size": 0,
3968             "alloc_hint_flags": 0,
3969             "manifest": {
3970               "type": 0
3971             },
3972             "watchers": {}
3973           },
3974           "size": 4096,
3975           "shard": 0,
3976           "errors": [
3977             "size_mismatch_info",
3978             "obj_size_info_mismatch"
3979           ],
3980           "osd": 1,
3981           "primary": true
3982         },
3983         {
3984           "size": 2048,
3985           "shard": 1,
3986           "errors": [],
3987           "osd": 2,
3988           "primary": false
3989         }
3990       ],
3991       "selected_object_info": {
3992         "oid": {
3993           "oid": "EOBJ5",
3994           "key": "",
3995           "snapid": -2,
3996           "hash": 2918945441,
3997           "max": 0,
3998           "pool": 3,
3999           "namespace": ""
4000         },
4001         "version": "59'7",
4002         "prior_version": "0'0",
4003         "last_reqid": "client.4382.0:1",
4004         "user_version": 7,
4005         "size": 7,
4006         "mtime": "",
4007         "local_mtime": "",
4008         "lost": 0,
4009         "flags": [
4010           "dirty",
4011           "data_digest"
4012         ],
4013         "truncate_seq": 0,
4014         "truncate_size": 0,
4015         "data_digest": "0x2ddbf8f5",
4016         "omap_digest": "0xffffffff",
4017         "expected_object_size": 0,
4018         "expected_write_size": 0,
4019         "alloc_hint_flags": 0,
4020         "manifest": {
4021           "type": 0
4022         },
4023         "watchers": {}
4024       },
4025       "union_shard_errors": [
4026         "size_mismatch_info",
4027         "obj_size_info_mismatch"
4028       ],
4029       "errors": [
4030         "size_mismatch"
4031       ],
4032       "object": {
4033         "version": 7,
4034         "snap": "head",
4035         "locator": "",
4036         "nspace": "",
4037         "name": "EOBJ5"
4038       }
4039    },
4040    {
4041      "errors": [],
4042      "object": {
4043        "locator": "",
4044        "name": "EOBJ6",
4045        "nspace": "",
4046        "snap": "head",
4047        "version": 8
4048      },
4049      "selected_object_info": {
4050         "oid": {
4051           "oid": "EOBJ6",
4052           "key": "",
4053           "snapid": -2,
4054           "hash": 3050890866,
4055           "max": 0,
4056           "pool": 3,
4057           "namespace": ""
4058         },
4059         "version": "65'8",
4060         "prior_version": "0'0",
4061         "last_reqid": "client.4418.0:1",
4062         "user_version": 8,
4063         "size": 7,
4064         "mtime": "",
4065         "local_mtime": "",
4066         "lost": 0,
4067         "flags": [
4068           "dirty",
4069           "data_digest"
4070         ],
4071         "truncate_seq": 0,
4072         "truncate_size": 0,
4073         "data_digest": "0x2ddbf8f5",
4074         "omap_digest": "0xffffffff",
4075         "expected_object_size": 0,
4076         "expected_write_size": 0,
4077         "alloc_hint_flags": 0,
4078         "manifest": {
4079           "type": 0
4080         },
4081         "watchers": {}
4082      },
4083      "shards": [
4084        {
4085          "errors": [
4086            "hinfo_missing"
4087          ],
4088          "osd": 0,
4089          "primary": false,
4090          "shard": 2,
4091          "size": 2048
4092        },
4093        {
4094          "errors": [
4095            "hinfo_corrupted"
4096          ],
4097          "osd": 1,
4098          "primary": true,
4099          "shard": 0,
4100          "hashinfo": "bad-val",
4101          "size": 2048
4102        },
4103        {
4104          "errors": [],
4105          "osd": 2,
4106          "primary": false,
4107          "shard": 1,
4108          "size": 2048,
4109          "hashinfo": {
4110            "cumulative_shard_hashes": [
4111             {
4112               "hash": 80717615,
4113               "shard": 0
4114             },
4115             {
4116               "hash": 1534491824,
4117               "shard": 1
4118             },
4119             {
4120               "hash": 80717615,
4121               "shard": 2
4122             }
4123            ],
4124            "total_chunk_size": 2048
4125          }
4126        }
4127      ],
4128      "union_shard_errors": [
4129        "hinfo_missing",
4130        "hinfo_corrupted"
4131      ]
4132    },
4133    {
4134      "errors": [
4135        "hinfo_inconsistency"
4136      ],
4137      "object": {
4138        "locator": "",
4139        "name": "EOBJ7",
4140        "nspace": "",
4141        "snap": "head",
4142        "version": 10
4143      },
4144      "selected_object_info": {
4145         "oid": {
4146           "oid": "EOBJ7",
4147           "key": "",
4148           "snapid": -2,
4149           "hash": 3258066308,
4150           "max": 0,
4151           "pool": 3,
4152           "namespace": ""
4153         },
4154         "version": "75'10",
4155         "prior_version": "75'9",
4156         "last_reqid": "client.4482.0:1",
4157         "user_version": 10,
4158         "size": 34,
4159         "mtime": "",
4160         "local_mtime": "",
4161         "lost": 0,
4162         "flags": [
4163           "dirty",
4164           "data_digest"
4165         ],
4166         "truncate_seq": 0,
4167         "truncate_size": 0,
4168         "data_digest": "0x136e4e27",
4169         "omap_digest": "0xffffffff",
4170         "expected_object_size": 0,
4171         "expected_write_size": 0,
4172         "alloc_hint_flags": 0,
4173         "manifest": {
4174           "type": 0
4175         },
4176         "watchers": {}
4177      },
4178      "shards": [
4179        {
4180          "hashinfo": {
4181            "cumulative_shard_hashes": [
4182             {
4183               "hash": 80717615,
4184               "shard": 0
4185             },
4186             {
4187               "hash": 1534491824,
4188               "shard": 1
4189             },
4190             {
4191               "hash": 80717615,
4192               "shard": 2
4193             }
4194            ],
4195            "total_chunk_size": 2048
4196          },
4197          "errors": [],
4198          "osd": 0,
4199          "primary": false,
4200          "shard": 2,
4201          "size": 2048
4202        },
4203        {
4204          "hashinfo": {
4205            "cumulative_shard_hashes": [
4206             {
4207               "hash": 1534350760,
4208               "shard": 0
4209             },
4210             {
4211               "hash": 1534491824,
4212               "shard": 1
4213             },
4214             {
4215               "hash": 1534350760,
4216               "shard": 2
4217             }
4218            ],
4219            "total_chunk_size": 2048
4220          },
4221          "errors": [],
4222          "osd": 1,
4223          "primary": true,
4224          "shard": 0,
4225          "size": 2048
4226        },
4227        {
4228          "hashinfo": {
4229            "cumulative_shard_hashes": [
4230             {
4231               "hash": 1534350760,
4232               "shard": 0
4233             },
4234             {
4235               "hash": 1534491824,
4236               "shard": 1
4237             },
4238             {
4239               "hash": 1534350760,
4240               "shard": 2
4241             }
4242            ],
4243            "total_chunk_size": 2048
4244          },
4245          "errors": [],
4246          "osd": 2,
4247          "primary": false,
4248          "shard": 1,
4249          "size": 2048
4250        }
4251      ],
4252      "union_shard_errors": []
4253     }
4254   ],
4255   "epoch": 0
4256 }
4257 EOF
4258
4259     jq "$jqfilter" $dir/json | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/csjson
4260     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
4261     if test $getjson = "yes"
4262     then
4263         jq '.' $dir/json > save3.json
4264     fi
4265
4266     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
4267     then
4268       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
4269     fi
4270
4271     pg_deep_scrub $pg
4272
4273     rados list-inconsistent-pg $poolname > $dir/json || return 1
4274     # Check pg count
4275     test $(jq '. | length' $dir/json) = "1" || return 1
4276     # Check pgid
4277     test $(jq -r '.[0]' $dir/json) = $pg || return 1
4278
4279     rados list-inconsistent-obj $pg > $dir/json || return 1
4280     # Get epoch for repair-get requests
4281     epoch=$(jq .epoch $dir/json)
4282
4283     if [ "$allow_overwrites" = "true" ]
4284     then
4285       jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
4286 {
4287   "inconsistents": [
4288     {
4289       "shards": [
4290         {
4291           "data_digest": "0x00000000",
4292           "omap_digest": "0xffffffff",
4293           "size": 2048,
4294           "errors": [],
4295           "shard": 2,
4296           "osd": 0,
4297           "primary": false
4298         },
4299         {
4300           "object_info": {
4301             "oid": {
4302               "oid": "EOBJ1",
4303               "key": "",
4304               "snapid": -2,
4305               "hash": 560836233,
4306               "max": 0,
4307               "pool": 3,
4308               "namespace": ""
4309             },
4310             "version": "27'1",
4311             "prior_version": "0'0",
4312             "last_reqid": "client.4184.0:1",
4313             "user_version": 1,
4314             "size": 7,
4315             "mtime": "2018-04-05 14:31:33.837147",
4316             "local_mtime": "2018-04-05 14:31:33.840763",
4317             "lost": 0,
4318             "flags": [
4319               "dirty",
4320               "data_digest"
4321             ],
4322             "truncate_seq": 0,
4323             "truncate_size": 0,
4324             "data_digest": "0x2ddbf8f5",
4325             "omap_digest": "0xffffffff",
4326             "expected_object_size": 0,
4327             "expected_write_size": 0,
4328             "alloc_hint_flags": 0,
4329             "manifest": {
4330               "type": 0
4331             },
4332             "watchers": {}
4333           },
4334           "size": 9,
4335           "shard": 0,
4336           "errors": [
4337             "read_error",
4338             "size_mismatch_info",
4339             "obj_size_info_mismatch"
4340           ],
4341           "osd": 1,
4342           "primary": true
4343         },
4344         {
4345           "data_digest": "0x00000000",
4346           "omap_digest": "0xffffffff",
4347           "size": 2048,
4348           "shard": 1,
4349           "errors": [],
4350           "osd": 2,
4351           "primary": false
4352         }
4353       ],
4354       "selected_object_info": {
4355         "oid": {
4356           "oid": "EOBJ1",
4357           "key": "",
4358           "snapid": -2,
4359           "hash": 560836233,
4360           "max": 0,
4361           "pool": 3,
4362           "namespace": ""
4363         },
4364         "version": "27'1",
4365         "prior_version": "0'0",
4366         "last_reqid": "client.4184.0:1",
4367         "user_version": 1,
4368         "size": 7,
4369         "mtime": "2018-04-05 14:31:33.837147",
4370         "local_mtime": "2018-04-05 14:31:33.840763",
4371         "lost": 0,
4372         "flags": [
4373           "dirty",
4374           "data_digest"
4375         ],
4376         "truncate_seq": 0,
4377         "truncate_size": 0,
4378         "data_digest": "0x2ddbf8f5",
4379         "omap_digest": "0xffffffff",
4380         "expected_object_size": 0,
4381         "expected_write_size": 0,
4382         "alloc_hint_flags": 0,
4383         "manifest": {
4384           "type": 0
4385         },
4386         "watchers": {}
4387       },
4388       "union_shard_errors": [
4389         "read_error",
4390         "size_mismatch_info",
4391         "obj_size_info_mismatch"
4392       ],
4393       "errors": [
4394         "size_mismatch"
4395       ],
4396       "object": {
4397         "version": 1,
4398         "snap": "head",
4399         "locator": "",
4400         "nspace": "",
4401         "name": "EOBJ1"
4402       }
4403     },
4404     {
4405       "shards": [
4406         {
4407           "data_digest": "0x00000000",
4408           "omap_digest": "0xffffffff",
4409           "size": 2048,
4410           "errors": [],
4411           "shard": 2,
4412           "osd": 0,
4413           "primary": false
4414         },
4415         {
4416           "shard": 0,
4417           "errors": [
4418             "missing"
4419           ],
4420           "osd": 1,
4421           "primary": true
4422         },
4423         {
4424           "data_digest": "0x00000000",
4425           "omap_digest": "0xffffffff",
4426           "size": 2048,
4427           "shard": 1,
4428           "errors": [],
4429           "osd": 2,
4430           "primary": false
4431         }
4432       ],
4433       "selected_object_info": {
4434         "oid": {
4435           "oid": "EOBJ3",
4436           "key": "",
4437           "snapid": -2,
4438           "hash": 3125668237,
4439           "max": 0,
4440           "pool": 3,
4441           "namespace": ""
4442         },
4443         "version": "39'3",
4444         "prior_version": "0'0",
4445         "last_reqid": "client.4252.0:1",
4446         "user_version": 3,
4447         "size": 7,
4448         "mtime": "2018-04-05 14:31:46.841145",
4449         "local_mtime": "2018-04-05 14:31:46.844996",
4450         "lost": 0,
4451         "flags": [
4452           "dirty",
4453           "data_digest"
4454         ],
4455         "truncate_seq": 0,
4456         "truncate_size": 0,
4457         "data_digest": "0x2ddbf8f5",
4458         "omap_digest": "0xffffffff",
4459         "expected_object_size": 0,
4460         "expected_write_size": 0,
4461         "alloc_hint_flags": 0,
4462         "manifest": {
4463           "type": 0
4464         },
4465         "watchers": {}
4466       },
4467       "union_shard_errors": [
4468         "missing"
4469       ],
4470       "errors": [],
4471       "object": {
4472         "version": 3,
4473         "snap": "head",
4474         "locator": "",
4475         "nspace": "",
4476         "name": "EOBJ3"
4477       }
4478     },
4479     {
4480       "shards": [
4481         {
4482           "attrs": [
4483             {
4484               "Base64": false,
4485               "value": "bad-val",
4486               "name": "key1-EOBJ4"
4487             },
4488             {
4489               "Base64": false,
4490               "value": "val2-EOBJ4",
4491               "name": "key2-EOBJ4"
4492             }
4493           ],
4494           "data_digest": "0x00000000",
4495           "omap_digest": "0xffffffff",
4496           "size": 2048,
4497           "errors": [],
4498           "shard": 2,
4499           "osd": 0,
4500           "primary": false
4501         },
4502         {
4503           "attrs": [
4504             {
4505               "Base64": false,
4506               "value": "val1-EOBJ4",
4507               "name": "key1-EOBJ4"
4508             },
4509             {
4510               "Base64": false,
4511               "value": "val2-EOBJ4",
4512               "name": "key2-EOBJ4"
4513             }
4514           ],
4515           "data_digest": "0x00000000",
4516           "omap_digest": "0xffffffff",
4517           "size": 2048,
4518           "errors": [],
4519           "shard": 0,
4520           "osd": 1,
4521           "primary": true
4522         },
4523         {
4524           "attrs": [
4525             {
4526               "Base64": false,
4527               "value": "val1-EOBJ4",
4528               "name": "key1-EOBJ4"
4529             },
4530             {
4531               "Base64": false,
4532               "value": "val3-EOBJ4",
4533               "name": "key3-EOBJ4"
4534             }
4535           ],
4536           "data_digest": "0x00000000",
4537           "omap_digest": "0xffffffff",
4538           "size": 2048,
4539           "errors": [],
4540           "shard": 1,
4541           "osd": 2,
4542           "primary": false
4543         }
4544       ],
4545       "selected_object_info": {
4546         "oid": {
4547           "oid": "EOBJ4",
4548           "key": "",
4549           "snapid": -2,
4550           "hash": 1618759290,
4551           "max": 0,
4552           "pool": 3,
4553           "namespace": ""
4554         },
4555         "version": "45'6",
4556         "prior_version": "45'5",
4557         "last_reqid": "client.4294.0:1",
4558         "user_version": 6,
4559         "size": 7,
4560         "mtime": "2018-04-05 14:31:54.663622",
4561         "local_mtime": "2018-04-05 14:31:54.664527",
4562         "lost": 0,
4563         "flags": [
4564           "dirty",
4565           "data_digest"
4566         ],
4567         "truncate_seq": 0,
4568         "truncate_size": 0,
4569         "data_digest": "0x2ddbf8f5",
4570         "omap_digest": "0xffffffff",
4571         "expected_object_size": 0,
4572         "expected_write_size": 0,
4573         "alloc_hint_flags": 0,
4574         "manifest": {
4575           "type": 0
4576         },
4577         "watchers": {}
4578       },
4579       "union_shard_errors": [],
4580       "errors": [
4581         "attr_value_mismatch",
4582         "attr_name_mismatch"
4583       ],
4584       "object": {
4585         "version": 6,
4586         "snap": "head",
4587         "locator": "",
4588         "nspace": "",
4589         "name": "EOBJ4"
4590       }
4591     },
4592     {
4593       "shards": [
4594         {
4595           "data_digest": "0x00000000",
4596           "omap_digest": "0xffffffff",
4597           "size": 2048,
4598           "errors": [],
4599           "shard": 2,
4600           "osd": 0,
4601           "primary": false
4602         },
4603         {
4604           "data_digest": "0x00000000",
4605           "omap_digest": "0xffffffff",
4606           "object_info": {
4607             "oid": {
4608               "oid": "EOBJ5",
4609               "key": "",
4610               "snapid": -2,
4611               "hash": 2918945441,
4612               "max": 0,
4613               "pool": 3,
4614               "namespace": ""
4615             },
4616             "version": "59'7",
4617             "prior_version": "0'0",
4618             "last_reqid": "client.4382.0:1",
4619             "user_version": 7,
4620             "size": 7,
4621             "mtime": "2018-04-05 14:32:12.929161",
4622             "local_mtime": "2018-04-05 14:32:12.934707",
4623             "lost": 0,
4624             "flags": [
4625               "dirty",
4626               "data_digest"
4627             ],
4628             "truncate_seq": 0,
4629             "truncate_size": 0,
4630             "data_digest": "0x2ddbf8f5",
4631             "omap_digest": "0xffffffff",
4632             "expected_object_size": 0,
4633             "expected_write_size": 0,
4634             "alloc_hint_flags": 0,
4635             "manifest": {
4636               "type": 0
4637             },
4638             "watchers": {}
4639           },
4640           "size": 4096,
4641           "errors": [
4642             "size_mismatch_info",
4643             "obj_size_info_mismatch"
4644           ],
4645           "shard": 0,
4646           "osd": 1,
4647           "primary": true
4648         },
4649         {
4650           "data_digest": "0x00000000",
4651           "omap_digest": "0xffffffff",
4652           "size": 2048,
4653           "errors": [],
4654           "shard": 1,
4655           "osd": 2,
4656           "primary": false
4657         }
4658       ],
4659       "selected_object_info": {
4660         "oid": {
4661           "oid": "EOBJ5",
4662           "key": "",
4663           "snapid": -2,
4664           "hash": 2918945441,
4665           "max": 0,
4666           "pool": 3,
4667           "namespace": ""
4668         },
4669         "version": "59'7",
4670         "prior_version": "0'0",
4671         "last_reqid": "client.4382.0:1",
4672         "user_version": 7,
4673         "size": 7,
4674         "mtime": "2018-04-05 14:32:12.929161",
4675         "local_mtime": "2018-04-05 14:32:12.934707",
4676         "lost": 0,
4677         "flags": [
4678           "dirty",
4679           "data_digest"
4680         ],
4681         "truncate_seq": 0,
4682         "truncate_size": 0,
4683         "data_digest": "0x2ddbf8f5",
4684         "omap_digest": "0xffffffff",
4685         "expected_object_size": 0,
4686         "expected_write_size": 0,
4687         "alloc_hint_flags": 0,
4688         "manifest": {
4689           "type": 0
4690         },
4691         "watchers": {}
4692       },
4693       "union_shard_errors": [
4694         "size_mismatch_info",
4695         "obj_size_info_mismatch"
4696       ],
4697       "errors": [
4698         "size_mismatch"
4699       ],
4700       "object": {
4701         "version": 7,
4702         "snap": "head",
4703         "locator": "",
4704         "nspace": "",
4705         "name": "EOBJ5"
4706       }
4707     },
4708     {
4709       "object": {
4710         "name": "EOBJ6",
4711         "nspace": "",
4712         "locator": "",
4713         "snap": "head",
4714         "version": 8
4715       },
4716       "errors": [],
4717       "union_shard_errors": [
4718         "read_error",
4719         "hinfo_missing",
4720         "hinfo_corrupted"
4721       ],
4722       "selected_object_info": {
4723         "oid": {
4724           "oid": "EOBJ6",
4725           "key": "",
4726           "snapid": -2,
4727           "hash": 3050890866,
4728           "max": 0,
4729           "pool": 3,
4730           "namespace": ""
4731         },
4732         "version": "65'8",
4733         "prior_version": "0'0",
4734         "last_reqid": "client.4418.0:1",
4735         "user_version": 8,
4736         "size": 7,
4737         "mtime": "2018-04-05 14:32:20.634116",
4738         "local_mtime": "2018-04-05 14:32:20.637999",
4739         "lost": 0,
4740         "flags": [
4741           "dirty",
4742           "data_digest"
4743         ],
4744         "truncate_seq": 0,
4745         "truncate_size": 0,
4746         "data_digest": "0x2ddbf8f5",
4747         "omap_digest": "0xffffffff",
4748         "expected_object_size": 0,
4749         "expected_write_size": 0,
4750         "alloc_hint_flags": 0,
4751         "manifest": {
4752           "type": 0
4753         },
4754         "watchers": {}
4755       },
4756       "shards": [
4757         {
4758           "osd": 0,
4759           "primary": false,
4760           "shard": 2,
4761           "errors": [
4762             "read_error",
4763             "hinfo_missing"
4764           ],
4765           "size": 2048
4766         },
4767         {
4768           "osd": 1,
4769           "primary": true,
4770           "shard": 0,
4771           "errors": [
4772             "read_error",
4773             "hinfo_corrupted"
4774           ],
4775           "size": 2048,
4776           "hashinfo": "bad-val"
4777         },
4778         {
4779           "osd": 2,
4780           "primary": false,
4781           "shard": 1,
4782           "errors": [],
4783           "size": 2048,
4784           "omap_digest": "0xffffffff",
4785           "data_digest": "0x00000000",
4786           "hashinfo": {
4787             "cumulative_shard_hashes": [
4788             {
4789               "hash": 80717615,
4790               "shard": 0
4791             },
4792             {
4793               "hash": 1534491824,
4794               "shard": 1
4795             },
4796             {
4797               "hash": 80717615,
4798               "shard": 2
4799             }
4800            ],
4801            "total_chunk_size": 2048
4802          }
4803         }
4804       ]
4805     },
4806     {
4807       "object": {
4808         "name": "EOBJ7",
4809         "nspace": "",
4810         "locator": "",
4811         "snap": "head",
4812         "version": 10
4813       },
4814       "errors": [
4815         "hinfo_inconsistency"
4816       ],
4817       "union_shard_errors": [],
4818       "selected_object_info": {
4819         "oid": {
4820           "oid": "EOBJ7",
4821           "key": "",
4822           "snapid": -2,
4823           "hash": 3258066308,
4824           "max": 0,
4825           "pool": 3,
4826           "namespace": ""
4827         },
4828         "version": "75'10",
4829         "prior_version": "75'9",
4830         "last_reqid": "client.4482.0:1",
4831         "user_version": 10,
4832         "size": 34,
4833         "mtime": "2018-04-05 14:32:33.058782",
4834         "local_mtime": "2018-04-05 14:32:33.059679",
4835         "lost": 0,
4836         "flags": [
4837           "dirty",
4838           "data_digest"
4839         ],
4840         "truncate_seq": 0,
4841         "truncate_size": 0,
4842         "data_digest": "0x136e4e27",
4843         "omap_digest": "0xffffffff",
4844         "expected_object_size": 0,
4845         "expected_write_size": 0,
4846         "alloc_hint_flags": 0,
4847         "manifest": {
4848           "type": 0
4849         },
4850         "watchers": {}
4851       },
4852       "shards": [
4853         {
4854           "osd": 0,
4855           "primary": false,
4856           "shard": 2,
4857           "errors": [],
4858           "size": 2048,
4859           "omap_digest": "0xffffffff",
4860           "data_digest": "0x00000000",
4861           "hashinfo": {
4862            "cumulative_shard_hashes": [
4863             {
4864               "hash": 80717615,
4865               "shard": 0
4866             },
4867             {
4868               "hash": 1534491824,
4869               "shard": 1
4870             },
4871             {
4872               "hash": 80717615,
4873               "shard": 2
4874             }
4875            ],
4876            "total_chunk_size": 2048
4877           }
4878         },
4879         {
4880           "osd": 1,
4881           "primary": true,
4882           "shard": 0,
4883           "errors": [],
4884           "size": 2048,
4885           "omap_digest": "0xffffffff",
4886           "data_digest": "0x00000000",
4887           "hashinfo": {
4888            "cumulative_shard_hashes": [
4889             {
4890               "hash": 1534350760,
4891               "shard": 0
4892             },
4893             {
4894               "hash": 1534491824,
4895               "shard": 1
4896             },
4897             {
4898               "hash": 1534350760,
4899               "shard": 2
4900             }
4901            ],
4902            "total_chunk_size": 2048
4903           }
4904         },
4905         {
4906           "osd": 2,
4907           "primary": false,
4908           "shard": 1,
4909           "errors": [],
4910           "size": 2048,
4911           "omap_digest": "0xffffffff",
4912           "data_digest": "0x00000000",
4913           "hashinfo": {
4914            "cumulative_shard_hashes": [
4915             {
4916               "hash": 1534350760,
4917               "shard": 0
4918             },
4919             {
4920               "hash": 1534491824,
4921               "shard": 1
4922             },
4923             {
4924               "hash": 1534350760,
4925               "shard": 2
4926             }
4927            ],
4928            "total_chunk_size": 2048
4929           }
4930         }
4931       ]
4932     }
4933   ],
4934   "epoch": 0
4935 }
4936 EOF
4937
4938     else
4939
4940       jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
4941 {
4942   "inconsistents": [
4943     {
4944       "shards": [
4945         {
4946           "data_digest": "0x04cfa72f",
4947           "omap_digest": "0xffffffff",
4948           "size": 2048,
4949           "errors": [],
4950           "shard": 2,
4951           "osd": 0,
4952           "primary": false
4953         },
4954         {
4955           "object_info": {
4956             "oid": {
4957               "oid": "EOBJ1",
4958               "key": "",
4959               "snapid": -2,
4960               "hash": 560836233,
4961               "max": 0,
4962               "pool": 3,
4963               "namespace": ""
4964             },
4965             "version": "27'1",
4966             "prior_version": "0'0",
4967             "last_reqid": "client.4192.0:1",
4968             "user_version": 1,
4969             "size": 7,
4970             "mtime": "2018-04-05 14:30:10.688009",
4971             "local_mtime": "2018-04-05 14:30:10.691774",
4972             "lost": 0,
4973             "flags": [
4974               "dirty",
4975               "data_digest"
4976             ],
4977             "truncate_seq": 0,
4978             "truncate_size": 0,
4979             "data_digest": "0x2ddbf8f5",
4980             "omap_digest": "0xffffffff",
4981             "expected_object_size": 0,
4982             "expected_write_size": 0,
4983             "alloc_hint_flags": 0,
4984             "manifest": {
4985               "type": 0
4986             },
4987             "watchers": {}
4988           },
4989           "size": 9,
4990           "shard": 0,
4991           "errors": [
4992             "read_error",
4993             "size_mismatch_info",
4994             "obj_size_info_mismatch"
4995           ],
4996           "osd": 1,
4997           "primary": true
4998         },
4999         {
5000           "data_digest": "0x04cfa72f",
5001           "omap_digest": "0xffffffff",
5002           "size": 2048,
5003           "shard": 1,
5004           "errors": [],
5005           "osd": 2,
5006           "primary": false
5007         }
5008       ],
5009       "selected_object_info": {
5010         "oid": {
5011           "oid": "EOBJ1",
5012           "key": "",
5013           "snapid": -2,
5014           "hash": 560836233,
5015           "max": 0,
5016           "pool": 3,
5017           "namespace": ""
5018         },
5019         "version": "27'1",
5020         "prior_version": "0'0",
5021         "last_reqid": "client.4192.0:1",
5022         "user_version": 1,
5023         "size": 7,
5024         "mtime": "2018-04-05 14:30:10.688009",
5025         "local_mtime": "2018-04-05 14:30:10.691774",
5026         "lost": 0,
5027         "flags": [
5028           "dirty",
5029           "data_digest"
5030         ],
5031         "truncate_seq": 0,
5032         "truncate_size": 0,
5033         "data_digest": "0x2ddbf8f5",
5034         "omap_digest": "0xffffffff",
5035         "expected_object_size": 0,
5036         "expected_write_size": 0,
5037         "alloc_hint_flags": 0,
5038         "manifest": {
5039           "type": 0
5040         },
5041         "watchers": {}
5042       },
5043       "union_shard_errors": [
5044         "read_error",
5045         "size_mismatch_info",
5046         "obj_size_info_mismatch"
5047       ],
5048       "errors": [
5049         "size_mismatch"
5050       ],
5051       "object": {
5052         "version": 1,
5053         "snap": "head",
5054         "locator": "",
5055         "nspace": "",
5056         "name": "EOBJ1"
5057       }
5058     },
5059     {
5060       "shards": [
5061         {
5062           "size": 2048,
5063           "errors": [
5064             "ec_hash_error"
5065           ],
5066           "shard": 2,
5067           "osd": 0,
5068           "primary": false
5069         },
5070         {
5071           "data_digest": "0x04cfa72f",
5072           "omap_digest": "0xffffffff",
5073           "size": 2048,
5074           "errors": [],
5075           "shard": 0,
5076           "osd": 1,
5077           "primary": true
5078         },
5079         {
5080           "data_digest": "0x04cfa72f",
5081           "omap_digest": "0xffffffff",
5082           "size": 2048,
5083           "errors": [],
5084           "shard": 1,
5085           "osd": 2,
5086           "primary": false
5087         }
5088       ],
5089       "selected_object_info": {
5090         "oid": {
5091           "oid": "EOBJ2",
5092           "key": "",
5093           "snapid": -2,
5094           "hash": 562812377,
5095           "max": 0,
5096           "pool": 3,
5097           "namespace": ""
5098         },
5099         "version": "33'2",
5100         "prior_version": "0'0",
5101         "last_reqid": "client.4224.0:1",
5102         "user_version": 2,
5103         "size": 7,
5104         "mtime": "2018-04-05 14:30:14.152945",
5105         "local_mtime": "2018-04-05 14:30:14.154014",
5106         "lost": 0,
5107         "flags": [
5108           "dirty",
5109           "data_digest"
5110         ],
5111         "truncate_seq": 0,
5112         "truncate_size": 0,
5113         "data_digest": "0x2ddbf8f5",
5114         "omap_digest": "0xffffffff",
5115         "expected_object_size": 0,
5116         "expected_write_size": 0,
5117         "alloc_hint_flags": 0,
5118         "manifest": {
5119           "type": 0
5120         },
5121         "watchers": {}
5122       },
5123       "union_shard_errors": [
5124         "ec_hash_error"
5125       ],
5126       "errors": [],
5127       "object": {
5128         "version": 2,
5129         "snap": "head",
5130         "locator": "",
5131         "nspace": "",
5132         "name": "EOBJ2"
5133       }
5134     },
5135     {
5136       "shards": [
5137         {
5138           "data_digest": "0x04cfa72f",
5139           "omap_digest": "0xffffffff",
5140           "size": 2048,
5141           "errors": [],
5142           "shard": 2,
5143           "osd": 0,
5144           "primary": false
5145         },
5146         {
5147           "osd": 1,
5148           "primary": true,
5149           "shard": 0,
5150           "errors": [
5151             "missing"
5152           ]
5153         },
5154         {
5155           "data_digest": "0x04cfa72f",
5156           "omap_digest": "0xffffffff",
5157           "size": 2048,
5158           "shard": 1,
5159           "errors": [],
5160           "osd": 2,
5161           "primary": false
5162         }
5163       ],
5164       "selected_object_info": {
5165         "oid": {
5166           "oid": "EOBJ3",
5167           "key": "",
5168           "snapid": -2,
5169           "hash": 3125668237,
5170           "max": 0,
5171           "pool": 3,
5172           "namespace": ""
5173         },
5174         "version": "39'3",
5175         "prior_version": "0'0",
5176         "last_reqid": "client.4258.0:1",
5177         "user_version": 3,
5178         "size": 7,
5179         "mtime": "2018-04-05 14:30:18.875544",
5180         "local_mtime": "2018-04-05 14:30:18.880153",
5181         "lost": 0,
5182         "flags": [
5183           "dirty",
5184           "data_digest"
5185         ],
5186         "truncate_seq": 0,
5187         "truncate_size": 0,
5188         "data_digest": "0x2ddbf8f5",
5189         "omap_digest": "0xffffffff",
5190         "expected_object_size": 0,
5191         "expected_write_size": 0,
5192         "alloc_hint_flags": 0,
5193         "manifest": {
5194           "type": 0
5195         },
5196         "watchers": {}
5197       },
5198       "union_shard_errors": [
5199         "missing"
5200       ],
5201       "errors": [],
5202       "object": {
5203         "version": 3,
5204         "snap": "head",
5205         "locator": "",
5206         "nspace": "",
5207         "name": "EOBJ3"
5208       }
5209     },
5210     {
5211       "shards": [
5212         {
5213           "attrs": [
5214             {
5215               "Base64": false,
5216               "value": "bad-val",
5217               "name": "key1-EOBJ4"
5218             },
5219             {
5220               "Base64": false,
5221               "value": "val2-EOBJ4",
5222               "name": "key2-EOBJ4"
5223             }
5224           ],
5225           "data_digest": "0x04cfa72f",
5226           "omap_digest": "0xffffffff",
5227           "size": 2048,
5228           "errors": [],
5229           "shard": 2,
5230           "osd": 0,
5231           "primary": false
5232         },
5233         {
5234           "osd": 1,
5235           "primary": true,
5236           "shard": 0,
5237           "errors": [],
5238           "size": 2048,
5239           "omap_digest": "0xffffffff",
5240           "data_digest": "0x04cfa72f",
5241           "attrs": [
5242             {
5243               "Base64": false,
5244               "value": "val1-EOBJ4",
5245               "name": "key1-EOBJ4"
5246             },
5247             {
5248               "Base64": false,
5249               "value": "val2-EOBJ4",
5250               "name": "key2-EOBJ4"
5251             }
5252           ]
5253         },
5254         {
5255           "osd": 2,
5256           "primary": false,
5257           "shard": 1,
5258           "errors": [],
5259           "size": 2048,
5260           "omap_digest": "0xffffffff",
5261           "data_digest": "0x04cfa72f",
5262           "attrs": [
5263             {
5264               "Base64": false,
5265               "value": "val1-EOBJ4",
5266               "name": "key1-EOBJ4"
5267             },
5268             {
5269               "Base64": false,
5270               "value": "val3-EOBJ4",
5271               "name": "key3-EOBJ4"
5272             }
5273           ]
5274         }
5275       ],
5276       "selected_object_info": {
5277         "oid": {
5278           "oid": "EOBJ4",
5279           "key": "",
5280           "snapid": -2,
5281           "hash": 1618759290,
5282           "max": 0,
5283           "pool": 3,
5284           "namespace": ""
5285         },
5286         "version": "45'6",
5287         "prior_version": "45'5",
5288         "last_reqid": "client.4296.0:1",
5289         "user_version": 6,
5290         "size": 7,
5291         "mtime": "2018-04-05 14:30:22.271983",
5292         "local_mtime": "2018-04-05 14:30:22.272840",
5293         "lost": 0,
5294         "flags": [
5295           "dirty",
5296           "data_digest"
5297         ],
5298         "truncate_seq": 0,
5299         "truncate_size": 0,
5300         "data_digest": "0x2ddbf8f5",
5301         "omap_digest": "0xffffffff",
5302         "expected_object_size": 0,
5303         "expected_write_size": 0,
5304         "alloc_hint_flags": 0,
5305         "manifest": {
5306           "type": 0
5307         },
5308         "watchers": {}
5309       },
5310       "union_shard_errors": [],
5311       "errors": [
5312         "attr_value_mismatch",
5313         "attr_name_mismatch"
5314       ],
5315       "object": {
5316         "version": 6,
5317         "snap": "head",
5318         "locator": "",
5319         "nspace": "",
5320         "name": "EOBJ4"
5321       }
5322     },
5323     {
5324       "shards": [
5325         {
5326           "data_digest": "0x04cfa72f",
5327           "omap_digest": "0xffffffff",
5328           "size": 2048,
5329           "errors": [],
5330           "shard": 2,
5331           "osd": 0,
5332           "primary": false
5333         },
5334         {
5335           "object_info": {
5336             "oid": {
5337               "oid": "EOBJ5",
5338               "key": "",
5339               "snapid": -2,
5340               "hash": 2918945441,
5341               "max": 0,
5342               "pool": 3,
5343               "namespace": ""
5344             },
5345             "version": "59'7",
5346             "prior_version": "0'0",
5347             "last_reqid": "client.4384.0:1",
5348             "user_version": 7,
5349             "size": 7,
5350             "mtime": "2018-04-05 14:30:35.162395",
5351             "local_mtime": "2018-04-05 14:30:35.166390",
5352             "lost": 0,
5353             "flags": [
5354               "dirty",
5355               "data_digest"
5356             ],
5357             "truncate_seq": 0,
5358             "truncate_size": 0,
5359             "data_digest": "0x2ddbf8f5",
5360             "omap_digest": "0xffffffff",
5361             "expected_object_size": 0,
5362             "expected_write_size": 0,
5363             "alloc_hint_flags": 0,
5364             "manifest": {
5365               "type": 0
5366             },
5367             "watchers": {}
5368           },
5369           "size": 4096,
5370           "shard": 0,
5371           "errors": [
5372             "size_mismatch_info",
5373             "ec_size_error",
5374             "obj_size_info_mismatch"
5375           ],
5376           "osd": 1,
5377           "primary": true
5378         },
5379         {
5380           "data_digest": "0x04cfa72f",
5381           "omap_digest": "0xffffffff",
5382           "size": 2048,
5383           "shard": 1,
5384           "errors": [],
5385           "osd": 2,
5386           "primary": false
5387         }
5388       ],
5389       "selected_object_info": {
5390         "oid": {
5391           "oid": "EOBJ5",
5392           "key": "",
5393           "snapid": -2,
5394           "hash": 2918945441,
5395           "max": 0,
5396           "pool": 3,
5397           "namespace": ""
5398         },
5399         "version": "59'7",
5400         "prior_version": "0'0",
5401         "last_reqid": "client.4384.0:1",
5402         "user_version": 7,
5403         "size": 7,
5404         "mtime": "2018-04-05 14:30:35.162395",
5405         "local_mtime": "2018-04-05 14:30:35.166390",
5406         "lost": 0,
5407         "flags": [
5408           "dirty",
5409           "data_digest"
5410         ],
5411         "truncate_seq": 0,
5412         "truncate_size": 0,
5413         "data_digest": "0x2ddbf8f5",
5414         "omap_digest": "0xffffffff",
5415         "expected_object_size": 0,
5416         "expected_write_size": 0,
5417         "alloc_hint_flags": 0,
5418         "manifest": {
5419           "type": 0
5420         },
5421         "watchers": {}
5422       },
5423       "union_shard_errors": [
5424         "size_mismatch_info",
5425         "ec_size_error",
5426         "obj_size_info_mismatch"
5427       ],
5428       "errors": [
5429         "size_mismatch"
5430       ],
5431       "object": {
5432         "version": 7,
5433         "snap": "head",
5434         "locator": "",
5435         "nspace": "",
5436         "name": "EOBJ5"
5437       }
5438     },
5439     {
5440       "object": {
5441         "name": "EOBJ6",
5442         "nspace": "",
5443         "locator": "",
5444         "snap": "head",
5445         "version": 8
5446       },
5447       "errors": [],
5448       "union_shard_errors": [
5449         "read_error",
5450         "hinfo_missing",
5451         "hinfo_corrupted"
5452       ],
5453       "selected_object_info": {
5454         "oid": {
5455           "oid": "EOBJ6",
5456           "key": "",
5457           "snapid": -2,
5458           "hash": 3050890866,
5459           "max": 0,
5460           "pool": 3,
5461           "namespace": ""
5462         },
5463         "version": "65'8",
5464         "prior_version": "0'0",
5465         "last_reqid": "client.4420.0:1",
5466         "user_version": 8,
5467         "size": 7,
5468         "mtime": "2018-04-05 14:30:40.914673",
5469         "local_mtime": "2018-04-05 14:30:40.917705",
5470         "lost": 0,
5471         "flags": [
5472           "dirty",
5473           "data_digest"
5474         ],
5475         "truncate_seq": 0,
5476         "truncate_size": 0,
5477         "data_digest": "0x2ddbf8f5",
5478         "omap_digest": "0xffffffff",
5479         "expected_object_size": 0,
5480         "expected_write_size": 0,
5481         "alloc_hint_flags": 0,
5482         "manifest": {
5483           "type": 0
5484         },
5485         "watchers": {}
5486       },
5487       "shards": [
5488         {
5489           "osd": 0,
5490           "primary": false,
5491           "shard": 2,
5492           "errors": [
5493             "read_error",
5494             "hinfo_missing"
5495           ],
5496           "size": 2048
5497         },
5498         {
5499           "osd": 1,
5500           "primary": true,
5501           "shard": 0,
5502           "errors": [
5503             "read_error",
5504             "hinfo_corrupted"
5505           ],
5506           "size": 2048,
5507           "hashinfo": "bad-val"
5508         },
5509         {
5510           "osd": 2,
5511           "primary": false,
5512           "shard": 1,
5513           "errors": [],
5514           "size": 2048,
5515           "omap_digest": "0xffffffff",
5516           "data_digest": "0x04cfa72f",
5517           "hashinfo": {
5518            "cumulative_shard_hashes": [
5519             {
5520               "hash": 80717615,
5521               "shard": 0
5522             },
5523             {
5524               "hash": 1534491824,
5525               "shard": 1
5526             },
5527             {
5528               "hash": 80717615,
5529               "shard": 2
5530             }
5531            ],
5532            "total_chunk_size": 2048
5533           }
5534         }
5535       ]
5536     },
5537     {
5538       "object": {
5539         "name": "EOBJ7",
5540         "nspace": "",
5541         "locator": "",
5542         "snap": "head",
5543         "version": 10
5544       },
5545       "errors": [
5546         "hinfo_inconsistency"
5547       ],
5548       "union_shard_errors": [
5549         "ec_hash_error"
5550       ],
5551       "selected_object_info": {
5552         "oid": {
5553           "oid": "EOBJ7",
5554           "key": "",
5555           "snapid": -2,
5556           "hash": 3258066308,
5557           "max": 0,
5558           "pool": 3,
5559           "namespace": ""
5560         },
5561         "version": "75'10",
5562         "prior_version": "75'9",
5563         "last_reqid": "client.4486.0:1",
5564         "user_version": 10,
5565         "size": 34,
5566         "mtime": "2018-04-05 14:30:50.995009",
5567         "local_mtime": "2018-04-05 14:30:50.996112",
5568         "lost": 0,
5569         "flags": [
5570           "dirty",
5571           "data_digest"
5572         ],
5573         "truncate_seq": 0,
5574         "truncate_size": 0,
5575         "data_digest": "0x136e4e27",
5576         "omap_digest": "0xffffffff",
5577         "expected_object_size": 0,
5578         "expected_write_size": 0,
5579         "alloc_hint_flags": 0,
5580         "manifest": {
5581           "type": 0
5582         },
5583         "watchers": {}
5584       },
5585       "shards": [
5586         {
5587           "osd": 0,
5588           "primary": false,
5589           "shard": 2,
5590           "errors": [
5591             "ec_hash_error"
5592           ],
5593           "size": 2048,
5594           "hashinfo": {
5595            "cumulative_shard_hashes": [
5596             {
5597               "hash": 80717615,
5598               "shard": 0
5599             },
5600             {
5601               "hash": 1534491824,
5602               "shard": 1
5603             },
5604             {
5605               "hash": 80717615,
5606               "shard": 2
5607             }
5608            ],
5609            "total_chunk_size": 2048
5610          }
5611         },
5612         {
5613           "osd": 1,
5614           "primary": true,
5615           "shard": 0,
5616           "errors": [],
5617           "size": 2048,
5618           "omap_digest": "0xffffffff",
5619           "data_digest": "0x5b7455a8",
5620           "hashinfo": {
5621            "cumulative_shard_hashes": [
5622             {
5623               "hash": 1534350760,
5624               "shard": 0
5625             },
5626             {
5627               "hash": 1534491824,
5628               "shard": 1
5629             },
5630             {
5631               "hash": 1534350760,
5632               "shard": 2
5633             }
5634            ],
5635            "total_chunk_size": 2048
5636           }
5637         },
5638         {
5639           "osd": 2,
5640           "primary": false,
5641           "shard": 1,
5642           "errors": [],
5643           "size": 2048,
5644           "omap_digest": "0xffffffff",
5645           "data_digest": "0x5b7455a8",
5646           "hashinfo": {
5647            "cumulative_shard_hashes": [
5648             {
5649               "hash": 1534350760,
5650               "shard": 0
5651             },
5652             {
5653               "hash": 1534491824,
5654               "shard": 1
5655             },
5656             {
5657               "hash": 1534350760,
5658               "shard": 2
5659             }
5660            ],
5661            "total_chunk_size": 2048
5662           }
5663         }
5664       ]
5665     }
5666   ],
5667   "epoch": 0
5668 }
5669 EOF
5670
5671     fi
5672
5673     jq "$jqfilter" $dir/json | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/csjson
5674     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
5675     if test $getjson = "yes"
5676     then
5677       if [ "$allow_overwrites" = "true" ]
5678       then
5679         num=4
5680       else
5681         num=5
5682       fi
5683       jq '.' $dir/json > save${num}.json
5684     fi
5685
5686     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
5687     then
5688       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
5689     fi
5690
5691     ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it
5692     teardown $dir || return 1
5693 }
5694
5695 function TEST_corrupt_scrub_erasure_appends() {
5696     corrupt_scrub_erasure $1 false
5697 }
5698
5699 function TEST_corrupt_scrub_erasure_overwrites() {
5700     if [ "$use_ec_overwrite" = "true" ]; then
5701         corrupt_scrub_erasure $1 true
5702     fi
5703 }
5704
5705 #
5706 # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost
5707 #
5708 function TEST_periodic_scrub_replicated() {
5709     local dir=$1
5710     local poolname=psr_pool
5711     local objname=POBJ
5712
5713     setup $dir || return 1
5714     run_mon $dir a --osd_pool_default_size=2 || return 1
5715     run_mgr $dir x || return 1
5716     local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
5717     ceph_osd_args+="--osd_scrub_backoff_ratio=0"
5718     run_osd $dir 0 $ceph_osd_args || return 1
5719     run_osd $dir 1 $ceph_osd_args || return 1
5720     create_rbd_pool || return 1
5721     wait_for_clean || return 1
5722
5723     create_pool $poolname 1 1 || return 1
5724     wait_for_clean || return 1
5725
5726     local osd=0
5727     add_something $dir $poolname $objname scrub || return 1
5728     local primary=$(get_primary $poolname $objname)
5729     local pg=$(get_pg $poolname $objname)
5730
5731     # Add deep-scrub only error
5732     local payload=UVWXYZ
5733     echo $payload > $dir/CORRUPT
5734     # Uses $ceph_osd_args for osd restart
5735     objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
5736
5737     # No scrub information available, so expect failure
5738     set -o pipefail
5739     !  rados list-inconsistent-obj $pg | jq '.' || return 1
5740     set +o pipefail
5741
5742     pg_deep_scrub $pg || return 1
5743
5744     # Make sure bad object found
5745     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5746
5747     flush_pg_stats
5748     local last_scrub=$(get_last_scrub_stamp $pg)
5749     # Fake a schedule scrub
5750     ceph tell $pg scrub || return 1
5751     # Wait for schedule regular scrub
5752     wait_for_scrub $pg "$last_scrub"
5753
5754     # It needed to be upgraded
5755     grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1
5756
5757     # Bad object still known
5758     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5759
5760     # Can't upgrade with this set
5761     ceph osd set nodeep-scrub
5762     # Let map change propagate to OSDs
5763     ceph tell osd.0 get_latest_osdmap
5764     flush_pg_stats
5765     sleep 5
5766
5767     # Fake a schedule scrub
5768     ceph tell $pg scrub || return 1
5769     # Wait for schedule regular scrub
5770     # to notice scrub and skip it
5771     local found=false
5772     for i in $(seq 14 -1 0)
5773     do
5774       sleep 1
5775       ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; }
5776       echo Time left: $i seconds
5777     done
5778     test $found = "true" || return 1
5779
5780     # Bad object still known
5781     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5782
5783     flush_pg_stats
5784     # Request a regular scrub and it will be done
5785     pg_scrub $pg
5786     grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1
5787
5788     # deep-scrub error is no longer present
5789     rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1
5790 }
5791
5792 function TEST_scrub_warning() {
5793     local dir=$1
5794     local poolname=psr_pool
5795     local objname=POBJ
5796     local scrubs=5
5797     local deep_scrubs=5
5798     local i1_day=86400
5799     local i7_days=$(calc $i1_day \* 7)
5800     local i14_days=$(calc $i1_day \* 14)
5801     local overdue=0.5
5802     local conf_overdue_seconds=$(calc $i7_days + $i1_day + \( $i7_days \* $overdue \) )
5803     local pool_overdue_seconds=$(calc $i14_days + $i1_day + \( $i14_days \* $overdue \) )
5804
5805     setup $dir || return 1
5806     run_mon $dir a --osd_pool_default_size=1 || return 1
5807     run_mgr $dir x --mon_warn_pg_not_scrubbed_ratio=${overdue} --mon_warn_pg_not_deep_scrubbed_ratio=${overdue} || return 1
5808     run_osd $dir 0 $ceph_osd_args --osd_scrub_backoff_ratio=0 || return 1
5809
5810     for i in $(seq 1 $(expr $scrubs + $deep_scrubs))
5811     do
5812       create_pool $poolname-$i 1 1 || return 1
5813       wait_for_clean || return 1
5814       if [ $i = "1" ];
5815       then
5816         ceph osd pool set $poolname-$i scrub_max_interval $i14_days
5817       fi
5818       if [ $i = $(expr $scrubs + 1) ];
5819       then
5820         ceph osd pool set $poolname-$i deep_scrub_interval $i14_days
5821       fi
5822     done
5823
5824     # Only 1 osd
5825     local primary=0
5826
5827     ceph osd set noscrub || return 1
5828     ceph osd set nodeep-scrub || return 1
5829     ceph config set global osd_scrub_interval_randomize_ratio 0
5830     ceph config set global osd_deep_scrub_randomize_ratio 0
5831     ceph config set global osd_scrub_max_interval ${i7_days}
5832     ceph config set global osd_deep_scrub_interval ${i7_days}
5833
5834     # Fake schedule scrubs
5835     for i in $(seq 1 $scrubs)
5836     do
5837       if [ $i = "1" ];
5838       then
5839         overdue_seconds=$pool_overdue_seconds
5840       else
5841         overdue_seconds=$conf_overdue_seconds
5842       fi
5843       ceph tell ${i}.0 scrub $(expr ${overdue_seconds} + ${i}00) || return 1
5844     done
5845     # Fake schedule deep scrubs
5846     for i in $(seq $(expr $scrubs + 1) $(expr $scrubs + $deep_scrubs))
5847     do
5848       if [ $i = "$(expr $scrubs + 1)" ];
5849       then
5850         overdue_seconds=$pool_overdue_seconds
5851       else
5852         overdue_seconds=$conf_overdue_seconds
5853       fi
5854       ceph tell ${i}.0 deep_scrub $(expr ${overdue_seconds} + ${i}00) || return 1
5855     done
5856     flush_pg_stats
5857
5858     ceph health
5859     ceph health detail
5860     ceph health | grep -q "$deep_scrubs pgs not deep-scrubbed in time" || return 1
5861     ceph health | grep -q "$scrubs pgs not scrubbed in time" || return 1
5862     COUNT=$(ceph health detail | grep "not scrubbed since" | wc -l)
5863     if [ "$COUNT" != $scrubs ]; then
5864       ceph health detail | grep "not scrubbed since"
5865       return 1
5866     fi
5867     COUNT=$(ceph health detail | grep "not deep-scrubbed since" | wc -l)
5868     if [ "$COUNT" != $deep_scrubs ]; then
5869       ceph health detail | grep "not deep-scrubbed since"
5870       return 1
5871     fi
5872     return 0
5873 }
5874
5875 #
5876 # Corrupt snapset in replicated pool
5877 #
5878 function TEST_corrupt_snapset_scrub_rep() {
5879     local dir=$1
5880     local poolname=csr_pool
5881     local total_objs=2
5882
5883     setup $dir || return 1
5884     run_mon $dir a --osd_pool_default_size=2 || return 1
5885     run_mgr $dir x || return 1
5886     run_osd $dir 0 || return 1
5887     run_osd $dir 1 || return 1
5888     create_rbd_pool || return 1
5889     wait_for_clean || return 1
5890
5891     create_pool foo 1 || return 1
5892     create_pool $poolname 1 1 || return 1
5893     wait_for_clean || return 1
5894
5895     for i in $(seq 1 $total_objs) ; do
5896         objname=ROBJ${i}
5897         add_something $dir $poolname $objname || return 1
5898
5899         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
5900         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
5901     done
5902
5903     local pg=$(get_pg $poolname ROBJ0)
5904     local primary=$(get_primary $poolname ROBJ0)
5905
5906     rados -p $poolname mksnap snap1
5907     echo -n head_of_snapshot_data > $dir/change
5908
5909     for i in $(seq 1 $total_objs) ; do
5910         objname=ROBJ${i}
5911
5912         # Alternate corruption between osd.0 and osd.1
5913         local osd=$(expr $i % 2)
5914
5915         case $i in
5916         1)
5917           rados --pool $poolname put $objname $dir/change
5918           objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
5919           ;;
5920
5921         2)
5922           rados --pool $poolname put $objname $dir/change
5923           objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
5924           ;;
5925
5926         esac
5927     done
5928     rm $dir/change
5929
5930     pg_scrub $pg
5931
5932     rados list-inconsistent-pg $poolname > $dir/json || return 1
5933     # Check pg count
5934     test $(jq '. | length' $dir/json) = "1" || return 1
5935     # Check pgid
5936     test $(jq -r '.[0]' $dir/json) = $pg || return 1
5937
5938     rados list-inconsistent-obj $pg > $dir/json || return 1
5939
5940     jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
5941 {
5942   "epoch": 34,
5943   "inconsistents": [
5944     {
5945       "object": {
5946         "name": "ROBJ1",
5947         "nspace": "",
5948         "locator": "",
5949         "snap": "head",
5950         "version": 8
5951       },
5952       "errors": [
5953         "snapset_inconsistency"
5954       ],
5955       "union_shard_errors": [],
5956       "selected_object_info": {
5957         "oid": {
5958           "oid": "ROBJ1",
5959           "key": "",
5960           "snapid": -2,
5961           "hash": 1454963827,
5962           "max": 0,
5963           "pool": 3,
5964           "namespace": ""
5965         },
5966         "version": "24'8",
5967         "prior_version": "21'3",
5968         "last_reqid": "client.4195.0:1",
5969         "user_version": 8,
5970         "size": 21,
5971         "mtime": "2018-04-05 14:35:43.286117",
5972         "local_mtime": "2018-04-05 14:35:43.288990",
5973         "lost": 0,
5974         "flags": [
5975           "dirty",
5976           "omap",
5977           "data_digest"
5978         ],
5979         "truncate_seq": 0,
5980         "truncate_size": 0,
5981         "data_digest": "0x53acb008",
5982         "omap_digest": "0xffffffff",
5983         "expected_object_size": 0,
5984         "expected_write_size": 0,
5985         "alloc_hint_flags": 0,
5986         "manifest": {
5987           "type": 0
5988         },
5989         "watchers": {}
5990       },
5991       "shards": [
5992         {
5993           "osd": 0,
5994           "primary": false,
5995           "errors": [],
5996           "size": 21,
5997           "snapset": {
5998             "clones": [
5999               {
6000                 "overlap": "[]",
6001                 "size": 7,
6002                 "snap": 1,
6003                 "snaps": [
6004                   1
6005                 ]
6006               }
6007             ],
6008             "seq": 1
6009           }
6010         },
6011         {
6012           "osd": 1,
6013           "primary": true,
6014           "errors": [],
6015           "size": 21,
6016           "snapset": {
6017             "clones": [],
6018             "seq": 0
6019           }
6020         }
6021       ]
6022     },
6023     {
6024       "object": {
6025         "name": "ROBJ2",
6026         "nspace": "",
6027         "locator": "",
6028         "snap": "head",
6029         "version": 10
6030       },
6031       "errors": [
6032         "snapset_inconsistency"
6033       ],
6034       "union_shard_errors": [],
6035       "selected_object_info": {
6036         "oid": {
6037           "oid": "ROBJ2",
6038           "key": "",
6039           "snapid": -2,
6040           "hash": 2026323607,
6041           "max": 0,
6042           "pool": 3,
6043           "namespace": ""
6044         },
6045         "version": "28'10",
6046         "prior_version": "23'6",
6047         "last_reqid": "client.4223.0:1",
6048         "user_version": 10,
6049         "size": 21,
6050         "mtime": "2018-04-05 14:35:48.326856",
6051         "local_mtime": "2018-04-05 14:35:48.328097",
6052         "lost": 0,
6053         "flags": [
6054           "dirty",
6055           "omap",
6056           "data_digest"
6057         ],
6058         "truncate_seq": 0,
6059         "truncate_size": 0,
6060         "data_digest": "0x53acb008",
6061         "omap_digest": "0xffffffff",
6062         "expected_object_size": 0,
6063         "expected_write_size": 0,
6064         "alloc_hint_flags": 0,
6065         "manifest": {
6066           "type": 0
6067         },
6068         "watchers": {}
6069       },
6070       "shards": [
6071         {
6072           "osd": 0,
6073           "primary": false,
6074           "errors": [],
6075           "size": 21,
6076           "snapset": {
6077             "clones": [],
6078             "seq": 0
6079           }
6080         },
6081         {
6082           "osd": 1,
6083           "primary": true,
6084           "errors": [],
6085           "size": 21,
6086           "snapset": {
6087             "clones": [
6088               {
6089                 "overlap": "[]",
6090                 "size": 7,
6091                 "snap": 1,
6092                 "snaps": [
6093                   1
6094                 ]
6095               }
6096             ],
6097             "seq": 1
6098           }
6099         }
6100       ]
6101     }
6102   ]
6103 }
6104 EOF
6105
6106     jq "$jqfilter" $dir/json | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/csjson
6107     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
6108     if test $getjson = "yes"
6109     then
6110         jq '.' $dir/json > save6.json
6111     fi
6112
6113     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
6114     then
6115       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
6116     fi
6117
6118     ERRORS=0
6119     declare -a err_strings
6120     err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid [0-9]*:.*:::ROBJ1:head : snapset inconsistent"
6121     err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid [0-9]*:.*:::ROBJ2:head : snapset inconsistent"
6122     err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 [0-9]*:.*:::ROBJ1:1 : is an unexpected clone"
6123     err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub : stat mismatch, got 3/4 objects, 1/2 clones, 3/4 dirty, 3/4 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 49/56 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
6124     err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 0 missing, 2 inconsistent objects"
6125     err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 4 errors"
6126
6127     for err_string in "${err_strings[@]}"
6128     do
6129         if ! grep -q "$err_string" $dir/osd.${primary}.log
6130         then
6131             echo "Missing log message '$err_string'"
6132             ERRORS=$(expr $ERRORS + 1)
6133         fi
6134     done
6135
6136     if [ $ERRORS != "0" ];
6137     then
6138         echo "TEST FAILED WITH $ERRORS ERRORS"
6139         return 1
6140     fi
6141
6142     ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it
6143     teardown $dir || return 1
6144 }
6145
6146 function TEST_request_scrub_priority() {
6147     local dir=$1
6148     local poolname=psr_pool
6149     local objname=POBJ
6150     local OBJECTS=64
6151     local PGS=8
6152
6153     setup $dir || return 1
6154     run_mon $dir a --osd_pool_default_size=1 || return 1
6155     run_mgr $dir x || return 1
6156     local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
6157     ceph_osd_args+="--osd_scrub_backoff_ratio=0"
6158     run_osd $dir 0 $ceph_osd_args || return 1
6159
6160     create_pool $poolname $PGS $PGS || return 1
6161     wait_for_clean || return 1
6162
6163     local osd=0
6164     add_something $dir $poolname $objname noscrub || return 1
6165     local primary=$(get_primary $poolname $objname)
6166     local pg=$(get_pg $poolname $objname)
6167     poolid=$(ceph osd dump | grep "^pool.*[']${poolname}[']" | awk '{ print $2 }')
6168
6169     local otherpgs
6170     for i in $(seq 0 $(expr $PGS - 1))
6171     do
6172         opg="${poolid}.${i}"
6173         if [ "$opg" = "$pg" ]; then
6174           continue
6175         fi
6176         otherpgs="${otherpgs}${opg} "
6177         local other_last_scrub=$(get_last_scrub_stamp $pg)
6178         # Fake a schedule scrub
6179         ceph tell $opg scrub $opg || return 1
6180     done
6181
6182     sleep 15
6183     flush_pg_stats
6184
6185     # Request a regular scrub and it will be done
6186     local last_scrub=$(get_last_scrub_stamp $pg)
6187     ceph pg scrub $pg
6188
6189     ceph osd unset noscrub || return 1
6190     ceph osd unset nodeep-scrub || return 1
6191
6192     wait_for_scrub $pg "$last_scrub"
6193
6194     for opg in $otherpgs $pg
6195     do
6196         wait_for_scrub $opg "$other_last_scrub"
6197     done
6198
6199     # Verify that the requested scrub ran first
6200     grep "log_channel.*scrub ok" $dir/osd.${primary}.log | grep -v purged_snaps | head -1 | sed 's/.*[[]DBG[]]//' | grep -q $pg || return 1
6201
6202     return 0
6203 }
6204
6205
6206 main osd-scrub-repair "$@"
6207
6208 # Local Variables:
6209 # compile-command: "cd build ; make -j4 && \
6210 #    ../qa/run-standalone.sh osd-scrub-repair.sh"
6211 # End: