ceph/qa/standalone/scrub/osd-scrub-repair.sh

   1 #!/bin/bash -x
   2 #
   3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
   4 #
   5 # Author: Loic Dachary <loic@dachary.org>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  18
  19 if [ `uname` = FreeBSD ]; then
  20     # erasure coding overwrites are only tested on Bluestore
  21     # erasure coding on filestore is unsafe
  22     # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites
  23     use_ec_overwrite=false
  24 else
  25     use_ec_overwrite=true
  26 fi
  27
  28 # Test development and debugging
  29 # Set to "yes" in order to ignore diff errors and save results to update test
  30 getjson="no"
  31
  32 # Ignore the epoch and filter out the attr '_' value because it has date information and won't match
  33 jqfilter='.inconsistents | (.[].shards[].attrs[]? | select(.name == "_") | .value) |= "----Stripped-by-test----"'
  34 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
  35
  36 # Remove items are not consistent across runs, the pg interval and client
  37 sedfilter='s/\([ ]*\"\(selected_\)*object_info\":.*head[(]\)[^[:space:]]* [^[:space:]]* \(.*\)/\1\3/'
  38
  39 function run() {
  40     local dir=$1
  41     shift
  42
  43     export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
  44     export CEPH_ARGS
  45     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  46     CEPH_ARGS+="--mon-host=$CEPH_MON "
  47
  48     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  49     for func in $funcs ; do
  50         $func $dir || return 1
  51     done
  52 }
  53
  54 function add_something() {
  55     local dir=$1
  56     local poolname=$2
  57     local obj=${3:-SOMETHING}
  58     local scrub=${4:-noscrub}
  59
  60     if [ "$scrub" = "noscrub" ];
  61     then
  62         ceph osd set noscrub || return 1
  63         ceph osd set nodeep-scrub || return 1
  64     else
  65         ceph osd unset noscrub || return 1
  66         ceph osd unset nodeep-scrub || return 1
  67     fi
  68
  69     local payload=ABCDEF
  70     echo $payload > $dir/ORIGINAL
  71     rados --pool $poolname put $obj $dir/ORIGINAL || return 1
  72 }
  73
  74 #
  75 # Corrupt one copy of a replicated pool
  76 #
  77 function TEST_corrupt_and_repair_replicated() {
  78     local dir=$1
  79     local poolname=rbd
  80
  81     setup $dir || return 1
  82     run_mon $dir a --osd_pool_default_size=2 || return 1
  83     run_mgr $dir x || return 1
  84     run_osd $dir 0 || return 1
  85     run_osd $dir 1 || return 1
  86     create_rbd_pool || return 1
  87     wait_for_clean || return 1
  88
  89     add_something $dir $poolname || return 1
  90     corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
  91     # Reproduces http://tracker.ceph.com/issues/8914
  92     corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1
  93
  94     teardown $dir || return 1
  95 }
  96
  97 function corrupt_and_repair_two() {
  98     local dir=$1
  99     local poolname=$2
 100     local first=$3
 101     local second=$4
 102
 103     #
 104     # 1) remove the corresponding file from the OSDs
 105     #
 106     pids=""
 107     run_in_background pids objectstore_tool $dir $first SOMETHING remove
 108     run_in_background pids objectstore_tool $dir $second SOMETHING remove
 109     wait_background pids
 110     return_code=$?
 111     if [ $return_code -ne 0 ]; then return $return_code; fi
 112
 113     #
 114     # 2) repair the PG
 115     #
 116     local pg=$(get_pg $poolname SOMETHING)
 117     repair $pg
 118     #
 119     # 3) The files must be back
 120     #
 121     pids=""
 122     run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs
 123     run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs
 124     wait_background pids
 125     return_code=$?
 126     if [ $return_code -ne 0 ]; then return $return_code; fi
 127
 128     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 129     diff $dir/ORIGINAL $dir/COPY || return 1
 130 }
 131
 132 #
 133 # 1) add an object
 134 # 2) remove the corresponding file from a designated OSD
 135 # 3) repair the PG
 136 # 4) check that the file has been restored in the designated OSD
 137 #
 138 function corrupt_and_repair_one() {
 139     local dir=$1
 140     local poolname=$2
 141     local osd=$3
 142
 143     #
 144     # 1) remove the corresponding file from the OSD
 145     #
 146     objectstore_tool $dir $osd SOMETHING remove || return 1
 147     #
 148     # 2) repair the PG
 149     #
 150     local pg=$(get_pg $poolname SOMETHING)
 151     repair $pg
 152     #
 153     # 3) The file must be back
 154     #
 155     objectstore_tool $dir $osd SOMETHING list-attrs || return 1
 156     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 157     diff $dir/ORIGINAL $dir/COPY || return 1
 158 }
 159
 160 function corrupt_and_repair_erasure_coded() {
 161     local dir=$1
 162     local poolname=$2
 163
 164     add_something $dir $poolname || return 1
 165
 166     local primary=$(get_primary $poolname SOMETHING)
 167     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 168     local not_primary_first=${osds[0]}
 169     local not_primary_second=${osds[1]}
 170
 171     # Reproduces http://tracker.ceph.com/issues/10017
 172     corrupt_and_repair_one $dir $poolname $primary  || return 1
 173     # Reproduces http://tracker.ceph.com/issues/10409
 174     corrupt_and_repair_one $dir $poolname $not_primary_first || return 1
 175     corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1
 176     corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1
 177
 178 }
 179
 180 function create_ec_pool() {
 181     local pool_name=$1
 182     local allow_overwrites=$2
 183
 184     ceph osd erasure-code-profile set myprofile crush-failure-domain=osd $3 $4 $5 $6 $7 || return 1
 185
 186     ceph osd pool create "$poolname" 1 1 erasure myprofile || return 1
 187
 188     if [ "$allow_overwrites" = "true" ]; then
 189         ceph osd pool set "$poolname" allow_ec_overwrites true || return 1
 190     fi
 191
 192     wait_for_clean || return 1
 193     return 0
 194 }
 195
 196 function auto_repair_erasure_coded() {
 197     local dir=$1
 198     local allow_overwrites=$2
 199     local poolname=ecpool
 200
 201     # Launch a cluster with 5 seconds scrub interval
 202     setup $dir || return 1
 203     run_mon $dir a || return 1
 204     run_mgr $dir x || return 1
 205     local ceph_osd_args="--osd-scrub-auto-repair=true \
 206             --osd-deep-scrub-interval=5 \
 207             --osd-scrub-max-interval=5 \
 208             --osd-scrub-min-interval=5 \
 209             --osd-scrub-interval-randomize-ratio=0"
 210     for id in $(seq 0 2) ; do
 211         if [ "$allow_overwrites" = "true" ]; then
 212             run_osd_bluestore $dir $id $ceph_osd_args || return 1
 213         else
 214             run_osd $dir $id $ceph_osd_args || return 1
 215         fi
 216     done
 217     create_rbd_pool || return 1
 218     wait_for_clean || return 1
 219
 220     # Create an EC pool
 221     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 222
 223     # Put an object
 224     local payload=ABCDEF
 225     echo $payload > $dir/ORIGINAL
 226     rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
 227
 228     # Remove the object from one shard physically
 229     # Restarted osd get $ceph_osd_args passed
 230     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
 231     # Wait for auto repair
 232     local pgid=$(get_pg $poolname SOMETHING)
 233     wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)"
 234     wait_for_clean || return 1
 235     # Verify - the file should be back
 236     # Restarted osd get $ceph_osd_args passed
 237     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
 238     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 239     diff $dir/ORIGINAL $dir/COPY || return 1
 240
 241     # Tear down
 242     teardown $dir || return 1
 243 }
 244
 245 function TEST_auto_repair_erasure_coded_appends() {
 246     auto_repair_erasure_coded $1 false
 247 }
 248
 249 function TEST_auto_repair_erasure_coded_overwrites() {
 250     if [ "$use_ec_overwrite" = "true" ]; then
 251         auto_repair_erasure_coded $1 true
 252     fi
 253 }
 254
 255 function corrupt_and_repair_jerasure() {
 256     local dir=$1
 257     local allow_overwrites=$2
 258     local poolname=ecpool
 259
 260     setup $dir || return 1
 261     run_mon $dir a || return 1
 262     run_mgr $dir x || return 1
 263     for id in $(seq 0 3) ; do
 264         if [ "$allow_overwrites" = "true" ]; then
 265             run_osd_bluestore $dir $id || return 1
 266         else
 267             run_osd $dir $id || return 1
 268         fi
 269     done
 270     create_rbd_pool || return 1
 271     wait_for_clean || return 1
 272
 273     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 274     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 275
 276     teardown $dir || return 1
 277 }
 278
 279 function TEST_corrupt_and_repair_jerasure_appends() {
 280     corrupt_and_repair_jerasure $1
 281 }
 282
 283 function TEST_corrupt_and_repair_jerasure_overwrites() {
 284     if [ "$use_ec_overwrite" = "true" ]; then
 285         corrupt_and_repair_jerasure $1 true
 286     fi
 287 }
 288
 289 function corrupt_and_repair_lrc() {
 290     local dir=$1
 291     local allow_overwrites=$2
 292     local poolname=ecpool
 293
 294     setup $dir || return 1
 295     run_mon $dir a || return 1
 296     run_mgr $dir x || return 1
 297     for id in $(seq 0 9) ; do
 298         if [ "$allow_overwrites" = "true" ]; then
 299             run_osd_bluestore $dir $id || return 1
 300         else
 301             run_osd $dir $id || return 1
 302         fi
 303     done
 304     create_rbd_pool || return 1
 305     wait_for_clean || return 1
 306
 307     create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1
 308     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 309
 310     teardown $dir || return 1
 311 }
 312
 313 function TEST_corrupt_and_repair_lrc_appends() {
 314     corrupt_and_repair_jerasure $1
 315 }
 316
 317 function TEST_corrupt_and_repair_lrc_overwrites() {
 318     if [ "$use_ec_overwrite" = "true" ]; then
 319         corrupt_and_repair_jerasure $1 true
 320     fi
 321 }
 322
 323 function unfound_erasure_coded() {
 324     local dir=$1
 325     local allow_overwrites=$2
 326     local poolname=ecpool
 327     local payload=ABCDEF
 328
 329     setup $dir || return 1
 330     run_mon $dir a || return 1
 331     run_mgr $dir x || return 1
 332     for id in $(seq 0 3) ; do
 333         if [ "$allow_overwrites" = "true" ]; then
 334             run_osd_bluestore $dir $id || return 1
 335         else
 336             run_osd $dir $id || return 1
 337         fi
 338     done
 339     create_rbd_pool || return 1
 340     wait_for_clean || return 1
 341
 342     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 343
 344     add_something $dir $poolname || return 1
 345
 346     local primary=$(get_primary $poolname SOMETHING)
 347     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 348     local not_primary_first=${osds[0]}
 349     local not_primary_second=${osds[1]}
 350     local not_primary_third=${osds[2]}
 351
 352     #
 353     # 1) remove the corresponding file from the OSDs
 354     #
 355     pids=""
 356     run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove
 357     run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove
 358     run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove
 359     wait_background pids
 360     return_code=$?
 361     if [ $return_code -ne 0 ]; then return $return_code; fi
 362
 363     #
 364     # 2) repair the PG
 365     #
 366     local pg=$(get_pg $poolname SOMETHING)
 367     repair $pg
 368     #
 369     # 3) check pg state
 370     #
 371     # it may take a bit to appear due to mon/mgr asynchrony
 372     for f in `seq 1 60`; do
 373         ceph -s | grep "1/1 unfound" && break
 374         sleep 1
 375     done
 376     ceph -s|grep "4 osds: 4 up, 4 in" || return 1
 377     ceph -s|grep "1/1 unfound" || return 1
 378
 379     teardown $dir || return 1
 380 }
 381
 382 function TEST_unfound_erasure_coded_appends() {
 383     unfound_erasure_coded $1
 384 }
 385
 386 function TEST_unfound_erasure_coded_overwrites() {
 387     if [ "$use_ec_overwrite" = "true" ]; then
 388         unfound_erasure_coded $1 true
 389     fi
 390 }
 391
 392 #
 393 # list_missing for EC pool
 394 #
 395 function list_missing_erasure_coded() {
 396     local dir=$1
 397     local allow_overwrites=$2
 398     local poolname=ecpool
 399
 400     setup $dir || return 1
 401     run_mon $dir a || return 1
 402     run_mgr $dir x || return 1
 403     for id in $(seq 0 2) ; do
 404         if [ "$allow_overwrites" = "true" ]; then
 405             run_osd_bluestore $dir $id || return 1
 406         else
 407             run_osd $dir $id || return 1
 408         fi
 409     done
 410     create_rbd_pool || return 1
 411     wait_for_clean || return 1
 412
 413     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 414
 415     # Put an object and remove the two shards (including primary)
 416     add_something $dir $poolname MOBJ0 || return 1
 417     local -a osds0=($(get_osds $poolname MOBJ0))
 418
 419     # Put another object and remove two shards (excluding primary)
 420     add_something $dir $poolname MOBJ1 || return 1
 421     local -a osds1=($(get_osds $poolname MOBJ1))
 422
 423     # Stop all osd daemons
 424     for id in $(seq 0 2) ; do
 425         kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1
 426     done
 427
 428     id=${osds0[0]}
 429     ceph-objectstore-tool --data-path $dir/$id \
 430         MOBJ0 remove || return 1
 431     id=${osds0[1]}
 432     ceph-objectstore-tool --data-path $dir/$id \
 433         MOBJ0 remove || return 1
 434
 435     id=${osds1[1]}
 436     ceph-objectstore-tool --data-path $dir/$id \
 437         MOBJ1 remove || return 1
 438     id=${osds1[2]}
 439     ceph-objectstore-tool --data-path $dir/$id \
 440         MOBJ1 remove || return 1
 441
 442     for id in $(seq 0 2) ; do
 443         activate_osd $dir $id >&2 || return 1
 444     done
 445     create_rbd_pool || return 1
 446     wait_for_clean || return 1
 447
 448     # Get get - both objects should in the same PG
 449     local pg=$(get_pg $poolname MOBJ0)
 450
 451     # Repair the PG, which triggers the recovering,
 452     # and should mark the object as unfound
 453     repair $pg
 454
 455     for i in $(seq 0 120) ; do
 456         [ $i -lt 60 ] || return 1
 457         matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l)
 458         [ $matches -eq 2 ] && break
 459     done
 460
 461     teardown $dir || return 1
 462 }
 463
 464 function TEST_list_missing_erasure_coded_appends() {
 465     list_missing_erasure_coded $1 false
 466 }
 467
 468 function TEST_list_missing_erasure_coded_overwrites() {
 469     if [ "$use_ec_overwrite" = "true" ]; then
 470         list_missing_erasure_coded $1 true
 471     fi
 472 }
 473
 474 #
 475 # Corrupt one copy of a replicated pool
 476 #
 477 function TEST_corrupt_scrub_replicated() {
 478     local dir=$1
 479     local poolname=csr_pool
 480     local total_objs=15
 481
 482     setup $dir || return 1
 483     run_mon $dir a --osd_pool_default_size=2 || return 1
 484     run_mgr $dir x || return 1
 485     run_osd $dir 0 || return 1
 486     run_osd $dir 1 || return 1
 487     create_rbd_pool || return 1
 488     wait_for_clean || return 1
 489
 490     ceph osd pool create foo 1 || return 1
 491     ceph osd pool create $poolname 1 1 || return 1
 492     wait_for_clean || return 1
 493
 494     for i in $(seq 1 $total_objs) ; do
 495         objname=ROBJ${i}
 496         add_something $dir $poolname $objname || return 1
 497
 498         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
 499         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
 500     done
 501
 502     local pg=$(get_pg $poolname ROBJ0)
 503
 504     # Compute an old omap digest and save oi
 505     CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \
 506         config set osd_deep_scrub_update_digest_min_age 0
 507     CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \
 508         config set osd_deep_scrub_update_digest_min_age 0
 509     pg_deep_scrub $pg
 510
 511     for i in $(seq 1 $total_objs) ; do
 512         objname=ROBJ${i}
 513
 514         # Alternate corruption between osd.0 and osd.1
 515         local osd=$(expr $i % 2)
 516
 517         case $i in
 518         1)
 519             # Size (deep scrub data_digest too)
 520             local payload=UVWXYZZZ
 521             echo $payload > $dir/CORRUPT
 522             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 523             ;;
 524
 525         2)
 526             # digest (deep scrub only)
 527             local payload=UVWXYZ
 528             echo $payload > $dir/CORRUPT
 529             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 530             ;;
 531
 532         3)
 533              # missing
 534              objectstore_tool $dir $osd $objname remove || return 1
 535              ;;
 536
 537          4)
 538              # Modify omap value (deep scrub only)
 539              objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1
 540              ;;
 541
 542          5)
 543             # Delete omap key (deep scrub only)
 544             objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1
 545             ;;
 546
 547          6)
 548             # Add extra omap key (deep scrub only)
 549             echo extra > $dir/extra-val
 550             objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1
 551             rm $dir/extra-val
 552             ;;
 553
 554          7)
 555             # Modify omap header (deep scrub only)
 556             echo -n newheader > $dir/hdr
 557             objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1
 558             rm $dir/hdr
 559             ;;
 560
 561          8)
 562             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
 563             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
 564
 565             # Break xattrs
 566             echo -n bad-val > $dir/bad-val
 567             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
 568             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
 569             echo -n val3-$objname > $dir/newval
 570             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
 571             rm $dir/bad-val $dir/newval
 572             ;;
 573
 574         9)
 575             objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi
 576             echo -n D > $dir/change
 577             rados --pool $poolname put $objname $dir/change
 578             objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi
 579             rm $dir/oi $dir/change
 580             ;;
 581
 582           # ROBJ10 must be handled after digests are re-computed by a deep scrub below
 583           # ROBJ11 must be handled with config change before deep scrub
 584           # ROBJ12 must be handled with config change before scrubs
 585           # ROBJ13 must be handled before scrubs
 586
 587         14)
 588             echo -n bad-val > $dir/bad-val
 589             objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1
 590             objectstore_tool $dir 1 $objname rm-attr _ || return 1
 591             rm $dir/bad-val
 592             ;;
 593
 594         15)
 595             objectstore_tool $dir $osd $objname rm-attr _ || return 1
 596
 597         esac
 598     done
 599
 600     local pg=$(get_pg $poolname ROBJ0)
 601
 602     set_config osd 0 filestore_debug_inject_read_err true || return 1
 603     set_config osd 1 filestore_debug_inject_read_err true || return 1
 604     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \
 605              injectdataerr $poolname ROBJ11 || return 1
 606     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \
 607              injectmdataerr $poolname ROBJ12 || return 1
 608     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \
 609              injectmdataerr $poolname ROBJ13 || return 1
 610     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \
 611              injectdataerr $poolname ROBJ13 || return 1
 612
 613     pg_scrub $pg
 614
 615     rados list-inconsistent-pg $poolname > $dir/json || return 1
 616     # Check pg count
 617     test $(jq '. | length' $dir/json) = "1" || return 1
 618     # Check pgid
 619     test $(jq -r '.[0]' $dir/json) = $pg || return 1
 620
 621     rados list-inconsistent-obj $pg > $dir/json || return 1
 622     # Get epoch for repair-get requests
 623     epoch=$(jq .epoch $dir/json)
 624
 625     jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
 626 {
 627   "inconsistents": [
 628     {
 629       "shards": [
 630         {
 631           "size": 7,
 632           "errors": [],
 633           "osd": 0
 634         },
 635         {
 636           "size": 9,
 637           "errors": [
 638             "size_mismatch_oi"
 639           ],
 640           "osd": 1
 641         }
 642       ],
 643       "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])",
 644       "union_shard_errors": [
 645         "size_mismatch_oi"
 646       ],
 647       "errors": [
 648         "size_mismatch"
 649       ],
 650       "object": {
 651         "version": 3,
 652         "snap": "head",
 653         "locator": "",
 654         "nspace": "",
 655         "name": "ROBJ1"
 656       }
 657     },
 658     {
 659       "shards": [
 660         {
 661           "errors": [
 662             "stat_error"
 663           ],
 664           "osd": 0
 665         },
 666         {
 667           "size": 7,
 668           "errors": [],
 669           "osd": 1
 670         }
 671       ],
 672       "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])",
 673       "union_shard_errors": [
 674         "stat_error"
 675       ],
 676       "errors": [],
 677       "object": {
 678         "version": 36,
 679         "snap": "head",
 680         "locator": "",
 681         "nspace": "",
 682         "name": "ROBJ12"
 683       }
 684     },
 685     {
 686       "shards": [
 687         {
 688           "errors": [
 689             "stat_error"
 690           ],
 691           "osd": 0
 692         },
 693         {
 694           "size": 7,
 695           "errors": [],
 696           "osd": 1
 697         }
 698       ],
 699       "selected_object_info": "3:d60617f9:::ROBJ13:head(47'55 osd.0.0:54 dirty|omap|data_digest|omap_digest s 7 uv 39 dd 2ddbf8f5 od 6441854d alloc_hint [0 0 0])",
 700       "union_shard_errors": [
 701         "stat_error"
 702       ],
 703       "errors": [],
 704       "object": {
 705         "version": 39,
 706         "snap": "head",
 707         "locator": "",
 708         "nspace": "",
 709         "name": "ROBJ13"
 710       }
 711     },
 712     {
 713       "shards": [
 714         {
 715           "size": 7,
 716           "errors": [
 717             "oi_attr_corrupted"
 718           ],
 719           "osd": 0
 720         },
 721         {
 722           "size": 7,
 723           "errors": [
 724             "oi_attr_missing"
 725           ],
 726           "osd": 1
 727         }
 728       ],
 729       "union_shard_errors": [
 730         "oi_attr_missing",
 731         "oi_attr_corrupted"
 732       ],
 733       "errors": [],
 734       "object": {
 735         "version": 0,
 736         "snap": "head",
 737         "locator": "",
 738         "nspace": "",
 739         "name": "ROBJ14"
 740       }
 741     },
 742     {
 743       "shards": [
 744         {
 745           "attrs": [
 746             {
 747               "Base64": true,
 748               "value": "",
 749               "name": "_"
 750             },
 751             {
 752               "Base64": true,
 753               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 754               "name": "snapset"
 755             }
 756           ],
 757           "size": 7,
 758           "errors": [],
 759           "osd": 0
 760         },
 761         {
 762           "attrs": [
 763             {
 764               "Base64": true,
 765               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 766               "name": "snapset"
 767             }
 768           ],
 769           "size": 7,
 770           "errors": [
 771             "oi_attr_missing"
 772           ],
 773           "osd": 1
 774         }
 775       ],
 776       "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])",
 777       "union_shard_errors": [
 778         "oi_attr_missing"
 779       ],
 780       "errors": [
 781         "attr_name_mismatch"
 782       ],
 783       "object": {
 784         "version": 45,
 785         "snap": "head",
 786         "locator": "",
 787         "nspace": "",
 788         "name": "ROBJ15"
 789       }
 790     },
 791     {
 792       "shards": [
 793         {
 794           "size": 7,
 795           "errors": [],
 796           "osd": 0
 797         },
 798         {
 799           "errors": [
 800             "missing"
 801           ],
 802           "osd": 1
 803         }
 804       ],
 805       "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])",
 806       "union_shard_errors": [
 807         "missing"
 808       ],
 809       "errors": [],
 810       "object": {
 811         "version": 9,
 812         "snap": "head",
 813         "locator": "",
 814         "nspace": "",
 815         "name": "ROBJ3"
 816       }
 817     },
 818     {
 819       "shards": [
 820         {
 821           "attrs": [
 822             {
 823               "Base64": true,
 824               "value": "",
 825               "name": "_"
 826             },
 827             {
 828               "Base64": false,
 829               "value": "bad-val",
 830               "name": "_key1-ROBJ8"
 831             },
 832             {
 833               "Base64": false,
 834               "value": "val3-ROBJ8",
 835               "name": "_key3-ROBJ8"
 836             },
 837             {
 838               "Base64": true,
 839               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 840               "name": "snapset"
 841             }
 842           ],
 843           "size": 7,
 844           "errors": [],
 845           "osd": 0
 846         },
 847         {
 848           "attrs": [
 849             {
 850               "Base64": true,
 851               "value": "",
 852               "name": "_"
 853             },
 854             {
 855               "Base64": false,
 856               "value": "val1-ROBJ8",
 857               "name": "_key1-ROBJ8"
 858             },
 859             {
 860               "Base64": false,
 861               "value": "val2-ROBJ8",
 862               "name": "_key2-ROBJ8"
 863             },
 864             {
 865               "Base64": true,
 866               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 867               "name": "snapset"
 868             }
 869           ],
 870           "size": 7,
 871           "errors": [],
 872           "osd": 1
 873         }
 874       ],
 875       "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 62 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
 876       "union_shard_errors": [],
 877       "errors": [
 878         "attr_value_mismatch",
 879         "attr_name_mismatch"
 880       ],
 881       "object": {
 882         "version": 62,
 883         "snap": "head",
 884         "locator": "",
 885         "nspace": "",
 886         "name": "ROBJ8"
 887       }
 888     },
 889     {
 890       "shards": [
 891         {
 892           "attrs": [
 893             {
 894               "Base64": true,
 895               "value": "",
 896               "name": "_"
 897             },
 898             {
 899               "Base64": true,
 900               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 901               "name": "snapset"
 902             }
 903           ],
 904           "object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 63 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
 905           "size": 1,
 906           "errors": [],
 907           "osd": 0
 908         },
 909         {
 910           "attrs": [
 911             {
 912               "Base64": true,
 913               "value": "",
 914               "name": "_"
 915             },
 916             {
 917               "Base64": true,
 918               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 919               "name": "snapset"
 920             }
 921           ],
 922           "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])",
 923           "size": 1,
 924           "errors": [],
 925           "osd": 1
 926         }
 927       ],
 928       "selected_object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 63 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
 929       "union_shard_errors": [],
 930       "errors": [
 931         "object_info_inconsistency",
 932         "attr_value_mismatch"
 933       ],
 934       "object": {
 935         "version": 63,
 936         "snap": "head",
 937         "locator": "",
 938         "nspace": "",
 939         "name": "ROBJ9"
 940       }
 941     }
 942   ],
 943   "epoch": 0
 944 }
 945 EOF
 946
 947     jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
 948     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
 949     if test $getjson = "yes"
 950     then
 951         jq '.' $dir/json > save1.json
 952     fi
 953
 954     if which jsonschema > /dev/null;
 955     then
 956       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
 957     fi
 958
 959     objname=ROBJ9
 960     # Change data and size again because digest was recomputed
 961     echo -n ZZZ > $dir/change
 962     rados --pool $poolname put $objname $dir/change
 963     # Set one to an even older value
 964     objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi
 965     rm $dir/oi $dir/change
 966
 967     objname=ROBJ10
 968     objectstore_tool $dir 1 $objname get-attr _ > $dir/oi
 969     rados --pool $poolname setomapval $objname key2-$objname val2-$objname
 970     objectstore_tool $dir 0 $objname set-attr _ $dir/oi
 971     objectstore_tool $dir 1 $objname set-attr _ $dir/oi
 972     rm $dir/oi
 973
 974     set_config osd 0 filestore_debug_inject_read_err true || return 1
 975     set_config osd 1 filestore_debug_inject_read_err true || return 1
 976     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \
 977              injectdataerr $poolname ROBJ11 || return 1
 978     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \
 979              injectmdataerr $poolname ROBJ12 || return 1
 980     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \
 981              injectmdataerr $poolname ROBJ13 || return 1
 982     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \
 983              injectdataerr $poolname ROBJ13 || return 1
 984     pg_deep_scrub $pg
 985
 986     rados list-inconsistent-pg $poolname > $dir/json || return 1
 987     # Check pg count
 988     test $(jq '. | length' $dir/json) = "1" || return 1
 989     # Check pgid
 990     test $(jq -r '.[0]' $dir/json) = $pg || return 1
 991
 992     rados list-inconsistent-obj $pg > $dir/json || return 1
 993     # Get epoch for repair-get requests
 994     epoch=$(jq .epoch $dir/json)
 995
 996     jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
 997 {
 998   "inconsistents": [
 999     {
1000       "shards": [
1001         {
1002           "data_digest": "0x2ddbf8f5",
1003           "omap_digest": "0xf5fba2c6",
1004           "size": 7,
1005           "errors": [],
1006           "osd": 0
1007         },
1008         {
1009           "data_digest": "0x2d4a11c2",
1010           "omap_digest": "0xf5fba2c6",
1011           "size": 9,
1012           "errors": [
1013             "data_digest_mismatch_oi",
1014             "size_mismatch_oi"
1015           ],
1016           "osd": 1
1017         }
1018       ],
1019       "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])",
1020       "union_shard_errors": [
1021         "data_digest_mismatch_oi",
1022         "size_mismatch_oi"
1023       ],
1024       "errors": [
1025         "data_digest_mismatch",
1026         "size_mismatch"
1027       ],
1028       "object": {
1029         "version": 3,
1030         "snap": "head",
1031         "locator": "",
1032         "nspace": "",
1033         "name": "ROBJ1"
1034       }
1035     },
1036     {
1037       "shards": [
1038         {
1039           "data_digest": "0x2ddbf8f5",
1040           "omap_digest": "0xa8dd5adc",
1041           "size": 7,
1042           "errors": [
1043             "omap_digest_mismatch_oi"
1044           ],
1045           "osd": 0
1046         },
1047         {
1048           "data_digest": "0x2ddbf8f5",
1049           "omap_digest": "0xa8dd5adc",
1050           "size": 7,
1051           "errors": [
1052             "omap_digest_mismatch_oi"
1053           ],
1054           "osd": 1
1055         }
1056       ],
1057       "selected_object_info": "3:b1f19cbd:::ROBJ10:head(47'51 osd.0.0:50 dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [0 0 0])",
1058       "union_shard_errors": [
1059         "omap_digest_mismatch_oi"
1060       ],
1061       "errors": [],
1062       "object": {
1063         "version": 30,
1064         "snap": "head",
1065         "locator": "",
1066         "nspace": "",
1067         "name": "ROBJ10"
1068       }
1069     },
1070     {
1071       "shards": [
1072         {
1073           "data_digest": "0x2ddbf8f5",
1074           "omap_digest": "0xa03cef03",
1075           "size": 7,
1076           "errors": [],
1077           "osd": 0
1078         },
1079         {
1080           "size": 7,
1081           "errors": [
1082             "read_error"
1083           ],
1084           "osd": 1
1085         }
1086       ],
1087       "selected_object_info": "3:87abbf36:::ROBJ11:head(47'48 osd.0.0:47 dirty|omap|data_digest|omap_digest s 7 uv 33 dd 2ddbf8f5 od a03cef03 alloc_hint [0 0 0])",
1088       "union_shard_errors": [
1089         "read_error"
1090       ],
1091       "errors": [],
1092       "object": {
1093         "version": 33,
1094         "snap": "head",
1095         "locator": "",
1096         "nspace": "",
1097         "name": "ROBJ11"
1098       }
1099     },
1100     {
1101       "shards": [
1102         {
1103           "errors": [
1104             "stat_error"
1105           ],
1106           "osd": 0
1107         },
1108         {
1109           "data_digest": "0x2ddbf8f5",
1110           "omap_digest": "0x067f306a",
1111           "size": 7,
1112           "errors": [],
1113           "osd": 1
1114         }
1115       ],
1116       "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])",
1117       "union_shard_errors": [
1118         "stat_error"
1119       ],
1120       "errors": [],
1121       "object": {
1122         "version": 36,
1123         "snap": "head",
1124         "locator": "",
1125         "nspace": "",
1126         "name": "ROBJ12"
1127       }
1128     },
1129     {
1130       "shards": [
1131         {
1132           "errors": [
1133             "stat_error"
1134           ],
1135           "osd": 0
1136         },
1137         {
1138           "size": 7,
1139           "errors": [
1140             "read_error"
1141           ],
1142           "osd": 1
1143         }
1144       ],
1145       "union_shard_errors": [
1146         "stat_error",
1147         "read_error"
1148       ],
1149       "errors": [],
1150       "object": {
1151         "version": 0,
1152         "snap": "head",
1153         "locator": "",
1154         "nspace": "",
1155         "name": "ROBJ13"
1156       }
1157     },
1158     {
1159       "shards": [
1160         {
1161           "data_digest": "0x2ddbf8f5",
1162           "omap_digest": "0x4f14f849",
1163           "size": 7,
1164           "errors": [
1165             "oi_attr_corrupted"
1166           ],
1167           "osd": 0
1168         },
1169         {
1170           "data_digest": "0x2ddbf8f5",
1171           "omap_digest": "0x4f14f849",
1172           "size": 7,
1173           "errors": [
1174             "oi_attr_missing"
1175           ],
1176           "osd": 1
1177         }
1178       ],
1179       "union_shard_errors": [
1180         "oi_attr_missing",
1181         "oi_attr_corrupted"
1182       ],
1183       "errors": [],
1184       "object": {
1185         "version": 0,
1186         "snap": "head",
1187         "locator": "",
1188         "nspace": "",
1189         "name": "ROBJ14"
1190       }
1191     },
1192     {
1193       "shards": [
1194         {
1195           "attrs": [
1196             {
1197               "Base64": true,
1198               "value": "",
1199               "name": "_"
1200             },
1201             {
1202               "Base64": true,
1203               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1204               "name": "snapset"
1205             }
1206           ],
1207           "data_digest": "0x2ddbf8f5",
1208           "omap_digest": "0x2d2a4d6e",
1209           "size": 7,
1210           "errors": [],
1211           "osd": 0
1212         },
1213         {
1214           "attrs": [
1215             {
1216               "Base64": true,
1217               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1218               "name": "snapset"
1219             }
1220           ],
1221           "data_digest": "0x2ddbf8f5",
1222           "omap_digest": "0x2d2a4d6e",
1223           "size": 7,
1224           "errors": [
1225             "oi_attr_missing"
1226           ],
1227           "osd": 1
1228         }
1229       ],
1230       "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])",
1231       "union_shard_errors": [
1232         "oi_attr_missing"
1233       ],
1234       "errors": [
1235         "attr_name_mismatch"
1236       ],
1237       "object": {
1238         "version": 45,
1239         "snap": "head",
1240         "locator": "",
1241         "nspace": "",
1242         "name": "ROBJ15"
1243       }
1244     },
1245     {
1246       "shards": [
1247         {
1248           "data_digest": "0x578a4830",
1249           "omap_digest": "0xf8e11918",
1250           "size": 7,
1251           "errors": [
1252             "data_digest_mismatch_oi"
1253           ],
1254           "osd": 0
1255         },
1256         {
1257           "data_digest": "0x2ddbf8f5",
1258           "omap_digest": "0xf8e11918",
1259           "size": 7,
1260           "errors": [],
1261           "osd": 1
1262         }
1263       ],
1264       "selected_object_info": "3:e97ce31e:::ROBJ2:head(47'56 osd.0.0:55 dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [0 0 0])",
1265       "union_shard_errors": [
1266         "data_digest_mismatch_oi"
1267       ],
1268       "errors": [
1269         "data_digest_mismatch"
1270       ],
1271       "object": {
1272         "version": 6,
1273         "snap": "head",
1274         "locator": "",
1275         "nspace": "",
1276         "name": "ROBJ2"
1277       }
1278     },
1279     {
1280       "shards": [
1281         {
1282           "data_digest": "0x2ddbf8f5",
1283           "omap_digest": "0x00b35dfd",
1284           "size": 7,
1285           "errors": [],
1286           "osd": 0
1287         },
1288         {
1289           "errors": [
1290             "missing"
1291           ],
1292           "osd": 1
1293         }
1294       ],
1295       "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])",
1296       "union_shard_errors": [
1297         "missing"
1298       ],
1299       "errors": [],
1300       "object": {
1301         "version": 9,
1302         "snap": "head",
1303         "locator": "",
1304         "nspace": "",
1305         "name": "ROBJ3"
1306       }
1307     },
1308     {
1309       "shards": [
1310         {
1311           "data_digest": "0x2ddbf8f5",
1312           "omap_digest": "0xd7178dfe",
1313           "size": 7,
1314           "errors": [
1315             "omap_digest_mismatch_oi"
1316           ],
1317           "osd": 0
1318         },
1319         {
1320           "data_digest": "0x2ddbf8f5",
1321           "omap_digest": "0xe2d46ea4",
1322           "size": 7,
1323           "errors": [],
1324           "osd": 1
1325         }
1326       ],
1327       "selected_object_info": "3:f4981d31:::ROBJ4:head(47'58 osd.0.0:57 dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [0 0 0])",
1328       "union_shard_errors": [
1329         "omap_digest_mismatch_oi"
1330       ],
1331       "errors": [
1332         "omap_digest_mismatch"
1333       ],
1334       "object": {
1335         "version": 12,
1336         "snap": "head",
1337         "locator": "",
1338         "nspace": "",
1339         "name": "ROBJ4"
1340       }
1341     },
1342     {
1343       "shards": [
1344         {
1345           "data_digest": "0x2ddbf8f5",
1346           "omap_digest": "0x1a862a41",
1347           "size": 7,
1348           "errors": [],
1349           "osd": 0
1350         },
1351         {
1352           "data_digest": "0x2ddbf8f5",
1353           "omap_digest": "0x06cac8f6",
1354           "size": 7,
1355           "errors": [
1356             "omap_digest_mismatch_oi"
1357           ],
1358           "osd": 1
1359         }
1360       ],
1361       "selected_object_info": "3:f4bfd4d1:::ROBJ5:head(47'59 osd.0.0:58 dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [0 0 0])",
1362       "union_shard_errors": [
1363         "omap_digest_mismatch_oi"
1364       ],
1365       "errors": [
1366         "omap_digest_mismatch"
1367       ],
1368       "object": {
1369         "version": 15,
1370         "snap": "head",
1371         "locator": "",
1372         "nspace": "",
1373         "name": "ROBJ5"
1374       }
1375     },
1376     {
1377       "shards": [
1378         {
1379           "data_digest": "0x2ddbf8f5",
1380           "omap_digest": "0x689ee887",
1381           "size": 7,
1382           "errors": [
1383             "omap_digest_mismatch_oi"
1384           ],
1385           "osd": 0
1386         },
1387         {
1388           "data_digest": "0x2ddbf8f5",
1389           "omap_digest": "0x179c919f",
1390           "size": 7,
1391           "errors": [],
1392           "osd": 1
1393         }
1394       ],
1395       "selected_object_info": "3:a53c12e8:::ROBJ6:head(47'50 osd.0.0:49 dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [0 0 0])",
1396       "union_shard_errors": [
1397         "omap_digest_mismatch_oi"
1398       ],
1399       "errors": [
1400         "omap_digest_mismatch"
1401       ],
1402       "object": {
1403         "version": 18,
1404         "snap": "head",
1405         "locator": "",
1406         "nspace": "",
1407         "name": "ROBJ6"
1408       }
1409     },
1410     {
1411       "shards": [
1412         {
1413           "data_digest": "0x2ddbf8f5",
1414           "omap_digest": "0xefced57a",
1415           "size": 7,
1416           "errors": [],
1417           "osd": 0
1418         },
1419         {
1420           "data_digest": "0x2ddbf8f5",
1421           "omap_digest": "0x6a73cc07",
1422           "size": 7,
1423           "errors": [
1424             "omap_digest_mismatch_oi"
1425           ],
1426           "osd": 1
1427         }
1428       ],
1429       "selected_object_info": "3:8b55fa4b:::ROBJ7:head(47'49 osd.0.0:48 dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [0 0 0])",
1430       "union_shard_errors": [
1431         "omap_digest_mismatch_oi"
1432       ],
1433       "errors": [
1434         "omap_digest_mismatch"
1435       ],
1436       "object": {
1437         "version": 21,
1438         "snap": "head",
1439         "locator": "",
1440         "nspace": "",
1441         "name": "ROBJ7"
1442       }
1443     },
1444     {
1445       "shards": [
1446         {
1447           "attrs": [
1448             {
1449               "Base64": true,
1450               "value": "",
1451               "name": "_"
1452             },
1453             {
1454               "Base64": false,
1455               "value": "bad-val",
1456               "name": "_key1-ROBJ8"
1457             },
1458             {
1459               "Base64": false,
1460               "value": "val3-ROBJ8",
1461               "name": "_key3-ROBJ8"
1462             },
1463             {
1464               "Base64": true,
1465               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1466               "name": "snapset"
1467             }
1468           ],
1469           "data_digest": "0x2ddbf8f5",
1470           "omap_digest": "0xd6be81dc",
1471           "size": 7,
1472           "errors": [],
1473           "osd": 0
1474         },
1475         {
1476           "attrs": [
1477             {
1478               "Base64": true,
1479               "value": "",
1480               "name": "_"
1481             },
1482             {
1483               "Base64": false,
1484               "value": "val1-ROBJ8",
1485               "name": "_key1-ROBJ8"
1486             },
1487             {
1488               "Base64": false,
1489               "value": "val2-ROBJ8",
1490               "name": "_key2-ROBJ8"
1491             },
1492             {
1493               "Base64": true,
1494               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1495               "name": "snapset"
1496             }
1497           ],
1498           "data_digest": "0x2ddbf8f5",
1499           "omap_digest": "0xd6be81dc",
1500           "size": 7,
1501           "errors": [],
1502           "osd": 1
1503         }
1504       ],
1505       "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 62 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
1506       "union_shard_errors": [],
1507       "errors": [
1508         "attr_value_mismatch",
1509         "attr_name_mismatch"
1510       ],
1511       "object": {
1512         "version": 62,
1513         "snap": "head",
1514         "locator": "",
1515         "nspace": "",
1516         "name": "ROBJ8"
1517       }
1518     },
1519     {
1520       "shards": [
1521         {
1522           "attrs": [
1523             {
1524               "Base64": true,
1525               "value": "",
1526               "name": "_"
1527             },
1528             {
1529               "Base64": true,
1530               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1531               "name": "snapset"
1532             }
1533           ],
1534           "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])",
1535           "data_digest": "0x1f26fb26",
1536           "omap_digest": "0x2eecc539",
1537           "size": 3,
1538           "errors": [],
1539           "osd": 0
1540         },
1541         {
1542           "attrs": [
1543             {
1544               "Base64": true,
1545               "value": "",
1546               "name": "_"
1547             },
1548             {
1549               "Base64": true,
1550               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1551               "name": "snapset"
1552             }
1553           ],
1554           "object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 64 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
1555           "data_digest": "0x1f26fb26",
1556           "omap_digest": "0x2eecc539",
1557           "size": 3,
1558           "errors": [],
1559           "osd": 1
1560         }
1561       ],
1562       "selected_object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 64 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
1563       "union_shard_errors": [],
1564       "errors": [
1565         "object_info_inconsistency",
1566         "attr_value_mismatch"
1567       ],
1568       "object": {
1569         "version": 64,
1570         "snap": "head",
1571         "locator": "",
1572         "nspace": "",
1573         "name": "ROBJ9"
1574       }
1575     }
1576   ],
1577   "epoch": 0
1578 }
1579 EOF
1580
1581     jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
1582     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
1583     if test $getjson = "yes"
1584     then
1585         jq '.' $dir/json > save2.json
1586     fi
1587
1588     if which jsonschema > /dev/null;
1589     then
1590       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
1591     fi
1592
1593     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
1594     teardown $dir || return 1
1595 }
1596
1597
1598 #
1599 # Test scrub errors for an erasure coded pool
1600 #
1601 function corrupt_scrub_erasure() {
1602     local dir=$1
1603     local allow_overwrites=$2
1604     local poolname=ecpool
1605     local total_objs=5
1606
1607     setup $dir || return 1
1608     run_mon $dir a || return 1
1609     run_mgr $dir x || return 1
1610     for id in $(seq 0 2) ; do
1611         if [ "$allow_overwrites" = "true" ]; then
1612             run_osd_bluestore $dir $id || return 1
1613         else
1614             run_osd $dir $id || return 1
1615         fi
1616     done
1617     create_rbd_pool || return 1
1618     ceph osd pool create foo 1
1619
1620     create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1
1621     wait_for_clean || return 1
1622
1623     for i in $(seq 1 $total_objs) ; do
1624         objname=EOBJ${i}
1625         add_something $dir $poolname $objname || return 1
1626
1627         local osd=$(expr $i % 2)
1628
1629         case $i in
1630         1)
1631             # Size (deep scrub data_digest too)
1632             local payload=UVWXYZZZ
1633             echo $payload > $dir/CORRUPT
1634             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1635             ;;
1636
1637         2)
1638             # Corrupt EC shard
1639             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1
1640             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1641             ;;
1642
1643         3)
1644              # missing
1645              objectstore_tool $dir $osd $objname remove || return 1
1646              ;;
1647
1648         4)
1649             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
1650             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
1651
1652             # Break xattrs
1653             echo -n bad-val > $dir/bad-val
1654             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
1655             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
1656             echo -n val3-$objname > $dir/newval
1657             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
1658             rm $dir/bad-val $dir/newval
1659             ;;
1660
1661         5)
1662             # Corrupt EC shard
1663             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2
1664             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1665             ;;
1666
1667         esac
1668     done
1669
1670     local pg=$(get_pg $poolname EOBJ0)
1671
1672     pg_scrub $pg
1673
1674     rados list-inconsistent-pg $poolname > $dir/json || return 1
1675     # Check pg count
1676     test $(jq '. | length' $dir/json) = "1" || return 1
1677     # Check pgid
1678     test $(jq -r '.[0]' $dir/json) = $pg || return 1
1679
1680     rados list-inconsistent-obj $pg > $dir/json || return 1
1681     # Get epoch for repair-get requests
1682     epoch=$(jq .epoch $dir/json)
1683
1684     jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
1685 {
1686   "inconsistents": [
1687     {
1688       "shards": [
1689         {
1690           "size": 2048,
1691           "errors": [],
1692           "shard": 2,
1693           "osd": 0
1694         },
1695         {
1696           "size": 9,
1697           "shard": 0,
1698           "errors": [
1699             "size_mismatch_oi"
1700           ],
1701           "osd": 1
1702         },
1703         {
1704           "size": 2048,
1705           "shard": 1,
1706           "errors": [],
1707           "osd": 2
1708         }
1709       ],
1710       "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1711       "union_shard_errors": [
1712         "size_mismatch_oi"
1713       ],
1714       "errors": [
1715         "size_mismatch"
1716       ],
1717       "object": {
1718         "version": 1,
1719         "snap": "head",
1720         "locator": "",
1721         "nspace": "",
1722         "name": "EOBJ1"
1723       }
1724     },
1725     {
1726       "shards": [
1727         {
1728           "size": 2048,
1729           "errors": [],
1730           "shard": 2,
1731           "osd": 0
1732         },
1733         {
1734           "shard": 0,
1735           "errors": [
1736             "missing"
1737           ],
1738           "osd": 1
1739         },
1740         {
1741           "size": 2048,
1742           "shard": 1,
1743           "errors": [],
1744           "osd": 2
1745         }
1746       ],
1747       "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1748       "union_shard_errors": [
1749         "missing"
1750       ],
1751       "errors": [],
1752       "object": {
1753         "version": 3,
1754         "snap": "head",
1755         "locator": "",
1756         "nspace": "",
1757         "name": "EOBJ3"
1758       }
1759     },
1760     {
1761       "shards": [
1762         {
1763           "attrs": [
1764             {
1765               "Base64": true,
1766               "value": "",
1767               "name": "_"
1768             },
1769             {
1770               "Base64": false,
1771               "value": "bad-val",
1772               "name": "_key1-EOBJ4"
1773             },
1774             {
1775               "Base64": false,
1776               "value": "val3-EOBJ4",
1777               "name": "_key3-EOBJ4"
1778             },
1779             {
1780               "Base64": true,
1781               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1782               "name": "hinfo_key"
1783             },
1784             {
1785               "Base64": true,
1786               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1787               "name": "snapset"
1788             }
1789           ],
1790           "size": 2048,
1791           "errors": [],
1792           "shard": 2,
1793           "osd": 0
1794         },
1795         {
1796           "osd": 1,
1797           "shard": 0,
1798           "errors": [],
1799           "size": 2048,
1800           "attrs": [
1801             {
1802               "Base64": true,
1803               "value": "",
1804               "name": "_"
1805             },
1806             {
1807               "Base64": false,
1808               "value": "val1-EOBJ4",
1809               "name": "_key1-EOBJ4"
1810             },
1811             {
1812               "Base64": false,
1813               "value": "val2-EOBJ4",
1814               "name": "_key2-EOBJ4"
1815             },
1816             {
1817               "Base64": true,
1818               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1819               "name": "hinfo_key"
1820             },
1821             {
1822               "Base64": true,
1823               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1824               "name": "snapset"
1825             }
1826           ]
1827         },
1828         {
1829           "osd": 2,
1830           "shard": 1,
1831           "errors": [],
1832           "size": 2048,
1833           "attrs": [
1834             {
1835               "Base64": true,
1836               "value": "",
1837               "name": "_"
1838             },
1839             {
1840               "Base64": false,
1841               "value": "val1-EOBJ4",
1842               "name": "_key1-EOBJ4"
1843             },
1844             {
1845               "Base64": false,
1846               "value": "val2-EOBJ4",
1847               "name": "_key2-EOBJ4"
1848             },
1849             {
1850               "Base64": true,
1851               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1852               "name": "hinfo_key"
1853             },
1854             {
1855               "Base64": true,
1856               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1857               "name": "snapset"
1858             }
1859           ]
1860         }
1861       ],
1862       "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1863       "union_shard_errors": [],
1864       "errors": [
1865         "attr_value_mismatch",
1866         "attr_name_mismatch"
1867       ],
1868       "object": {
1869         "version": 6,
1870         "snap": "head",
1871         "locator": "",
1872         "nspace": "",
1873         "name": "EOBJ4"
1874       }
1875     },
1876     {
1877       "shards": [
1878         {
1879           "size": 2048,
1880           "errors": [],
1881           "shard": 2,
1882           "osd": 0
1883         },
1884         {
1885           "size": 4096,
1886           "shard": 0,
1887           "errors": [
1888             "size_mismatch_oi"
1889           ],
1890           "osd": 1
1891         },
1892         {
1893           "size": 2048,
1894           "shard": 1,
1895           "errors": [],
1896           "osd": 2
1897         }
1898       ],
1899       "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1900       "union_shard_errors": [
1901         "size_mismatch_oi"
1902       ],
1903       "errors": [
1904         "size_mismatch"
1905       ],
1906       "object": {
1907         "version": 7,
1908         "snap": "head",
1909         "locator": "",
1910         "nspace": "",
1911         "name": "EOBJ5"
1912       }
1913     }
1914   ],
1915   "epoch": 0
1916 }
1917 EOF
1918
1919     jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
1920     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
1921     if test $getjson = "yes"
1922     then
1923         jq '.' $dir/json > save3.json
1924     fi
1925
1926     if which jsonschema > /dev/null;
1927     then
1928       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
1929     fi
1930
1931     pg_deep_scrub $pg
1932
1933     rados list-inconsistent-pg $poolname > $dir/json || return 1
1934     # Check pg count
1935     test $(jq '. | length' $dir/json) = "1" || return 1
1936     # Check pgid
1937     test $(jq -r '.[0]' $dir/json) = $pg || return 1
1938
1939     rados list-inconsistent-obj $pg > $dir/json || return 1
1940     # Get epoch for repair-get requests
1941     epoch=$(jq .epoch $dir/json)
1942
1943     if [ "$allow_overwrites" = "true" ]
1944     then
1945       jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
1946 {
1947   "inconsistents": [
1948     {
1949       "shards": [
1950         {
1951           "data_digest": "0x00000000",
1952           "omap_digest": "0xffffffff",
1953           "size": 2048,
1954           "errors": [],
1955           "shard": 2,
1956           "osd": 0
1957         },
1958         {
1959           "size": 9,
1960           "shard": 0,
1961           "errors": [
1962             "read_error",
1963             "size_mismatch_oi"
1964           ],
1965           "osd": 1
1966         },
1967         {
1968           "data_digest": "0x00000000",
1969           "omap_digest": "0xffffffff",
1970           "size": 2048,
1971           "shard": 1,
1972           "errors": [],
1973           "osd": 2
1974         }
1975       ],
1976       "selected_object_info": "3:9175b684:::EOBJ1:head(27'1 client.4155.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1977       "union_shard_errors": [
1978         "read_error",
1979         "size_mismatch_oi"
1980       ],
1981       "errors": [
1982         "size_mismatch"
1983       ],
1984       "object": {
1985         "version": 1,
1986         "snap": "head",
1987         "locator": "",
1988         "nspace": "",
1989         "name": "EOBJ1"
1990       }
1991     },
1992     {
1993       "shards": [
1994         {
1995           "data_digest": "0x00000000",
1996           "omap_digest": "0xffffffff",
1997           "size": 2048,
1998           "errors": [],
1999           "shard": 2,
2000           "osd": 0
2001         },
2002         {
2003           "shard": 0,
2004           "errors": [
2005             "missing"
2006           ],
2007           "osd": 1
2008         },
2009         {
2010           "data_digest": "0x00000000",
2011           "omap_digest": "0xffffffff",
2012           "size": 2048,
2013           "shard": 1,
2014           "errors": [],
2015           "osd": 2
2016         }
2017       ],
2018       "selected_object_info": "3:b197b25d:::EOBJ3:head(41'3 client.4199.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2019       "union_shard_errors": [
2020         "missing"
2021       ],
2022       "errors": [],
2023       "object": {
2024         "version": 3,
2025         "snap": "head",
2026         "locator": "",
2027         "nspace": "",
2028         "name": "EOBJ3"
2029       }
2030     },
2031     {
2032       "shards": [
2033         {
2034           "attrs": [
2035             {
2036               "Base64": true,
2037               "value": "",
2038               "name": "_"
2039             },
2040             {
2041               "Base64": false,
2042               "value": "bad-val",
2043               "name": "_key1-EOBJ4"
2044             },
2045             {
2046               "Base64": false,
2047               "value": "val3-EOBJ4",
2048               "name": "_key3-EOBJ4"
2049             },
2050             {
2051               "Base64": true,
2052               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2053               "name": "hinfo_key"
2054             },
2055             {
2056               "Base64": true,
2057               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2058               "name": "snapset"
2059             }
2060           ],
2061           "data_digest": "0x00000000",
2062           "omap_digest": "0xffffffff",
2063           "size": 2048,
2064           "errors": [],
2065           "shard": 2,
2066           "osd": 0
2067         },
2068         {
2069           "attrs": [
2070             {
2071               "Base64": true,
2072               "value": "",
2073               "name": "_"
2074             },
2075             {
2076               "Base64": false,
2077               "value": "val1-EOBJ4",
2078               "name": "_key1-EOBJ4"
2079             },
2080             {
2081               "Base64": false,
2082               "value": "val2-EOBJ4",
2083               "name": "_key2-EOBJ4"
2084             },
2085             {
2086               "Base64": true,
2087               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2088               "name": "hinfo_key"
2089             },
2090             {
2091               "Base64": true,
2092               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2093               "name": "snapset"
2094             }
2095           ],
2096           "data_digest": "0x00000000",
2097           "omap_digest": "0xffffffff",
2098           "size": 2048,
2099           "errors": [],
2100           "shard": 0,
2101           "osd": 1
2102         },
2103         {
2104           "attrs": [
2105             {
2106               "Base64": true,
2107               "value": "",
2108               "name": "_"
2109             },
2110             {
2111               "Base64": false,
2112               "value": "val1-EOBJ4",
2113               "name": "_key1-EOBJ4"
2114             },
2115             {
2116               "Base64": false,
2117               "value": "val2-EOBJ4",
2118               "name": "_key2-EOBJ4"
2119             },
2120             {
2121               "Base64": true,
2122               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2123               "name": "hinfo_key"
2124             },
2125             {
2126               "Base64": true,
2127               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2128               "name": "snapset"
2129             }
2130           ],
2131           "data_digest": "0x00000000",
2132           "omap_digest": "0xffffffff",
2133           "size": 2048,
2134           "errors": [],
2135           "shard": 1,
2136           "osd": 2
2137         }
2138       ],
2139       "selected_object_info": "3:5e723e06:::EOBJ4:head(48'6 client.4223.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2140       "union_shard_errors": [],
2141       "errors": [
2142         "attr_value_mismatch",
2143         "attr_name_mismatch"
2144       ],
2145       "object": {
2146         "version": 6,
2147         "snap": "head",
2148         "locator": "",
2149         "nspace": "",
2150         "name": "EOBJ4"
2151       }
2152     },
2153     {
2154       "shards": [
2155         {
2156           "data_digest": "0x00000000",
2157           "omap_digest": "0xffffffff",
2158           "size": 2048,
2159           "errors": [],
2160           "shard": 2,
2161           "osd": 0
2162         },
2163         {
2164           "data_digest": "0x00000000",
2165           "omap_digest": "0xffffffff",
2166           "size": 4096,
2167           "errors": [
2168             "size_mismatch_oi"
2169           ],
2170           "shard": 0,
2171           "osd": 1
2172         },
2173         {
2174           "data_digest": "0x00000000",
2175           "omap_digest": "0xffffffff",
2176           "size": 2048,
2177           "errors": [],
2178           "shard": 1,
2179           "osd": 2
2180         }
2181       ],
2182       "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4288.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2183       "union_shard_errors": [
2184         "size_mismatch_oi"
2185       ],
2186       "errors": [
2187         "size_mismatch"
2188       ],
2189       "object": {
2190         "version": 7,
2191         "snap": "head",
2192         "locator": "",
2193         "nspace": "",
2194         "name": "EOBJ5"
2195       }
2196     }
2197   ],
2198   "epoch": 0
2199 }
2200 EOF
2201
2202     else
2203
2204       jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
2205 {
2206   "inconsistents": [
2207     {
2208       "shards": [
2209         {
2210           "data_digest": "0x04cfa72f",
2211           "omap_digest": "0xffffffff",
2212           "size": 2048,
2213           "errors": [],
2214           "shard": 2,
2215           "osd": 0
2216         },
2217         {
2218           "size": 9,
2219           "shard": 0,
2220           "errors": [
2221             "read_error",
2222             "size_mismatch_oi"
2223           ],
2224           "osd": 1
2225         },
2226         {
2227           "data_digest": "0x04cfa72f",
2228           "omap_digest": "0xffffffff",
2229           "size": 2048,
2230           "shard": 1,
2231           "errors": [],
2232           "osd": 2
2233         }
2234       ],
2235       "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2236       "union_shard_errors": [
2237         "read_error",
2238         "size_mismatch_oi"
2239       ],
2240       "errors": [
2241         "size_mismatch"
2242       ],
2243       "object": {
2244         "version": 1,
2245         "snap": "head",
2246         "locator": "",
2247         "nspace": "",
2248         "name": "EOBJ1"
2249       }
2250     },
2251     {
2252       "shards": [
2253         {
2254           "size": 2048,
2255           "errors": [
2256             "ec_hash_error"
2257           ],
2258           "shard": 2,
2259           "osd": 0
2260         },
2261         {
2262           "data_digest": "0x04cfa72f",
2263           "omap_digest": "0xffffffff",
2264           "size": 2048,
2265           "errors": [],
2266           "shard": 0,
2267           "osd": 1
2268         },
2269         {
2270           "data_digest": "0x04cfa72f",
2271           "omap_digest": "0xffffffff",
2272           "size": 2048,
2273           "errors": [],
2274           "shard": 1,
2275           "osd": 2
2276         }
2277       ],
2278       "selected_object_info": "3:9babd184:::EOBJ2:head(29'2 client.4217.0:1 dirty|data_digest|omap_digest s 7 uv 2 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2279       "union_shard_errors": [
2280         "ec_hash_error"
2281       ],
2282       "errors": [],
2283       "object": {
2284         "version": 2,
2285         "snap": "head",
2286         "locator": "",
2287         "nspace": "",
2288         "name": "EOBJ2"
2289       }
2290     },
2291     {
2292       "shards": [
2293         {
2294           "data_digest": "0x04cfa72f",
2295           "omap_digest": "0xffffffff",
2296           "size": 2048,
2297           "errors": [],
2298           "shard": 2,
2299           "osd": 0
2300         },
2301         {
2302           "osd": 1,
2303           "shard": 0,
2304           "errors": [
2305             "missing"
2306           ]
2307         },
2308         {
2309           "data_digest": "0x04cfa72f",
2310           "omap_digest": "0xffffffff",
2311           "size": 2048,
2312           "shard": 1,
2313           "errors": [],
2314           "osd": 2
2315         }
2316       ],
2317       "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2318       "union_shard_errors": [
2319         "missing"
2320       ],
2321       "errors": [],
2322       "object": {
2323         "version": 3,
2324         "snap": "head",
2325         "locator": "",
2326         "nspace": "",
2327         "name": "EOBJ3"
2328       }
2329     },
2330     {
2331       "shards": [
2332         {
2333           "attrs": [
2334             {
2335               "Base64": true,
2336               "value": "",
2337               "name": "_"
2338             },
2339             {
2340               "Base64": false,
2341               "value": "bad-val",
2342               "name": "_key1-EOBJ4"
2343             },
2344             {
2345               "Base64": false,
2346               "value": "val3-EOBJ4",
2347               "name": "_key3-EOBJ4"
2348             },
2349             {
2350               "Base64": true,
2351               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2352               "name": "hinfo_key"
2353             },
2354             {
2355               "Base64": true,
2356               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2357               "name": "snapset"
2358             }
2359           ],
2360           "data_digest": "0x04cfa72f",
2361           "omap_digest": "0xffffffff",
2362           "size": 2048,
2363           "errors": [],
2364           "shard": 2,
2365           "osd": 0
2366         },
2367         {
2368           "osd": 1,
2369           "shard": 0,
2370           "errors": [],
2371           "size": 2048,
2372           "omap_digest": "0xffffffff",
2373           "data_digest": "0x04cfa72f",
2374           "attrs": [
2375             {
2376               "Base64": true,
2377               "value": "",
2378               "name": "_"
2379             },
2380             {
2381               "Base64": false,
2382               "value": "val1-EOBJ4",
2383               "name": "_key1-EOBJ4"
2384             },
2385             {
2386               "Base64": false,
2387               "value": "val2-EOBJ4",
2388               "name": "_key2-EOBJ4"
2389             },
2390             {
2391               "Base64": true,
2392               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2393               "name": "hinfo_key"
2394             },
2395             {
2396               "Base64": true,
2397               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2398               "name": "snapset"
2399             }
2400           ]
2401         },
2402         {
2403           "osd": 2,
2404           "shard": 1,
2405           "errors": [],
2406           "size": 2048,
2407           "omap_digest": "0xffffffff",
2408           "data_digest": "0x04cfa72f",
2409           "attrs": [
2410             {
2411               "Base64": true,
2412               "value": "",
2413               "name": "_"
2414             },
2415             {
2416               "Base64": false,
2417               "value": "val1-EOBJ4",
2418               "name": "_key1-EOBJ4"
2419             },
2420             {
2421               "Base64": false,
2422               "value": "val2-EOBJ4",
2423               "name": "_key2-EOBJ4"
2424             },
2425             {
2426               "Base64": true,
2427               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2428               "name": "hinfo_key"
2429             },
2430             {
2431               "Base64": true,
2432               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2433               "name": "snapset"
2434             }
2435           ]
2436         }
2437       ],
2438       "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2439       "union_shard_errors": [],
2440       "errors": [
2441         "attr_value_mismatch",
2442         "attr_name_mismatch"
2443       ],
2444       "object": {
2445         "version": 6,
2446         "snap": "head",
2447         "locator": "",
2448         "nspace": "",
2449         "name": "EOBJ4"
2450       }
2451     },
2452     {
2453       "shards": [
2454         {
2455           "data_digest": "0x04cfa72f",
2456           "omap_digest": "0xffffffff",
2457           "size": 2048,
2458           "errors": [],
2459           "shard": 2,
2460           "osd": 0
2461         },
2462         {
2463           "size": 4096,
2464           "shard": 0,
2465           "errors": [
2466             "size_mismatch_oi",
2467             "ec_size_error"
2468           ],
2469           "osd": 1
2470         },
2471         {
2472           "data_digest": "0x04cfa72f",
2473           "omap_digest": "0xffffffff",
2474           "size": 2048,
2475           "shard": 1,
2476           "errors": [],
2477           "osd": 2
2478         }
2479       ],
2480       "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2481       "union_shard_errors": [
2482         "size_mismatch_oi",
2483         "ec_size_error"
2484       ],
2485       "errors": [
2486         "size_mismatch"
2487       ],
2488       "object": {
2489         "version": 7,
2490         "snap": "head",
2491         "locator": "",
2492         "nspace": "",
2493         "name": "EOBJ5"
2494       }
2495     }
2496   ],
2497   "epoch": 0
2498 }
2499 EOF
2500
2501     fi
2502
2503     jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
2504     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
2505     if test $getjson = "yes"
2506     then
2507       if [ "$allow_overwrites" = "true" ]
2508       then
2509         num=4
2510       else
2511         num=5
2512       fi
2513       jq '.' $dir/json > save${num}.json
2514     fi
2515
2516     if which jsonschema > /dev/null;
2517     then
2518       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
2519     fi
2520
2521     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
2522     teardown $dir || return 1
2523 }
2524
2525 function TEST_corrupt_scrub_erasure_appends() {
2526     corrupt_scrub_erasure $1 false
2527 }
2528
2529 function TEST_corrupt_scrub_erasure_overwrites() {
2530     if [ "$use_ec_overwrite" = "true" ]; then
2531         corrupt_scrub_erasure $1 true
2532     fi
2533 }
2534
2535 #
2536 # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost
2537 #
2538 function TEST_periodic_scrub_replicated() {
2539     local dir=$1
2540     local poolname=psr_pool
2541     local objname=POBJ
2542
2543     setup $dir || return 1
2544     run_mon $dir a --osd_pool_default_size=2 || return 1
2545     run_mgr $dir x || return 1
2546     local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0"
2547     run_osd $dir 0 $ceph_osd_args || return 1
2548     run_osd $dir 1 $ceph_osd_args || return 1
2549     create_rbd_pool || return 1
2550     wait_for_clean || return 1
2551
2552     ceph osd pool create $poolname 1 1 || return 1
2553     wait_for_clean || return 1
2554
2555     local osd=0
2556     add_something $dir $poolname $objname scrub || return 1
2557     local primary=$(get_primary $poolname $objname)
2558     local pg=$(get_pg $poolname $objname)
2559
2560     # Add deep-scrub only error
2561     local payload=UVWXYZ
2562     echo $payload > $dir/CORRUPT
2563     # Uses $ceph_osd_args for osd restart
2564     objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2565
2566     # No scrub information available, so expect failure
2567     set -o pipefail
2568     !  rados list-inconsistent-obj $pg | jq '.' || return 1
2569     set +o pipefail
2570
2571     pg_deep_scrub $pg || return 1
2572
2573     # Make sure bad object found
2574     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
2575
2576     local last_scrub=$(get_last_scrub_stamp $pg)
2577     # Fake a schedule scrub
2578     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
2579              trigger_scrub $pg || return 1
2580     # Wait for schedule regular scrub
2581     wait_for_scrub $pg "$last_scrub"
2582
2583     # It needed to be upgraded
2584     grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1
2585
2586     # Bad object still known
2587     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
2588
2589     # Can't upgrade with this set
2590     ceph osd set nodeep-scrub
2591     # Let map change propagate to OSDs
2592     sleep 2
2593
2594     # Fake a schedule scrub
2595     local last_scrub=$(get_last_scrub_stamp $pg)
2596     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
2597              trigger_scrub $pg || return 1
2598     # Wait for schedule regular scrub
2599     # to notice scrub and skip it
2600     local found=false
2601     for i in $(seq 14 -1 0)
2602     do
2603       sleep 1
2604       ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; }
2605       echo Time left: $i seconds
2606     done
2607     test $found = "true" || return 1
2608
2609     # Bad object still known
2610     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
2611
2612     # Request a regular scrub and it will be done
2613     local scrub_backoff_ratio=$(get_config osd ${primary} osd_scrub_backoff_ratio)
2614     set_config osd ${primary} osd_scrub_backoff_ratio 0
2615     pg_scrub $pg
2616     sleep 1
2617     set_config osd ${primary} osd_scrub_backoff_ratio $scrub_backoff_ratio
2618     grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1
2619
2620     # deep-scrub error is no longer present
2621     rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1
2622 }
2623
2624
2625 main osd-scrub-repair "$@"
2626
2627 # Local Variables:
2628 # compile-command: "cd ../.. ; make -j4 && \
2629 #    test/osd/osd-scrub-repair.sh # TEST_corrupt_and_repair_replicated"
2630 # End: