ceph/qa/standalone/scrub/osd-scrub-repair.sh

   1 #!/bin/bash -x
   2 #
   3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
   4 #
   5 # Author: Loic Dachary <loic@dachary.org>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  18
  19 if [ `uname` = FreeBSD ]; then
  20     # erasure coding overwrites are only tested on Bluestore
  21     # erasure coding on filestore is unsafe
  22     # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites
  23     use_ec_overwrite=false
  24 else
  25     use_ec_overwrite=true
  26 fi
  27
  28 # Test development and debugging
  29 # Set to "yes" in order to ignore diff errors and save results to update test
  30 getjson="no"
  31
  32 # Ignore the epoch and filter out the attr '_' value because it has date information and won't match
  33 if [ "$(jq --version 2>&1 | awk '{ print $3}')" = "1.3" ]; then # Not sure all versions that apply here
  34     jqfilter='.inconsistents | (.[].shards[].attrs[] | select(.name == "_") | .value) |= "----Stripped-by-test----"'
  35 else
  36     jqfilter='.inconsistents | (.[].shards[].attrs[]? | select(.name == "_") | .value) |= "----Stripped-by-test----"'
  37 fi
  38 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
  39
  40 # Remove items are not consistent across runs, the pg interval and client
  41 sedfilter='s/\([ ]*\"\(selected_\)*object_info\":.*head[(]\)[^[:space:]]* [^[:space:]]* \(.*\)/\1\3/'
  42
  43 function run() {
  44     local dir=$1
  45     shift
  46
  47     export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
  48     export CEPH_ARGS
  49     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  50     CEPH_ARGS+="--mon-host=$CEPH_MON "
  51
  52     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  53     for func in $funcs ; do
  54         $func $dir || return 1
  55     done
  56 }
  57
  58 function add_something() {
  59     local dir=$1
  60     local poolname=$2
  61     local obj=${3:-SOMETHING}
  62     local scrub=${4:-noscrub}
  63
  64     if [ "$scrub" = "noscrub" ];
  65     then
  66         ceph osd set noscrub || return 1
  67         ceph osd set nodeep-scrub || return 1
  68     else
  69         ceph osd unset noscrub || return 1
  70         ceph osd unset nodeep-scrub || return 1
  71     fi
  72
  73     local payload=ABCDEF
  74     echo $payload > $dir/ORIGINAL
  75     rados --pool $poolname put $obj $dir/ORIGINAL || return 1
  76 }
  77
  78 #
  79 # Corrupt one copy of a replicated pool
  80 #
  81 function TEST_corrupt_and_repair_replicated() {
  82     local dir=$1
  83     local poolname=rbd
  84
  85     setup $dir || return 1
  86     run_mon $dir a --osd_pool_default_size=2 || return 1
  87     run_mgr $dir x || return 1
  88     run_osd $dir 0 || return 1
  89     run_osd $dir 1 || return 1
  90     create_rbd_pool || return 1
  91     wait_for_clean || return 1
  92
  93     add_something $dir $poolname || return 1
  94     corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
  95     # Reproduces http://tracker.ceph.com/issues/8914
  96     corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1
  97
  98     teardown $dir || return 1
  99 }
 100
 101 function corrupt_and_repair_two() {
 102     local dir=$1
 103     local poolname=$2
 104     local first=$3
 105     local second=$4
 106
 107     #
 108     # 1) remove the corresponding file from the OSDs
 109     #
 110     pids=""
 111     run_in_background pids objectstore_tool $dir $first SOMETHING remove
 112     run_in_background pids objectstore_tool $dir $second SOMETHING remove
 113     wait_background pids
 114     return_code=$?
 115     if [ $return_code -ne 0 ]; then return $return_code; fi
 116
 117     #
 118     # 2) repair the PG
 119     #
 120     local pg=$(get_pg $poolname SOMETHING)
 121     repair $pg
 122     #
 123     # 3) The files must be back
 124     #
 125     pids=""
 126     run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs
 127     run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs
 128     wait_background pids
 129     return_code=$?
 130     if [ $return_code -ne 0 ]; then return $return_code; fi
 131
 132     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 133     diff $dir/ORIGINAL $dir/COPY || return 1
 134 }
 135
 136 #
 137 # 1) add an object
 138 # 2) remove the corresponding file from a designated OSD
 139 # 3) repair the PG
 140 # 4) check that the file has been restored in the designated OSD
 141 #
 142 function corrupt_and_repair_one() {
 143     local dir=$1
 144     local poolname=$2
 145     local osd=$3
 146
 147     #
 148     # 1) remove the corresponding file from the OSD
 149     #
 150     objectstore_tool $dir $osd SOMETHING remove || return 1
 151     #
 152     # 2) repair the PG
 153     #
 154     local pg=$(get_pg $poolname SOMETHING)
 155     repair $pg
 156     #
 157     # 3) The file must be back
 158     #
 159     objectstore_tool $dir $osd SOMETHING list-attrs || return 1
 160     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 161     diff $dir/ORIGINAL $dir/COPY || return 1
 162 }
 163
 164 function corrupt_and_repair_erasure_coded() {
 165     local dir=$1
 166     local poolname=$2
 167
 168     add_something $dir $poolname || return 1
 169
 170     local primary=$(get_primary $poolname SOMETHING)
 171     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 172     local not_primary_first=${osds[0]}
 173     local not_primary_second=${osds[1]}
 174
 175     # Reproduces http://tracker.ceph.com/issues/10017
 176     corrupt_and_repair_one $dir $poolname $primary  || return 1
 177     # Reproduces http://tracker.ceph.com/issues/10409
 178     corrupt_and_repair_one $dir $poolname $not_primary_first || return 1
 179     corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1
 180     corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1
 181
 182 }
 183
 184 function create_ec_pool() {
 185     local pool_name=$1
 186     local allow_overwrites=$2
 187
 188     ceph osd erasure-code-profile set myprofile crush-failure-domain=osd $3 $4 $5 $6 $7 || return 1
 189
 190     create_pool "$poolname" 1 1 erasure myprofile || return 1
 191
 192     if [ "$allow_overwrites" = "true" ]; then
 193         ceph osd pool set "$poolname" allow_ec_overwrites true || return 1
 194     fi
 195
 196     wait_for_clean || return 1
 197     return 0
 198 }
 199
 200 function auto_repair_erasure_coded() {
 201     local dir=$1
 202     local allow_overwrites=$2
 203     local poolname=ecpool
 204
 205     # Launch a cluster with 5 seconds scrub interval
 206     setup $dir || return 1
 207     run_mon $dir a || return 1
 208     run_mgr $dir x || return 1
 209     local ceph_osd_args="--osd-scrub-auto-repair=true \
 210             --osd-deep-scrub-interval=5 \
 211             --osd-scrub-max-interval=5 \
 212             --osd-scrub-min-interval=5 \
 213             --osd-scrub-interval-randomize-ratio=0"
 214     for id in $(seq 0 2) ; do
 215         if [ "$allow_overwrites" = "true" ]; then
 216             run_osd_bluestore $dir $id $ceph_osd_args || return 1
 217         else
 218             run_osd $dir $id $ceph_osd_args || return 1
 219         fi
 220     done
 221     create_rbd_pool || return 1
 222     wait_for_clean || return 1
 223
 224     # Create an EC pool
 225     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 226
 227     # Put an object
 228     local payload=ABCDEF
 229     echo $payload > $dir/ORIGINAL
 230     rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
 231
 232     # Remove the object from one shard physically
 233     # Restarted osd get $ceph_osd_args passed
 234     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
 235     # Wait for auto repair
 236     local pgid=$(get_pg $poolname SOMETHING)
 237     wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)"
 238     wait_for_clean || return 1
 239     # Verify - the file should be back
 240     # Restarted osd get $ceph_osd_args passed
 241     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
 242     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 243     diff $dir/ORIGINAL $dir/COPY || return 1
 244
 245     # Tear down
 246     teardown $dir || return 1
 247 }
 248
 249 function TEST_auto_repair_erasure_coded_appends() {
 250     auto_repair_erasure_coded $1 false
 251 }
 252
 253 function TEST_auto_repair_erasure_coded_overwrites() {
 254     if [ "$use_ec_overwrite" = "true" ]; then
 255         auto_repair_erasure_coded $1 true
 256     fi
 257 }
 258
 259 function corrupt_and_repair_jerasure() {
 260     local dir=$1
 261     local allow_overwrites=$2
 262     local poolname=ecpool
 263
 264     setup $dir || return 1
 265     run_mon $dir a || return 1
 266     run_mgr $dir x || return 1
 267     for id in $(seq 0 3) ; do
 268         if [ "$allow_overwrites" = "true" ]; then
 269             run_osd_bluestore $dir $id || return 1
 270         else
 271             run_osd $dir $id || return 1
 272         fi
 273     done
 274     create_rbd_pool || return 1
 275     wait_for_clean || return 1
 276
 277     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 278     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 279
 280     teardown $dir || return 1
 281 }
 282
 283 function TEST_corrupt_and_repair_jerasure_appends() {
 284     corrupt_and_repair_jerasure $1
 285 }
 286
 287 function TEST_corrupt_and_repair_jerasure_overwrites() {
 288     if [ "$use_ec_overwrite" = "true" ]; then
 289         corrupt_and_repair_jerasure $1 true
 290     fi
 291 }
 292
 293 function corrupt_and_repair_lrc() {
 294     local dir=$1
 295     local allow_overwrites=$2
 296     local poolname=ecpool
 297
 298     setup $dir || return 1
 299     run_mon $dir a || return 1
 300     run_mgr $dir x || return 1
 301     for id in $(seq 0 9) ; do
 302         if [ "$allow_overwrites" = "true" ]; then
 303             run_osd_bluestore $dir $id || return 1
 304         else
 305             run_osd $dir $id || return 1
 306         fi
 307     done
 308     create_rbd_pool || return 1
 309     wait_for_clean || return 1
 310
 311     create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1
 312     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 313
 314     teardown $dir || return 1
 315 }
 316
 317 function TEST_corrupt_and_repair_lrc_appends() {
 318     corrupt_and_repair_jerasure $1
 319 }
 320
 321 function TEST_corrupt_and_repair_lrc_overwrites() {
 322     if [ "$use_ec_overwrite" = "true" ]; then
 323         corrupt_and_repair_jerasure $1 true
 324     fi
 325 }
 326
 327 function unfound_erasure_coded() {
 328     local dir=$1
 329     local allow_overwrites=$2
 330     local poolname=ecpool
 331     local payload=ABCDEF
 332
 333     setup $dir || return 1
 334     run_mon $dir a || return 1
 335     run_mgr $dir x || return 1
 336     for id in $(seq 0 3) ; do
 337         if [ "$allow_overwrites" = "true" ]; then
 338             run_osd_bluestore $dir $id || return 1
 339         else
 340             run_osd $dir $id || return 1
 341         fi
 342     done
 343     create_rbd_pool || return 1
 344     wait_for_clean || return 1
 345
 346     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 347
 348     add_something $dir $poolname || return 1
 349
 350     local primary=$(get_primary $poolname SOMETHING)
 351     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 352     local not_primary_first=${osds[0]}
 353     local not_primary_second=${osds[1]}
 354     local not_primary_third=${osds[2]}
 355
 356     #
 357     # 1) remove the corresponding file from the OSDs
 358     #
 359     pids=""
 360     run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove
 361     run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove
 362     run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove
 363     wait_background pids
 364     return_code=$?
 365     if [ $return_code -ne 0 ]; then return $return_code; fi
 366
 367     #
 368     # 2) repair the PG
 369     #
 370     local pg=$(get_pg $poolname SOMETHING)
 371     repair $pg
 372     #
 373     # 3) check pg state
 374     #
 375     # it may take a bit to appear due to mon/mgr asynchrony
 376     for f in `seq 1 60`; do
 377         ceph -s | grep "1/1 objects unfound" && break
 378         sleep 1
 379     done
 380     ceph -s|grep "4 osds: 4 up, 4 in" || return 1
 381     ceph -s|grep "1/1 objects unfound" || return 1
 382
 383     teardown $dir || return 1
 384 }
 385
 386 function TEST_unfound_erasure_coded_appends() {
 387     unfound_erasure_coded $1
 388 }
 389
 390 function TEST_unfound_erasure_coded_overwrites() {
 391     if [ "$use_ec_overwrite" = "true" ]; then
 392         unfound_erasure_coded $1 true
 393     fi
 394 }
 395
 396 #
 397 # list_missing for EC pool
 398 #
 399 function list_missing_erasure_coded() {
 400     local dir=$1
 401     local allow_overwrites=$2
 402     local poolname=ecpool
 403
 404     setup $dir || return 1
 405     run_mon $dir a || return 1
 406     run_mgr $dir x || return 1
 407     for id in $(seq 0 2) ; do
 408         if [ "$allow_overwrites" = "true" ]; then
 409             run_osd_bluestore $dir $id || return 1
 410         else
 411             run_osd $dir $id || return 1
 412         fi
 413     done
 414     create_rbd_pool || return 1
 415     wait_for_clean || return 1
 416
 417     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 418
 419     # Put an object and remove the two shards (including primary)
 420     add_something $dir $poolname MOBJ0 || return 1
 421     local -a osds0=($(get_osds $poolname MOBJ0))
 422
 423     # Put another object and remove two shards (excluding primary)
 424     add_something $dir $poolname MOBJ1 || return 1
 425     local -a osds1=($(get_osds $poolname MOBJ1))
 426
 427     # Stop all osd daemons
 428     for id in $(seq 0 2) ; do
 429         kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1
 430     done
 431
 432     id=${osds0[0]}
 433     ceph-objectstore-tool --data-path $dir/$id \
 434         MOBJ0 remove || return 1
 435     id=${osds0[1]}
 436     ceph-objectstore-tool --data-path $dir/$id \
 437         MOBJ0 remove || return 1
 438
 439     id=${osds1[1]}
 440     ceph-objectstore-tool --data-path $dir/$id \
 441         MOBJ1 remove || return 1
 442     id=${osds1[2]}
 443     ceph-objectstore-tool --data-path $dir/$id \
 444         MOBJ1 remove || return 1
 445
 446     for id in $(seq 0 2) ; do
 447         activate_osd $dir $id >&2 || return 1
 448     done
 449     create_rbd_pool || return 1
 450     wait_for_clean || return 1
 451
 452     # Get get - both objects should in the same PG
 453     local pg=$(get_pg $poolname MOBJ0)
 454
 455     # Repair the PG, which triggers the recovering,
 456     # and should mark the object as unfound
 457     repair $pg
 458
 459     for i in $(seq 0 120) ; do
 460         [ $i -lt 60 ] || return 1
 461         matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l)
 462         [ $matches -eq 2 ] && break
 463     done
 464
 465     teardown $dir || return 1
 466 }
 467
 468 function TEST_list_missing_erasure_coded_appends() {
 469     list_missing_erasure_coded $1 false
 470 }
 471
 472 function TEST_list_missing_erasure_coded_overwrites() {
 473     if [ "$use_ec_overwrite" = "true" ]; then
 474         list_missing_erasure_coded $1 true
 475     fi
 476 }
 477
 478 #
 479 # Corrupt one copy of a replicated pool
 480 #
 481 function TEST_corrupt_scrub_replicated() {
 482     local dir=$1
 483     local poolname=csr_pool
 484     local total_objs=16
 485
 486     setup $dir || return 1
 487     run_mon $dir a --osd_pool_default_size=2 || return 1
 488     run_mgr $dir x || return 1
 489     run_osd $dir 0 || return 1
 490     run_osd $dir 1 || return 1
 491     create_rbd_pool || return 1
 492     wait_for_clean || return 1
 493
 494     create_pool foo 1 || return 1
 495     create_pool $poolname 1 1 || return 1
 496     wait_for_clean || return 1
 497
 498     for i in $(seq 1 $total_objs) ; do
 499         objname=ROBJ${i}
 500         add_something $dir $poolname $objname || return 1
 501
 502         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
 503         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
 504     done
 505
 506     local pg=$(get_pg $poolname ROBJ0)
 507
 508     # Compute an old omap digest and save oi
 509     CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \
 510         config set osd_deep_scrub_update_digest_min_age 0
 511     CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \
 512         config set osd_deep_scrub_update_digest_min_age 0
 513     pg_deep_scrub $pg
 514
 515     for i in $(seq 1 $total_objs) ; do
 516         objname=ROBJ${i}
 517
 518         # Alternate corruption between osd.0 and osd.1
 519         local osd=$(expr $i % 2)
 520
 521         case $i in
 522         1)
 523             # Size (deep scrub data_digest too)
 524             local payload=UVWXYZZZ
 525             echo $payload > $dir/CORRUPT
 526             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 527             ;;
 528
 529         2)
 530             # digest (deep scrub only)
 531             local payload=UVWXYZ
 532             echo $payload > $dir/CORRUPT
 533             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 534             ;;
 535
 536         3)
 537              # missing
 538              objectstore_tool $dir $osd $objname remove || return 1
 539              ;;
 540
 541          4)
 542              # Modify omap value (deep scrub only)
 543              objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1
 544              ;;
 545
 546          5)
 547             # Delete omap key (deep scrub only)
 548             objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1
 549             ;;
 550
 551          6)
 552             # Add extra omap key (deep scrub only)
 553             echo extra > $dir/extra-val
 554             objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1
 555             rm $dir/extra-val
 556             ;;
 557
 558          7)
 559             # Modify omap header (deep scrub only)
 560             echo -n newheader > $dir/hdr
 561             objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1
 562             rm $dir/hdr
 563             ;;
 564
 565          8)
 566             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
 567             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
 568
 569             # Break xattrs
 570             echo -n bad-val > $dir/bad-val
 571             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
 572             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
 573             echo -n val3-$objname > $dir/newval
 574             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
 575             rm $dir/bad-val $dir/newval
 576             ;;
 577
 578         9)
 579             objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi
 580             echo -n D > $dir/change
 581             rados --pool $poolname put $objname $dir/change
 582             objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi
 583             rm $dir/oi $dir/change
 584             ;;
 585
 586           # ROBJ10 must be handled after digests are re-computed by a deep scrub below
 587           # ROBJ11 must be handled with config change before deep scrub
 588           # ROBJ12 must be handled with config change before scrubs
 589           # ROBJ13 must be handled before scrubs
 590
 591         14)
 592             echo -n bad-val > $dir/bad-val
 593             objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1
 594             objectstore_tool $dir 1 $objname rm-attr _ || return 1
 595             rm $dir/bad-val
 596             ;;
 597
 598         15)
 599             objectstore_tool $dir $osd $objname rm-attr _ || return 1
 600             ;;
 601
 602         16)
 603             objectstore_tool $dir 0 $objname rm-attr snapset || return 1
 604             echo -n bad-val > $dir/bad-val
 605             objectstore_tool $dir 1 $objname set-attr snapset $dir/bad-val || return 1
 606
 607         esac
 608     done
 609
 610     local pg=$(get_pg $poolname ROBJ0)
 611
 612     inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
 613     inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
 614     inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
 615     inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
 616
 617     pg_scrub $pg
 618
 619     rados list-inconsistent-pg $poolname > $dir/json || return 1
 620     # Check pg count
 621     test $(jq '. | length' $dir/json) = "1" || return 1
 622     # Check pgid
 623     test $(jq -r '.[0]' $dir/json) = $pg || return 1
 624
 625     rados list-inconsistent-obj $pg > $dir/json || return 1
 626     # Get epoch for repair-get requests
 627     epoch=$(jq .epoch $dir/json)
 628
 629     jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
 630 {
 631   "inconsistents": [
 632     {
 633       "shards": [
 634         {
 635           "size": 7,
 636           "errors": [],
 637           "osd": 0,
 638           "primary": false
 639         },
 640         {
 641           "size": 9,
 642           "errors": [
 643             "size_mismatch_oi",
 644             "obj_size_oi_mismatch"
 645           ],
 646           "osd": 1,
 647           "primary": true
 648         }
 649       ],
 650       "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])",
 651       "union_shard_errors": [
 652         "size_mismatch_oi",
 653         "obj_size_oi_mismatch"
 654       ],
 655       "errors": [
 656         "size_mismatch"
 657       ],
 658       "object": {
 659         "version": 3,
 660         "snap": "head",
 661         "locator": "",
 662         "nspace": "",
 663         "name": "ROBJ1"
 664       }
 665     },
 666     {
 667       "shards": [
 668         {
 669           "errors": [
 670             "stat_error"
 671           ],
 672           "osd": 0,
 673           "primary": false
 674         },
 675         {
 676           "size": 7,
 677           "errors": [],
 678           "osd": 1,
 679           "primary": true
 680         }
 681       ],
 682       "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])",
 683       "union_shard_errors": [
 684         "stat_error"
 685       ],
 686       "errors": [],
 687       "object": {
 688         "version": 36,
 689         "snap": "head",
 690         "locator": "",
 691         "nspace": "",
 692         "name": "ROBJ12"
 693       }
 694     },
 695     {
 696       "shards": [
 697         {
 698           "errors": [
 699             "stat_error"
 700           ],
 701           "osd": 0,
 702           "primary": false
 703         },
 704         {
 705           "size": 7,
 706           "errors": [],
 707           "osd": 1,
 708           "primary": true
 709         }
 710       ],
 711       "selected_object_info": "3:d60617f9:::ROBJ13:head(47'55 osd.0.0:54 dirty|omap|data_digest|omap_digest s 7 uv 39 dd 2ddbf8f5 od 6441854d alloc_hint [0 0 0])",
 712       "union_shard_errors": [
 713         "stat_error"
 714       ],
 715       "errors": [],
 716       "object": {
 717         "version": 39,
 718         "snap": "head",
 719         "locator": "",
 720         "nspace": "",
 721         "name": "ROBJ13"
 722       }
 723     },
 724     {
 725       "shards": [
 726         {
 727           "size": 7,
 728           "attrs": [
 729             {
 730               "Base64": false,
 731               "value": "",
 732               "name": "_"
 733             },
 734             {
 735               "Base64": true,
 736               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 737               "name": "snapset"
 738             }
 739           ],
 740           "errors": [
 741             "oi_attr_corrupted"
 742           ],
 743           "osd": 0,
 744           "primary": false
 745         },
 746         {
 747           "size": 7,
 748           "attrs": [
 749             {
 750               "Base64": true,
 751               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 752               "name": "snapset"
 753             }
 754           ],
 755           "errors": [
 756             "oi_attr_missing"
 757           ],
 758           "osd": 1,
 759           "primary": true
 760         }
 761       ],
 762       "union_shard_errors": [
 763         "oi_attr_missing",
 764         "oi_attr_corrupted"
 765       ],
 766       "errors": [],
 767       "object": {
 768         "version": 0,
 769         "snap": "head",
 770         "locator": "",
 771         "nspace": "",
 772         "name": "ROBJ14"
 773       }
 774     },
 775     {
 776       "shards": [
 777         {
 778           "attrs": [
 779             {
 780               "Base64": true,
 781               "value": "",
 782               "name": "_"
 783             },
 784             {
 785               "Base64": true,
 786               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 787               "name": "snapset"
 788             }
 789           ],
 790           "size": 7,
 791           "errors": [],
 792           "osd": 0,
 793           "primary": false
 794         },
 795         {
 796           "attrs": [
 797             {
 798               "Base64": true,
 799               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 800               "name": "snapset"
 801             }
 802           ],
 803           "size": 7,
 804           "errors": [
 805             "oi_attr_missing"
 806           ],
 807           "osd": 1,
 808           "primary": true
 809         }
 810       ],
 811       "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])",
 812       "union_shard_errors": [
 813         "oi_attr_missing"
 814       ],
 815       "errors": [],
 816       "object": {
 817         "version": 45,
 818         "snap": "head",
 819         "locator": "",
 820         "nspace": "",
 821         "name": "ROBJ15"
 822       }
 823     },
 824     {
 825       "errors": [],
 826       "object": {
 827       "locator": "",
 828       "name": "ROBJ16",
 829       "nspace": "",
 830       "snap": "head",
 831       "version": 0
 832        },
 833         "shards": [
 834       {
 835         "attrs": [
 836           {
 837             "Base64": true,
 838             "name": "_",
 839             "value": ""
 840           }
 841         ],
 842         "errors": [
 843           "ss_attr_missing"
 844         ],
 845         "osd": 0,
 846         "primary": false,
 847         "size": 7
 848       },
 849       {
 850         "attrs": [
 851           {
 852             "Base64": true,
 853             "name": "_",
 854             "value": ""
 855           },
 856           {
 857             "Base64": false,
 858             "name": "snapset",
 859             "value": "bad-val"
 860           }
 861         ],
 862         "errors": [
 863           "ss_attr_corrupted"
 864         ],
 865         "osd": 1,
 866         "primary": true,
 867         "size": 7
 868       }
 869       ],
 870       "union_shard_errors": [
 871         "ss_attr_missing",
 872         "ss_attr_corrupted"
 873       ]
 874     },
 875     {
 876       "shards": [
 877         {
 878           "size": 7,
 879           "errors": [],
 880           "osd": 0,
 881           "primary": false
 882         },
 883         {
 884           "errors": [
 885             "missing"
 886           ],
 887           "osd": 1,
 888           "primary": true
 889         }
 890       ],
 891       "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])",
 892       "union_shard_errors": [
 893         "missing"
 894       ],
 895       "errors": [],
 896       "object": {
 897         "version": 9,
 898         "snap": "head",
 899         "locator": "",
 900         "nspace": "",
 901         "name": "ROBJ3"
 902       }
 903     },
 904     {
 905       "shards": [
 906         {
 907           "attrs": [
 908             {
 909               "Base64": true,
 910               "value": "",
 911               "name": "_"
 912             },
 913             {
 914               "Base64": false,
 915               "value": "bad-val",
 916               "name": "_key1-ROBJ8"
 917             },
 918             {
 919               "Base64": false,
 920               "value": "val3-ROBJ8",
 921               "name": "_key3-ROBJ8"
 922             },
 923             {
 924               "Base64": true,
 925               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 926               "name": "snapset"
 927             }
 928           ],
 929           "size": 7,
 930           "errors": [],
 931           "osd": 0,
 932           "primary": false
 933         },
 934         {
 935           "attrs": [
 936             {
 937               "Base64": true,
 938               "value": "",
 939               "name": "_"
 940             },
 941             {
 942               "Base64": false,
 943               "value": "val1-ROBJ8",
 944               "name": "_key1-ROBJ8"
 945             },
 946             {
 947               "Base64": false,
 948               "value": "val2-ROBJ8",
 949               "name": "_key2-ROBJ8"
 950             },
 951             {
 952               "Base64": true,
 953               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 954               "name": "snapset"
 955             }
 956           ],
 957           "size": 7,
 958           "errors": [],
 959           "osd": 1,
 960           "primary": true
 961         }
 962       ],
 963       "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 66 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
 964       "union_shard_errors": [],
 965       "errors": [
 966         "attr_value_mismatch",
 967         "attr_name_mismatch"
 968       ],
 969       "object": {
 970         "version": 66,
 971         "snap": "head",
 972         "locator": "",
 973         "nspace": "",
 974         "name": "ROBJ8"
 975       }
 976     },
 977     {
 978       "shards": [
 979         {
 980           "object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 67 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
 981           "size": 1,
 982           "errors": [],
 983           "osd": 0,
 984           "primary": false
 985         },
 986         {
 987           "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])",
 988           "size": 1,
 989           "errors": [
 990             "obj_size_oi_mismatch"
 991           ],
 992           "osd": 1,
 993           "primary": true
 994         }
 995       ],
 996       "selected_object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 67 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
 997       "union_shard_errors": [
 998          "obj_size_oi_mismatch"
 999       ],
1000       "errors": [
1001         "object_info_inconsistency"
1002       ],
1003       "object": {
1004         "version": 67,
1005         "snap": "head",
1006         "locator": "",
1007         "nspace": "",
1008         "name": "ROBJ9"
1009       }
1010     }
1011   ],
1012   "epoch": 0
1013 }
1014 EOF
1015
1016     jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
1017     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
1018     if test $getjson = "yes"
1019     then
1020         jq '.' $dir/json > save1.json
1021     fi
1022
1023     if which jsonschema > /dev/null;
1024     then
1025       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
1026     fi
1027
1028     objname=ROBJ9
1029     # Change data and size again because digest was recomputed
1030     echo -n ZZZ > $dir/change
1031     rados --pool $poolname put $objname $dir/change
1032     # Set one to an even older value
1033     objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi
1034     rm $dir/oi $dir/change
1035
1036     objname=ROBJ10
1037     objectstore_tool $dir 1 $objname get-attr _ > $dir/oi
1038     rados --pool $poolname setomapval $objname key2-$objname val2-$objname
1039     objectstore_tool $dir 0 $objname set-attr _ $dir/oi
1040     objectstore_tool $dir 1 $objname set-attr _ $dir/oi
1041     rm $dir/oi
1042
1043     inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1044     inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1045     inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1046     inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1047     pg_deep_scrub $pg
1048
1049     rados list-inconsistent-pg $poolname > $dir/json || return 1
1050     # Check pg count
1051     test $(jq '. | length' $dir/json) = "1" || return 1
1052     # Check pgid
1053     test $(jq -r '.[0]' $dir/json) = $pg || return 1
1054
1055     rados list-inconsistent-obj $pg > $dir/json || return 1
1056     # Get epoch for repair-get requests
1057     epoch=$(jq .epoch $dir/json)
1058
1059     jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
1060 {
1061   "inconsistents": [
1062     {
1063       "shards": [
1064         {
1065           "data_digest": "0x2ddbf8f5",
1066           "omap_digest": "0xf5fba2c6",
1067           "size": 7,
1068           "errors": [],
1069           "osd": 0,
1070           "primary": false
1071         },
1072         {
1073           "data_digest": "0x2d4a11c2",
1074           "omap_digest": "0xf5fba2c6",
1075           "size": 9,
1076           "errors": [
1077             "data_digest_mismatch_oi",
1078             "size_mismatch_oi",
1079             "obj_size_oi_mismatch"
1080           ],
1081           "osd": 1,
1082           "primary": true
1083         }
1084       ],
1085       "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])",
1086       "union_shard_errors": [
1087         "data_digest_mismatch_oi",
1088         "size_mismatch_oi",
1089         "obj_size_oi_mismatch"
1090       ],
1091       "errors": [
1092         "data_digest_mismatch",
1093         "size_mismatch"
1094       ],
1095       "object": {
1096         "version": 3,
1097         "snap": "head",
1098         "locator": "",
1099         "nspace": "",
1100         "name": "ROBJ1"
1101       }
1102     },
1103     {
1104       "shards": [
1105         {
1106           "data_digest": "0x2ddbf8f5",
1107           "omap_digest": "0xa8dd5adc",
1108           "size": 7,
1109           "errors": [
1110             "omap_digest_mismatch_oi"
1111           ],
1112           "osd": 0,
1113           "primary": false
1114         },
1115         {
1116           "data_digest": "0x2ddbf8f5",
1117           "omap_digest": "0xa8dd5adc",
1118           "size": 7,
1119           "errors": [
1120             "omap_digest_mismatch_oi"
1121           ],
1122           "osd": 1,
1123           "primary": true
1124         }
1125       ],
1126       "selected_object_info": "3:b1f19cbd:::ROBJ10:head(47'51 osd.0.0:50 dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [0 0 0])",
1127       "union_shard_errors": [
1128         "omap_digest_mismatch_oi"
1129       ],
1130       "errors": [],
1131       "object": {
1132         "version": 30,
1133         "snap": "head",
1134         "locator": "",
1135         "nspace": "",
1136         "name": "ROBJ10"
1137       }
1138     },
1139     {
1140       "shards": [
1141         {
1142           "data_digest": "0x2ddbf8f5",
1143           "omap_digest": "0xa03cef03",
1144           "size": 7,
1145           "errors": [],
1146           "osd": 0,
1147           "primary": false
1148         },
1149         {
1150           "size": 7,
1151           "errors": [
1152             "read_error"
1153           ],
1154           "osd": 1,
1155           "primary": true
1156         }
1157       ],
1158       "selected_object_info": "3:87abbf36:::ROBJ11:head(47'48 osd.0.0:47 dirty|omap|data_digest|omap_digest s 7 uv 33 dd 2ddbf8f5 od a03cef03 alloc_hint [0 0 0])",
1159       "union_shard_errors": [
1160         "read_error"
1161       ],
1162       "errors": [],
1163       "object": {
1164         "version": 33,
1165         "snap": "head",
1166         "locator": "",
1167         "nspace": "",
1168         "name": "ROBJ11"
1169       }
1170     },
1171     {
1172       "shards": [
1173         {
1174           "errors": [
1175             "stat_error"
1176           ],
1177           "osd": 0,
1178           "primary": false
1179         },
1180         {
1181           "data_digest": "0x2ddbf8f5",
1182           "omap_digest": "0x067f306a",
1183           "size": 7,
1184           "errors": [],
1185           "osd": 1,
1186           "primary": true
1187         }
1188       ],
1189       "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])",
1190       "union_shard_errors": [
1191         "stat_error"
1192       ],
1193       "errors": [],
1194       "object": {
1195         "version": 36,
1196         "snap": "head",
1197         "locator": "",
1198         "nspace": "",
1199         "name": "ROBJ12"
1200       }
1201     },
1202     {
1203       "shards": [
1204         {
1205           "errors": [
1206             "stat_error"
1207           ],
1208           "osd": 0,
1209           "primary": false
1210         },
1211         {
1212           "size": 7,
1213           "errors": [
1214             "read_error"
1215           ],
1216           "osd": 1,
1217           "primary": true
1218         }
1219       ],
1220       "union_shard_errors": [
1221         "stat_error",
1222         "read_error"
1223       ],
1224       "errors": [],
1225       "object": {
1226         "version": 0,
1227         "snap": "head",
1228         "locator": "",
1229         "nspace": "",
1230         "name": "ROBJ13"
1231       }
1232     },
1233     {
1234       "shards": [
1235         {
1236           "attrs": [
1237             {
1238               "Base64": false,
1239               "value": "",
1240               "name": "_"
1241             },
1242             {
1243               "Base64": true,
1244               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1245               "name": "snapset"
1246             }
1247           ],
1248           "data_digest": "0x2ddbf8f5",
1249           "omap_digest": "0x4f14f849",
1250           "size": 7,
1251           "errors": [
1252             "oi_attr_corrupted"
1253           ],
1254           "osd": 0,
1255           "primary": false
1256         },
1257         {
1258           "attrs": [
1259             {
1260               "Base64": true,
1261               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1262               "name": "snapset"
1263             }
1264           ],
1265           "data_digest": "0x2ddbf8f5",
1266           "omap_digest": "0x4f14f849",
1267           "size": 7,
1268           "errors": [
1269             "oi_attr_missing"
1270           ],
1271           "osd": 1,
1272           "primary": true
1273         }
1274       ],
1275       "union_shard_errors": [
1276         "oi_attr_missing",
1277         "oi_attr_corrupted"
1278       ],
1279       "errors": [],
1280       "object": {
1281         "version": 0,
1282         "snap": "head",
1283         "locator": "",
1284         "nspace": "",
1285         "name": "ROBJ14"
1286       }
1287     },
1288     {
1289       "shards": [
1290         {
1291           "attrs": [
1292             {
1293               "Base64": true,
1294               "value": "",
1295               "name": "_"
1296             },
1297             {
1298               "Base64": true,
1299               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1300               "name": "snapset"
1301             }
1302           ],
1303           "data_digest": "0x2ddbf8f5",
1304           "omap_digest": "0x2d2a4d6e",
1305           "size": 7,
1306           "errors": [],
1307           "osd": 0,
1308           "primary": false
1309         },
1310         {
1311           "attrs": [
1312             {
1313               "Base64": true,
1314               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1315               "name": "snapset"
1316             }
1317           ],
1318           "data_digest": "0x2ddbf8f5",
1319           "omap_digest": "0x2d2a4d6e",
1320           "size": 7,
1321           "errors": [
1322             "oi_attr_missing"
1323           ],
1324           "osd": 1,
1325           "primary": true
1326         }
1327       ],
1328       "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])",
1329       "union_shard_errors": [
1330         "oi_attr_missing"
1331       ],
1332       "errors": [],
1333       "object": {
1334         "version": 45,
1335         "snap": "head",
1336         "locator": "",
1337         "nspace": "",
1338         "name": "ROBJ15"
1339       }
1340     },
1341     {
1342       "errors": [],
1343       "object": {
1344       "locator": "",
1345       "name": "ROBJ16",
1346       "nspace": "",
1347       "snap": "head",
1348       "version": 0
1349        },
1350         "shards": [
1351       {
1352         "attrs": [
1353           {
1354             "Base64": true,
1355             "name": "_",
1356             "value": ""
1357           }
1358         ],
1359         "data_digest": "0x2ddbf8f5",
1360         "errors": [
1361           "ss_attr_missing"
1362         ],
1363         "omap_digest": "0x8b699207",
1364         "osd": 0,
1365         "primary": false,
1366         "size": 7
1367       },
1368       {
1369         "attrs": [
1370           {
1371             "Base64": true,
1372             "name": "_",
1373             "value": ""
1374           },
1375           {
1376             "Base64": false,
1377             "name": "snapset",
1378             "value": "bad-val"
1379           }
1380         ],
1381         "data_digest": "0x2ddbf8f5",
1382         "errors": [
1383           "ss_attr_corrupted"
1384         ],
1385         "omap_digest": "0x8b699207",
1386         "osd": 1,
1387         "primary": true,
1388         "size": 7
1389       }
1390       ],
1391       "union_shard_errors": [
1392         "ss_attr_missing",
1393         "ss_attr_corrupted"
1394       ]
1395     },
1396     {
1397       "shards": [
1398         {
1399           "data_digest": "0x578a4830",
1400           "omap_digest": "0xf8e11918",
1401           "size": 7,
1402           "errors": [
1403             "data_digest_mismatch_oi"
1404           ],
1405           "osd": 0,
1406           "primary": false
1407         },
1408         {
1409           "data_digest": "0x2ddbf8f5",
1410           "omap_digest": "0xf8e11918",
1411           "size": 7,
1412           "errors": [],
1413           "osd": 1,
1414           "primary": true
1415         }
1416       ],
1417       "selected_object_info": "3:e97ce31e:::ROBJ2:head(47'56 osd.0.0:55 dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [0 0 0])",
1418       "union_shard_errors": [
1419         "data_digest_mismatch_oi"
1420       ],
1421       "errors": [
1422         "data_digest_mismatch"
1423       ],
1424       "object": {
1425         "version": 6,
1426         "snap": "head",
1427         "locator": "",
1428         "nspace": "",
1429         "name": "ROBJ2"
1430       }
1431     },
1432     {
1433       "shards": [
1434         {
1435           "data_digest": "0x2ddbf8f5",
1436           "omap_digest": "0x00b35dfd",
1437           "size": 7,
1438           "errors": [],
1439           "osd": 0,
1440           "primary": false
1441         },
1442         {
1443           "errors": [
1444             "missing"
1445           ],
1446           "osd": 1,
1447           "primary": true
1448         }
1449       ],
1450       "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])",
1451       "union_shard_errors": [
1452         "missing"
1453       ],
1454       "errors": [],
1455       "object": {
1456         "version": 9,
1457         "snap": "head",
1458         "locator": "",
1459         "nspace": "",
1460         "name": "ROBJ3"
1461       }
1462     },
1463     {
1464       "shards": [
1465         {
1466           "data_digest": "0x2ddbf8f5",
1467           "omap_digest": "0xd7178dfe",
1468           "size": 7,
1469           "errors": [
1470             "omap_digest_mismatch_oi"
1471           ],
1472           "osd": 0,
1473           "primary": false
1474         },
1475         {
1476           "data_digest": "0x2ddbf8f5",
1477           "omap_digest": "0xe2d46ea4",
1478           "size": 7,
1479           "errors": [],
1480           "osd": 1,
1481           "primary": true
1482         }
1483       ],
1484       "selected_object_info": "3:f4981d31:::ROBJ4:head(47'58 osd.0.0:57 dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [0 0 0])",
1485       "union_shard_errors": [
1486         "omap_digest_mismatch_oi"
1487       ],
1488       "errors": [
1489         "omap_digest_mismatch"
1490       ],
1491       "object": {
1492         "version": 12,
1493         "snap": "head",
1494         "locator": "",
1495         "nspace": "",
1496         "name": "ROBJ4"
1497       }
1498     },
1499     {
1500       "shards": [
1501         {
1502           "data_digest": "0x2ddbf8f5",
1503           "omap_digest": "0x1a862a41",
1504           "size": 7,
1505           "errors": [],
1506           "osd": 0,
1507           "primary": false
1508         },
1509         {
1510           "data_digest": "0x2ddbf8f5",
1511           "omap_digest": "0x06cac8f6",
1512           "size": 7,
1513           "errors": [
1514             "omap_digest_mismatch_oi"
1515           ],
1516           "osd": 1,
1517           "primary": true
1518         }
1519       ],
1520       "selected_object_info": "3:f4bfd4d1:::ROBJ5:head(47'59 osd.0.0:58 dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [0 0 0])",
1521       "union_shard_errors": [
1522         "omap_digest_mismatch_oi"
1523       ],
1524       "errors": [
1525         "omap_digest_mismatch"
1526       ],
1527       "object": {
1528         "version": 15,
1529         "snap": "head",
1530         "locator": "",
1531         "nspace": "",
1532         "name": "ROBJ5"
1533       }
1534     },
1535     {
1536       "shards": [
1537         {
1538           "data_digest": "0x2ddbf8f5",
1539           "omap_digest": "0x689ee887",
1540           "size": 7,
1541           "errors": [
1542             "omap_digest_mismatch_oi"
1543           ],
1544           "osd": 0,
1545           "primary": false
1546         },
1547         {
1548           "data_digest": "0x2ddbf8f5",
1549           "omap_digest": "0x179c919f",
1550           "size": 7,
1551           "errors": [],
1552           "osd": 1,
1553           "primary": true
1554         }
1555       ],
1556       "selected_object_info": "3:a53c12e8:::ROBJ6:head(47'50 osd.0.0:49 dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [0 0 0])",
1557       "union_shard_errors": [
1558         "omap_digest_mismatch_oi"
1559       ],
1560       "errors": [
1561         "omap_digest_mismatch"
1562       ],
1563       "object": {
1564         "version": 18,
1565         "snap": "head",
1566         "locator": "",
1567         "nspace": "",
1568         "name": "ROBJ6"
1569       }
1570     },
1571     {
1572       "shards": [
1573         {
1574           "data_digest": "0x2ddbf8f5",
1575           "omap_digest": "0xefced57a",
1576           "size": 7,
1577           "errors": [],
1578           "osd": 0,
1579           "primary": false
1580         },
1581         {
1582           "data_digest": "0x2ddbf8f5",
1583           "omap_digest": "0x6a73cc07",
1584           "size": 7,
1585           "errors": [
1586             "omap_digest_mismatch_oi"
1587           ],
1588           "osd": 1,
1589           "primary": true
1590         }
1591       ],
1592       "selected_object_info": "3:8b55fa4b:::ROBJ7:head(47'49 osd.0.0:48 dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [0 0 0])",
1593       "union_shard_errors": [
1594         "omap_digest_mismatch_oi"
1595       ],
1596       "errors": [
1597         "omap_digest_mismatch"
1598       ],
1599       "object": {
1600         "version": 21,
1601         "snap": "head",
1602         "locator": "",
1603         "nspace": "",
1604         "name": "ROBJ7"
1605       }
1606     },
1607     {
1608       "shards": [
1609         {
1610           "attrs": [
1611             {
1612               "Base64": true,
1613               "value": "",
1614               "name": "_"
1615             },
1616             {
1617               "Base64": false,
1618               "value": "bad-val",
1619               "name": "_key1-ROBJ8"
1620             },
1621             {
1622               "Base64": false,
1623               "value": "val3-ROBJ8",
1624               "name": "_key3-ROBJ8"
1625             },
1626             {
1627               "Base64": true,
1628               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1629               "name": "snapset"
1630             }
1631           ],
1632           "data_digest": "0x2ddbf8f5",
1633           "omap_digest": "0xd6be81dc",
1634           "size": 7,
1635           "errors": [],
1636           "osd": 0,
1637           "primary": false
1638         },
1639         {
1640           "attrs": [
1641             {
1642               "Base64": true,
1643               "value": "",
1644               "name": "_"
1645             },
1646             {
1647               "Base64": false,
1648               "value": "val1-ROBJ8",
1649               "name": "_key1-ROBJ8"
1650             },
1651             {
1652               "Base64": false,
1653               "value": "val2-ROBJ8",
1654               "name": "_key2-ROBJ8"
1655             },
1656             {
1657               "Base64": true,
1658               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1659               "name": "snapset"
1660             }
1661           ],
1662           "data_digest": "0x2ddbf8f5",
1663           "omap_digest": "0xd6be81dc",
1664           "size": 7,
1665           "errors": [],
1666           "osd": 1,
1667           "primary": true
1668         }
1669       ],
1670       "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 66 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
1671       "union_shard_errors": [],
1672       "errors": [
1673         "attr_value_mismatch",
1674         "attr_name_mismatch"
1675       ],
1676       "object": {
1677         "version": 66,
1678         "snap": "head",
1679         "locator": "",
1680         "nspace": "",
1681         "name": "ROBJ8"
1682       }
1683     },
1684     {
1685       "shards": [
1686         {
1687           "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])",
1688           "data_digest": "0x1f26fb26",
1689           "omap_digest": "0x2eecc539",
1690           "size": 3,
1691           "errors": [
1692             "obj_size_oi_mismatch"
1693           ],
1694           "osd": 0,
1695           "primary": false
1696         },
1697         {
1698           "object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 68 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
1699           "data_digest": "0x1f26fb26",
1700           "omap_digest": "0x2eecc539",
1701           "size": 3,
1702           "errors": [],
1703           "osd": 1,
1704           "primary": true
1705         }
1706       ],
1707       "selected_object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 68 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
1708       "union_shard_errors": [
1709         "obj_size_oi_mismatch"
1710       ],
1711       "errors": [
1712         "object_info_inconsistency"
1713       ],
1714       "object": {
1715         "version": 68,
1716         "snap": "head",
1717         "locator": "",
1718         "nspace": "",
1719         "name": "ROBJ9"
1720       }
1721     }
1722   ],
1723   "epoch": 0
1724 }
1725 EOF
1726
1727     jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
1728     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
1729     if test $getjson = "yes"
1730     then
1731         jq '.' $dir/json > save2.json
1732     fi
1733
1734     if which jsonschema > /dev/null;
1735     then
1736       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
1737     fi
1738
1739     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
1740     teardown $dir || return 1
1741 }
1742
1743
1744 #
1745 # Test scrub errors for an erasure coded pool
1746 #
1747 function corrupt_scrub_erasure() {
1748     local dir=$1
1749     local allow_overwrites=$2
1750     local poolname=ecpool
1751     local total_objs=5
1752
1753     setup $dir || return 1
1754     run_mon $dir a || return 1
1755     run_mgr $dir x || return 1
1756     for id in $(seq 0 2) ; do
1757         if [ "$allow_overwrites" = "true" ]; then
1758             run_osd_bluestore $dir $id || return 1
1759         else
1760             run_osd $dir $id || return 1
1761         fi
1762     done
1763     create_rbd_pool || return 1
1764     create_pool foo 1
1765
1766     create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1
1767     wait_for_clean || return 1
1768
1769     for i in $(seq 1 $total_objs) ; do
1770         objname=EOBJ${i}
1771         add_something $dir $poolname $objname || return 1
1772
1773         local osd=$(expr $i % 2)
1774
1775         case $i in
1776         1)
1777             # Size (deep scrub data_digest too)
1778             local payload=UVWXYZZZ
1779             echo $payload > $dir/CORRUPT
1780             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1781             ;;
1782
1783         2)
1784             # Corrupt EC shard
1785             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1
1786             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1787             ;;
1788
1789         3)
1790              # missing
1791              objectstore_tool $dir $osd $objname remove || return 1
1792              ;;
1793
1794         4)
1795             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
1796             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
1797
1798             # Break xattrs
1799             echo -n bad-val > $dir/bad-val
1800             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
1801             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
1802             echo -n val3-$objname > $dir/newval
1803             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
1804             rm $dir/bad-val $dir/newval
1805             ;;
1806
1807         5)
1808             # Corrupt EC shard
1809             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2
1810             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1811             ;;
1812
1813         esac
1814     done
1815
1816     local pg=$(get_pg $poolname EOBJ0)
1817
1818     pg_scrub $pg
1819
1820     rados list-inconsistent-pg $poolname > $dir/json || return 1
1821     # Check pg count
1822     test $(jq '. | length' $dir/json) = "1" || return 1
1823     # Check pgid
1824     test $(jq -r '.[0]' $dir/json) = $pg || return 1
1825
1826     rados list-inconsistent-obj $pg > $dir/json || return 1
1827     # Get epoch for repair-get requests
1828     epoch=$(jq .epoch $dir/json)
1829
1830     jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
1831 {
1832   "inconsistents": [
1833     {
1834       "shards": [
1835         {
1836           "size": 2048,
1837           "errors": [],
1838           "shard": 2,
1839           "osd": 0,
1840           "primary": false
1841         },
1842         {
1843           "size": 9,
1844           "shard": 0,
1845           "errors": [
1846             "size_mismatch_oi",
1847             "obj_size_oi_mismatch"
1848           ],
1849           "osd": 1,
1850           "primary": true
1851         },
1852         {
1853           "size": 2048,
1854           "shard": 1,
1855           "errors": [],
1856           "osd": 2,
1857           "primary": false
1858         }
1859       ],
1860       "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1861       "union_shard_errors": [
1862         "size_mismatch_oi",
1863         "obj_size_oi_mismatch"
1864       ],
1865       "errors": [
1866         "size_mismatch"
1867       ],
1868       "object": {
1869         "version": 1,
1870         "snap": "head",
1871         "locator": "",
1872         "nspace": "",
1873         "name": "EOBJ1"
1874       }
1875     },
1876     {
1877       "shards": [
1878         {
1879           "size": 2048,
1880           "errors": [],
1881           "shard": 2,
1882           "osd": 0,
1883           "primary": false
1884         },
1885         {
1886           "shard": 0,
1887           "errors": [
1888             "missing"
1889           ],
1890           "osd": 1,
1891           "primary": true
1892         },
1893         {
1894           "size": 2048,
1895           "shard": 1,
1896           "errors": [],
1897           "osd": 2,
1898           "primary": false
1899         }
1900       ],
1901       "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1902       "union_shard_errors": [
1903         "missing"
1904       ],
1905       "errors": [],
1906       "object": {
1907         "version": 3,
1908         "snap": "head",
1909         "locator": "",
1910         "nspace": "",
1911         "name": "EOBJ3"
1912       }
1913     },
1914     {
1915       "shards": [
1916         {
1917           "attrs": [
1918             {
1919               "Base64": true,
1920               "value": "",
1921               "name": "_"
1922             },
1923             {
1924               "Base64": false,
1925               "value": "bad-val",
1926               "name": "_key1-EOBJ4"
1927             },
1928             {
1929               "Base64": false,
1930               "value": "val3-EOBJ4",
1931               "name": "_key3-EOBJ4"
1932             },
1933             {
1934               "Base64": true,
1935               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1936               "name": "hinfo_key"
1937             },
1938             {
1939               "Base64": true,
1940               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1941               "name": "snapset"
1942             }
1943           ],
1944           "size": 2048,
1945           "errors": [],
1946           "shard": 2,
1947           "osd": 0,
1948           "primary": false
1949         },
1950         {
1951           "osd": 1,
1952           "primary": true,
1953           "shard": 0,
1954           "errors": [],
1955           "size": 2048,
1956           "attrs": [
1957             {
1958               "Base64": true,
1959               "value": "",
1960               "name": "_"
1961             },
1962             {
1963               "Base64": false,
1964               "value": "val1-EOBJ4",
1965               "name": "_key1-EOBJ4"
1966             },
1967             {
1968               "Base64": false,
1969               "value": "val2-EOBJ4",
1970               "name": "_key2-EOBJ4"
1971             },
1972             {
1973               "Base64": true,
1974               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1975               "name": "hinfo_key"
1976             },
1977             {
1978               "Base64": true,
1979               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1980               "name": "snapset"
1981             }
1982           ]
1983         },
1984         {
1985           "osd": 2,
1986           "primary": false,
1987           "shard": 1,
1988           "errors": [],
1989           "size": 2048,
1990           "attrs": [
1991             {
1992               "Base64": true,
1993               "value": "",
1994               "name": "_"
1995             },
1996             {
1997               "Base64": false,
1998               "value": "val1-EOBJ4",
1999               "name": "_key1-EOBJ4"
2000             },
2001             {
2002               "Base64": false,
2003               "value": "val2-EOBJ4",
2004               "name": "_key2-EOBJ4"
2005             },
2006             {
2007               "Base64": true,
2008               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2009               "name": "hinfo_key"
2010             },
2011             {
2012               "Base64": true,
2013               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2014               "name": "snapset"
2015             }
2016           ]
2017         }
2018       ],
2019       "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2020       "union_shard_errors": [],
2021       "errors": [
2022         "attr_value_mismatch",
2023         "attr_name_mismatch"
2024       ],
2025       "object": {
2026         "version": 6,
2027         "snap": "head",
2028         "locator": "",
2029         "nspace": "",
2030         "name": "EOBJ4"
2031       }
2032     },
2033     {
2034       "shards": [
2035         {
2036           "size": 2048,
2037           "errors": [],
2038           "shard": 2,
2039           "osd": 0,
2040           "primary": false
2041         },
2042         {
2043           "size": 4096,
2044           "shard": 0,
2045           "errors": [
2046             "size_mismatch_oi",
2047             "obj_size_oi_mismatch"
2048           ],
2049           "osd": 1,
2050           "primary": true
2051         },
2052         {
2053           "size": 2048,
2054           "shard": 1,
2055           "errors": [],
2056           "osd": 2,
2057           "primary": false
2058         }
2059       ],
2060       "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2061       "union_shard_errors": [
2062         "size_mismatch_oi",
2063         "obj_size_oi_mismatch"
2064       ],
2065       "errors": [
2066         "size_mismatch"
2067       ],
2068       "object": {
2069         "version": 7,
2070         "snap": "head",
2071         "locator": "",
2072         "nspace": "",
2073         "name": "EOBJ5"
2074       }
2075     }
2076   ],
2077   "epoch": 0
2078 }
2079 EOF
2080
2081     jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
2082     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
2083     if test $getjson = "yes"
2084     then
2085         jq '.' $dir/json > save3.json
2086     fi
2087
2088     if which jsonschema > /dev/null;
2089     then
2090       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
2091     fi
2092
2093     pg_deep_scrub $pg
2094
2095     rados list-inconsistent-pg $poolname > $dir/json || return 1
2096     # Check pg count
2097     test $(jq '. | length' $dir/json) = "1" || return 1
2098     # Check pgid
2099     test $(jq -r '.[0]' $dir/json) = $pg || return 1
2100
2101     rados list-inconsistent-obj $pg > $dir/json || return 1
2102     # Get epoch for repair-get requests
2103     epoch=$(jq .epoch $dir/json)
2104
2105     if [ "$allow_overwrites" = "true" ]
2106     then
2107       jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
2108 {
2109   "inconsistents": [
2110     {
2111       "shards": [
2112         {
2113           "data_digest": "0x00000000",
2114           "omap_digest": "0xffffffff",
2115           "size": 2048,
2116           "errors": [],
2117           "shard": 2,
2118           "osd": 0,
2119           "primary": false
2120         },
2121         {
2122           "size": 9,
2123           "shard": 0,
2124           "errors": [
2125             "read_error",
2126             "size_mismatch_oi",
2127             "obj_size_oi_mismatch"
2128           ],
2129           "osd": 1,
2130           "primary": true
2131         },
2132         {
2133           "data_digest": "0x00000000",
2134           "omap_digest": "0xffffffff",
2135           "size": 2048,
2136           "shard": 1,
2137           "errors": [],
2138           "osd": 2,
2139           "primary": false
2140         }
2141       ],
2142       "selected_object_info": "3:9175b684:::EOBJ1:head(27'1 client.4155.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2143       "union_shard_errors": [
2144         "read_error",
2145         "size_mismatch_oi",
2146         "obj_size_oi_mismatch"
2147       ],
2148       "errors": [
2149         "size_mismatch"
2150       ],
2151       "object": {
2152         "version": 1,
2153         "snap": "head",
2154         "locator": "",
2155         "nspace": "",
2156         "name": "EOBJ1"
2157       }
2158     },
2159     {
2160       "shards": [
2161         {
2162           "data_digest": "0x00000000",
2163           "omap_digest": "0xffffffff",
2164           "size": 2048,
2165           "errors": [],
2166           "shard": 2,
2167           "osd": 0,
2168           "primary": false
2169         },
2170         {
2171           "shard": 0,
2172           "errors": [
2173             "missing"
2174           ],
2175           "osd": 1,
2176           "primary": true
2177         },
2178         {
2179           "data_digest": "0x00000000",
2180           "omap_digest": "0xffffffff",
2181           "size": 2048,
2182           "shard": 1,
2183           "errors": [],
2184           "osd": 2,
2185           "primary": false
2186         }
2187       ],
2188       "selected_object_info": "3:b197b25d:::EOBJ3:head(41'3 client.4199.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2189       "union_shard_errors": [
2190         "missing"
2191       ],
2192       "errors": [],
2193       "object": {
2194         "version": 3,
2195         "snap": "head",
2196         "locator": "",
2197         "nspace": "",
2198         "name": "EOBJ3"
2199       }
2200     },
2201     {
2202       "shards": [
2203         {
2204           "attrs": [
2205             {
2206               "Base64": true,
2207               "value": "",
2208               "name": "_"
2209             },
2210             {
2211               "Base64": false,
2212               "value": "bad-val",
2213               "name": "_key1-EOBJ4"
2214             },
2215             {
2216               "Base64": false,
2217               "value": "val3-EOBJ4",
2218               "name": "_key3-EOBJ4"
2219             },
2220             {
2221               "Base64": true,
2222               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2223               "name": "hinfo_key"
2224             },
2225             {
2226               "Base64": true,
2227               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2228               "name": "snapset"
2229             }
2230           ],
2231           "data_digest": "0x00000000",
2232           "omap_digest": "0xffffffff",
2233           "size": 2048,
2234           "errors": [],
2235           "shard": 2,
2236           "osd": 0,
2237           "primary": false
2238         },
2239         {
2240           "attrs": [
2241             {
2242               "Base64": true,
2243               "value": "",
2244               "name": "_"
2245             },
2246             {
2247               "Base64": false,
2248               "value": "val1-EOBJ4",
2249               "name": "_key1-EOBJ4"
2250             },
2251             {
2252               "Base64": false,
2253               "value": "val2-EOBJ4",
2254               "name": "_key2-EOBJ4"
2255             },
2256             {
2257               "Base64": true,
2258               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2259               "name": "hinfo_key"
2260             },
2261             {
2262               "Base64": true,
2263               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2264               "name": "snapset"
2265             }
2266           ],
2267           "data_digest": "0x00000000",
2268           "omap_digest": "0xffffffff",
2269           "size": 2048,
2270           "errors": [],
2271           "shard": 0,
2272           "osd": 1,
2273           "primary": true
2274         },
2275         {
2276           "attrs": [
2277             {
2278               "Base64": true,
2279               "value": "",
2280               "name": "_"
2281             },
2282             {
2283               "Base64": false,
2284               "value": "val1-EOBJ4",
2285               "name": "_key1-EOBJ4"
2286             },
2287             {
2288               "Base64": false,
2289               "value": "val2-EOBJ4",
2290               "name": "_key2-EOBJ4"
2291             },
2292             {
2293               "Base64": true,
2294               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2295               "name": "hinfo_key"
2296             },
2297             {
2298               "Base64": true,
2299               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2300               "name": "snapset"
2301             }
2302           ],
2303           "data_digest": "0x00000000",
2304           "omap_digest": "0xffffffff",
2305           "size": 2048,
2306           "errors": [],
2307           "shard": 1,
2308           "osd": 2,
2309           "primary": false
2310         }
2311       ],
2312       "selected_object_info": "3:5e723e06:::EOBJ4:head(48'6 client.4223.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2313       "union_shard_errors": [],
2314       "errors": [
2315         "attr_value_mismatch",
2316         "attr_name_mismatch"
2317       ],
2318       "object": {
2319         "version": 6,
2320         "snap": "head",
2321         "locator": "",
2322         "nspace": "",
2323         "name": "EOBJ4"
2324       }
2325     },
2326     {
2327       "shards": [
2328         {
2329           "data_digest": "0x00000000",
2330           "omap_digest": "0xffffffff",
2331           "size": 2048,
2332           "errors": [],
2333           "shard": 2,
2334           "osd": 0,
2335           "primary": false
2336         },
2337         {
2338           "data_digest": "0x00000000",
2339           "omap_digest": "0xffffffff",
2340           "size": 4096,
2341           "errors": [
2342             "size_mismatch_oi",
2343             "obj_size_oi_mismatch"
2344           ],
2345           "shard": 0,
2346           "osd": 1,
2347           "primary": true
2348         },
2349         {
2350           "data_digest": "0x00000000",
2351           "omap_digest": "0xffffffff",
2352           "size": 2048,
2353           "errors": [],
2354           "shard": 1,
2355           "osd": 2,
2356           "primary": false
2357         }
2358       ],
2359       "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4288.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2360       "union_shard_errors": [
2361         "size_mismatch_oi",
2362         "obj_size_oi_mismatch"
2363       ],
2364       "errors": [
2365         "size_mismatch"
2366       ],
2367       "object": {
2368         "version": 7,
2369         "snap": "head",
2370         "locator": "",
2371         "nspace": "",
2372         "name": "EOBJ5"
2373       }
2374     }
2375   ],
2376   "epoch": 0
2377 }
2378 EOF
2379
2380     else
2381
2382       jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
2383 {
2384   "inconsistents": [
2385     {
2386       "shards": [
2387         {
2388           "data_digest": "0x04cfa72f",
2389           "omap_digest": "0xffffffff",
2390           "size": 2048,
2391           "errors": [],
2392           "shard": 2,
2393           "osd": 0,
2394           "primary": false
2395         },
2396         {
2397           "size": 9,
2398           "shard": 0,
2399           "errors": [
2400             "read_error",
2401             "size_mismatch_oi",
2402             "obj_size_oi_mismatch"
2403           ],
2404           "osd": 1,
2405           "primary": true
2406         },
2407         {
2408           "data_digest": "0x04cfa72f",
2409           "omap_digest": "0xffffffff",
2410           "size": 2048,
2411           "shard": 1,
2412           "errors": [],
2413           "osd": 2,
2414           "primary": false
2415         }
2416       ],
2417       "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2418       "union_shard_errors": [
2419         "read_error",
2420         "size_mismatch_oi",
2421         "obj_size_oi_mismatch"
2422       ],
2423       "errors": [
2424         "size_mismatch"
2425       ],
2426       "object": {
2427         "version": 1,
2428         "snap": "head",
2429         "locator": "",
2430         "nspace": "",
2431         "name": "EOBJ1"
2432       }
2433     },
2434     {
2435       "shards": [
2436         {
2437           "size": 2048,
2438           "errors": [
2439             "ec_hash_error"
2440           ],
2441           "shard": 2,
2442           "osd": 0,
2443           "primary": false
2444         },
2445         {
2446           "data_digest": "0x04cfa72f",
2447           "omap_digest": "0xffffffff",
2448           "size": 2048,
2449           "errors": [],
2450           "shard": 0,
2451           "osd": 1,
2452           "primary": true
2453         },
2454         {
2455           "data_digest": "0x04cfa72f",
2456           "omap_digest": "0xffffffff",
2457           "size": 2048,
2458           "errors": [],
2459           "shard": 1,
2460           "osd": 2,
2461           "primary": false
2462         }
2463       ],
2464       "selected_object_info": "3:9babd184:::EOBJ2:head(29'2 client.4217.0:1 dirty|data_digest|omap_digest s 7 uv 2 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2465       "union_shard_errors": [
2466         "ec_hash_error"
2467       ],
2468       "errors": [],
2469       "object": {
2470         "version": 2,
2471         "snap": "head",
2472         "locator": "",
2473         "nspace": "",
2474         "name": "EOBJ2"
2475       }
2476     },
2477     {
2478       "shards": [
2479         {
2480           "data_digest": "0x04cfa72f",
2481           "omap_digest": "0xffffffff",
2482           "size": 2048,
2483           "errors": [],
2484           "shard": 2,
2485           "osd": 0,
2486           "primary": false
2487         },
2488         {
2489           "osd": 1,
2490           "primary": true,
2491           "shard": 0,
2492           "errors": [
2493             "missing"
2494           ]
2495         },
2496         {
2497           "data_digest": "0x04cfa72f",
2498           "omap_digest": "0xffffffff",
2499           "size": 2048,
2500           "shard": 1,
2501           "errors": [],
2502           "osd": 2,
2503           "primary": false
2504         }
2505       ],
2506       "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2507       "union_shard_errors": [
2508         "missing"
2509       ],
2510       "errors": [],
2511       "object": {
2512         "version": 3,
2513         "snap": "head",
2514         "locator": "",
2515         "nspace": "",
2516         "name": "EOBJ3"
2517       }
2518     },
2519     {
2520       "shards": [
2521         {
2522           "attrs": [
2523             {
2524               "Base64": true,
2525               "value": "",
2526               "name": "_"
2527             },
2528             {
2529               "Base64": false,
2530               "value": "bad-val",
2531               "name": "_key1-EOBJ4"
2532             },
2533             {
2534               "Base64": false,
2535               "value": "val3-EOBJ4",
2536               "name": "_key3-EOBJ4"
2537             },
2538             {
2539               "Base64": true,
2540               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2541               "name": "hinfo_key"
2542             },
2543             {
2544               "Base64": true,
2545               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2546               "name": "snapset"
2547             }
2548           ],
2549           "data_digest": "0x04cfa72f",
2550           "omap_digest": "0xffffffff",
2551           "size": 2048,
2552           "errors": [],
2553           "shard": 2,
2554           "osd": 0,
2555           "primary": false
2556         },
2557         {
2558           "osd": 1,
2559           "primary": true,
2560           "shard": 0,
2561           "errors": [],
2562           "size": 2048,
2563           "omap_digest": "0xffffffff",
2564           "data_digest": "0x04cfa72f",
2565           "attrs": [
2566             {
2567               "Base64": true,
2568               "value": "",
2569               "name": "_"
2570             },
2571             {
2572               "Base64": false,
2573               "value": "val1-EOBJ4",
2574               "name": "_key1-EOBJ4"
2575             },
2576             {
2577               "Base64": false,
2578               "value": "val2-EOBJ4",
2579               "name": "_key2-EOBJ4"
2580             },
2581             {
2582               "Base64": true,
2583               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2584               "name": "hinfo_key"
2585             },
2586             {
2587               "Base64": true,
2588               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2589               "name": "snapset"
2590             }
2591           ]
2592         },
2593         {
2594           "osd": 2,
2595           "primary": false,
2596           "shard": 1,
2597           "errors": [],
2598           "size": 2048,
2599           "omap_digest": "0xffffffff",
2600           "data_digest": "0x04cfa72f",
2601           "attrs": [
2602             {
2603               "Base64": true,
2604               "value": "",
2605               "name": "_"
2606             },
2607             {
2608               "Base64": false,
2609               "value": "val1-EOBJ4",
2610               "name": "_key1-EOBJ4"
2611             },
2612             {
2613               "Base64": false,
2614               "value": "val2-EOBJ4",
2615               "name": "_key2-EOBJ4"
2616             },
2617             {
2618               "Base64": true,
2619               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2620               "name": "hinfo_key"
2621             },
2622             {
2623               "Base64": true,
2624               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2625               "name": "snapset"
2626             }
2627           ]
2628         }
2629       ],
2630       "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2631       "union_shard_errors": [],
2632       "errors": [
2633         "attr_value_mismatch",
2634         "attr_name_mismatch"
2635       ],
2636       "object": {
2637         "version": 6,
2638         "snap": "head",
2639         "locator": "",
2640         "nspace": "",
2641         "name": "EOBJ4"
2642       }
2643     },
2644     {
2645       "shards": [
2646         {
2647           "data_digest": "0x04cfa72f",
2648           "omap_digest": "0xffffffff",
2649           "size": 2048,
2650           "errors": [],
2651           "shard": 2,
2652           "osd": 0,
2653           "primary": false
2654         },
2655         {
2656           "size": 4096,
2657           "shard": 0,
2658           "errors": [
2659             "size_mismatch_oi",
2660             "ec_size_error",
2661             "obj_size_oi_mismatch"
2662           ],
2663           "osd": 1,
2664           "primary": true
2665         },
2666         {
2667           "data_digest": "0x04cfa72f",
2668           "omap_digest": "0xffffffff",
2669           "size": 2048,
2670           "shard": 1,
2671           "errors": [],
2672           "osd": 2,
2673           "primary": false
2674         }
2675       ],
2676       "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2677       "union_shard_errors": [
2678         "size_mismatch_oi",
2679         "ec_size_error",
2680         "obj_size_oi_mismatch"
2681       ],
2682       "errors": [
2683         "size_mismatch"
2684       ],
2685       "object": {
2686         "version": 7,
2687         "snap": "head",
2688         "locator": "",
2689         "nspace": "",
2690         "name": "EOBJ5"
2691       }
2692     }
2693   ],
2694   "epoch": 0
2695 }
2696 EOF
2697
2698     fi
2699
2700     jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
2701     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
2702     if test $getjson = "yes"
2703     then
2704       if [ "$allow_overwrites" = "true" ]
2705       then
2706         num=4
2707       else
2708         num=5
2709       fi
2710       jq '.' $dir/json > save${num}.json
2711     fi
2712
2713     if which jsonschema > /dev/null;
2714     then
2715       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
2716     fi
2717
2718     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
2719     teardown $dir || return 1
2720 }
2721
2722 function TEST_corrupt_scrub_erasure_appends() {
2723     corrupt_scrub_erasure $1 false
2724 }
2725
2726 function TEST_corrupt_scrub_erasure_overwrites() {
2727     if [ "$use_ec_overwrite" = "true" ]; then
2728         corrupt_scrub_erasure $1 true
2729     fi
2730 }
2731
2732 #
2733 # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost
2734 #
2735 function TEST_periodic_scrub_replicated() {
2736     local dir=$1
2737     local poolname=psr_pool
2738     local objname=POBJ
2739
2740     setup $dir || return 1
2741     run_mon $dir a --osd_pool_default_size=2 || return 1
2742     run_mgr $dir x || return 1
2743     local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
2744     ceph_osd_args+="--osd_scrub_backoff_ratio=0"
2745     run_osd $dir 0 $ceph_osd_args || return 1
2746     run_osd $dir 1 $ceph_osd_args || return 1
2747     create_rbd_pool || return 1
2748     wait_for_clean || return 1
2749
2750     create_pool $poolname 1 1 || return 1
2751     wait_for_clean || return 1
2752
2753     local osd=0
2754     add_something $dir $poolname $objname scrub || return 1
2755     local primary=$(get_primary $poolname $objname)
2756     local pg=$(get_pg $poolname $objname)
2757
2758     # Add deep-scrub only error
2759     local payload=UVWXYZ
2760     echo $payload > $dir/CORRUPT
2761     # Uses $ceph_osd_args for osd restart
2762     objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2763
2764     # No scrub information available, so expect failure
2765     set -o pipefail
2766     !  rados list-inconsistent-obj $pg | jq '.' || return 1
2767     set +o pipefail
2768
2769     pg_deep_scrub $pg || return 1
2770
2771     # Make sure bad object found
2772     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
2773
2774     flush_pg_stats
2775     local last_scrub=$(get_last_scrub_stamp $pg)
2776     # Fake a schedule scrub
2777     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
2778              trigger_scrub $pg || return 1
2779     # Wait for schedule regular scrub
2780     wait_for_scrub $pg "$last_scrub"
2781
2782     # It needed to be upgraded
2783     grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1
2784
2785     # Bad object still known
2786     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
2787
2788     # Can't upgrade with this set
2789     ceph osd set nodeep-scrub
2790     # Let map change propagate to OSDs
2791     flush pg_stats
2792     sleep 5
2793
2794     # Fake a schedule scrub
2795     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
2796              trigger_scrub $pg || return 1
2797     # Wait for schedule regular scrub
2798     # to notice scrub and skip it
2799     local found=false
2800     for i in $(seq 14 -1 0)
2801     do
2802       sleep 1
2803       ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; }
2804       echo Time left: $i seconds
2805     done
2806     test $found = "true" || return 1
2807
2808     # Bad object still known
2809     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
2810
2811     flush_pg_stats
2812     # Request a regular scrub and it will be done
2813     pg_scrub $pg
2814     grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1
2815
2816     # deep-scrub error is no longer present
2817     rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1
2818 }
2819
2820
2821 main osd-scrub-repair "$@"
2822
2823 # Local Variables:
2824 # compile-command: "cd ../.. ; make -j4 && \
2825 #    test/osd/osd-scrub-repair.sh # TEST_corrupt_and_repair_replicated"
2826 # End: