ceph/src/test/osd/osd-scrub-repair.sh

   1 #!/bin/bash -x
   2 #
   3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
   4 #
   5 # Author: Loic Dachary <loic@dachary.org>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17 source $(dirname $0)/../detect-build-env-vars.sh
  18 source $CEPH_ROOT/qa/workunits/ceph-helpers.sh
  19
  20 if [ `uname` = FreeBSD ]; then
  21     # erasure coding overwrites are only tested on Bluestore
  22     # erasure coding on filestore is unsafe
  23     # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites
  24     use_ec_overwrite=false
  25 else
  26     use_ec_overwrite=true
  27 fi
  28
  29 # Test development and debugging
  30 # Set to "yes" in order to ignore diff errors and save results to update test
  31 getjson="no"
  32
  33 # Ignore the epoch and filter out the attr '_' value because it has date information and won't match
  34 jqfilter='.inconsistents | (.[].shards[].attrs[]? | select(.name == "_") | .value) |= "----Stripped-by-test----"'
  35 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
  36
  37 # Remove items are not consistent across runs, the pg interval and client
  38 sedfilter='s/\([ ]*\"\(selected_\)*object_info\":.*head[(]\)[^[:space:]]* [^[:space:]]* \(.*\)/\1\3/'
  39
  40 function run() {
  41     local dir=$1
  42     shift
  43
  44     export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
  45     export CEPH_ARGS
  46     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  47     CEPH_ARGS+="--mon-host=$CEPH_MON "
  48
  49     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  50     for func in $funcs ; do
  51         $func $dir || return 1
  52     done
  53 }
  54
  55 function add_something() {
  56     local dir=$1
  57     local poolname=$2
  58     local obj=${3:-SOMETHING}
  59     local scrub=${4:-noscrub}
  60
  61     if [ "$scrub" = "noscrub" ];
  62     then
  63         ceph osd set noscrub || return 1
  64         ceph osd set nodeep-scrub || return 1
  65     else
  66         ceph osd unset noscrub || return 1
  67         ceph osd unset nodeep-scrub || return 1
  68     fi
  69
  70     local payload=ABCDEF
  71     echo $payload > $dir/ORIGINAL
  72     rados --pool $poolname put $obj $dir/ORIGINAL || return 1
  73 }
  74
  75 #
  76 # Corrupt one copy of a replicated pool
  77 #
  78 function TEST_corrupt_and_repair_replicated() {
  79     local dir=$1
  80     local poolname=rbd
  81
  82     setup $dir || return 1
  83     run_mon $dir a --osd_pool_default_size=2 || return 1
  84     run_mgr $dir x || return 1
  85     run_osd $dir 0 || return 1
  86     run_osd $dir 1 || return 1
  87     wait_for_clean || return 1
  88
  89     add_something $dir $poolname || return 1
  90     corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
  91     # Reproduces http://tracker.ceph.com/issues/8914
  92     corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1
  93
  94     teardown $dir || return 1
  95 }
  96
  97 function corrupt_and_repair_two() {
  98     local dir=$1
  99     local poolname=$2
 100     local first=$3
 101     local second=$4
 102
 103     #
 104     # 1) remove the corresponding file from the OSDs
 105     #
 106     pids=""
 107     run_in_background pids objectstore_tool $dir $first SOMETHING remove
 108     run_in_background pids objectstore_tool $dir $second SOMETHING remove
 109     wait_background pids
 110     return_code=$?
 111     if [ $return_code -ne 0 ]; then return $return_code; fi
 112
 113     #
 114     # 2) repair the PG
 115     #
 116     local pg=$(get_pg $poolname SOMETHING)
 117     repair $pg
 118     #
 119     # 3) The files must be back
 120     #
 121     pids=""
 122     run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs
 123     run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs
 124     wait_background pids
 125     return_code=$?
 126     if [ $return_code -ne 0 ]; then return $return_code; fi
 127
 128     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 129     diff $dir/ORIGINAL $dir/COPY || return 1
 130 }
 131
 132 #
 133 # 1) add an object
 134 # 2) remove the corresponding file from a designated OSD
 135 # 3) repair the PG
 136 # 4) check that the file has been restored in the designated OSD
 137 #
 138 function corrupt_and_repair_one() {
 139     local dir=$1
 140     local poolname=$2
 141     local osd=$3
 142
 143     #
 144     # 1) remove the corresponding file from the OSD
 145     #
 146     objectstore_tool $dir $osd SOMETHING remove || return 1
 147     #
 148     # 2) repair the PG
 149     #
 150     local pg=$(get_pg $poolname SOMETHING)
 151     repair $pg
 152     #
 153     # 3) The file must be back
 154     #
 155     objectstore_tool $dir $osd SOMETHING list-attrs || return 1
 156     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 157     diff $dir/ORIGINAL $dir/COPY || return 1
 158 }
 159
 160 function corrupt_and_repair_erasure_coded() {
 161     local dir=$1
 162     local poolname=$2
 163
 164     add_something $dir $poolname || return 1
 165
 166     local primary=$(get_primary $poolname SOMETHING)
 167     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 168     local not_primary_first=${osds[0]}
 169     local not_primary_second=${osds[1]}
 170
 171     # Reproduces http://tracker.ceph.com/issues/10017
 172     corrupt_and_repair_one $dir $poolname $primary  || return 1
 173     # Reproduces http://tracker.ceph.com/issues/10409
 174     corrupt_and_repair_one $dir $poolname $not_primary_first || return 1
 175     corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1
 176     corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1
 177
 178 }
 179
 180 function create_ec_pool() {
 181     local pool_name=$1
 182     local allow_overwrites=$2
 183
 184     ceph osd erasure-code-profile set myprofile crush-failure-domain=osd $3 $4 $5 $6 $7 || return 1
 185
 186     ceph osd pool create "$poolname" 1 1 erasure myprofile || return 1
 187
 188     if [ "$allow_overwrites" = "true" ]; then
 189         ceph osd pool set "$poolname" allow_ec_overwrites true || return 1
 190     fi
 191
 192     wait_for_clean || return 1
 193     return 0
 194 }
 195
 196 function auto_repair_erasure_coded() {
 197     local dir=$1
 198     local allow_overwrites=$2
 199     local poolname=ecpool
 200
 201     # Launch a cluster with 5 seconds scrub interval
 202     setup $dir || return 1
 203     run_mon $dir a || return 1
 204     run_mgr $dir x || return 1
 205     local ceph_osd_args="--osd-scrub-auto-repair=true \
 206             --osd-deep-scrub-interval=5 \
 207             --osd-scrub-max-interval=5 \
 208             --osd-scrub-min-interval=5 \
 209             --osd-scrub-interval-randomize-ratio=0"
 210     for id in $(seq 0 2) ; do
 211         if [ "$allow_overwrites" = "true" ]; then
 212             run_osd_bluestore $dir $id $ceph_osd_args || return 1
 213         else
 214             run_osd $dir $id $ceph_osd_args || return 1
 215         fi
 216     done
 217     wait_for_clean || return 1
 218
 219     # Create an EC pool
 220     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 221
 222     # Put an object
 223     local payload=ABCDEF
 224     echo $payload > $dir/ORIGINAL
 225     rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
 226
 227     # Remove the object from one shard physically
 228     # Restarted osd get $ceph_osd_args passed
 229     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
 230     # Wait for auto repair
 231     local pgid=$(get_pg $poolname SOMETHING)
 232     wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)"
 233     wait_for_clean || return 1
 234     # Verify - the file should be back
 235     # Restarted osd get $ceph_osd_args passed
 236     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
 237     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 238     diff $dir/ORIGINAL $dir/COPY || return 1
 239
 240     # Tear down
 241     teardown $dir || return 1
 242 }
 243
 244 function TEST_auto_repair_erasure_coded_appends() {
 245     auto_repair_erasure_coded $1 false
 246 }
 247
 248 function TEST_auto_repair_erasure_coded_overwrites() {
 249     if [ "$use_ec_overwrite" = "true" ]; then
 250         auto_repair_erasure_coded $1 true
 251     fi
 252 }
 253
 254 function corrupt_and_repair_jerasure() {
 255     local dir=$1
 256     local allow_overwrites=$2
 257     local poolname=ecpool
 258
 259     setup $dir || return 1
 260     run_mon $dir a || return 1
 261     run_mgr $dir x || return 1
 262     for id in $(seq 0 3) ; do
 263         if [ "$allow_overwrites" = "true" ]; then
 264             run_osd_bluestore $dir $id || return 1
 265         else
 266             run_osd $dir $id || return 1
 267         fi
 268     done
 269     wait_for_clean || return 1
 270
 271     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 272     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 273
 274     teardown $dir || return 1
 275 }
 276
 277 function TEST_corrupt_and_repair_jerasure_appends() {
 278     corrupt_and_repair_jerasure $1
 279 }
 280
 281 function TEST_corrupt_and_repair_jerasure_overwrites() {
 282     if [ "$use_ec_overwrite" = "true" ]; then
 283         corrupt_and_repair_jerasure $1 true
 284     fi
 285 }
 286
 287 function corrupt_and_repair_lrc() {
 288     local dir=$1
 289     local allow_overwrites=$2
 290     local poolname=ecpool
 291
 292     setup $dir || return 1
 293     run_mon $dir a || return 1
 294     run_mgr $dir x || return 1
 295     for id in $(seq 0 9) ; do
 296         if [ "$allow_overwrites" = "true" ]; then
 297             run_osd_bluestore $dir $id || return 1
 298         else
 299             run_osd $dir $id || return 1
 300         fi
 301     done
 302     wait_for_clean || return 1
 303
 304     create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1
 305     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 306
 307     teardown $dir || return 1
 308 }
 309
 310 function TEST_corrupt_and_repair_lrc_appends() {
 311     corrupt_and_repair_jerasure $1
 312 }
 313
 314 function TEST_corrupt_and_repair_lrc_overwrites() {
 315     if [ "$use_ec_overwrite" = "true" ]; then
 316         corrupt_and_repair_jerasure $1 true
 317     fi
 318 }
 319
 320 function unfound_erasure_coded() {
 321     local dir=$1
 322     local allow_overwrites=$2
 323     local poolname=ecpool
 324     local payload=ABCDEF
 325
 326     setup $dir || return 1
 327     run_mon $dir a || return 1
 328     run_mgr $dir x || return 1
 329     for id in $(seq 0 3) ; do
 330         if [ "$allow_overwrites" = "true" ]; then
 331             run_osd_bluestore $dir $id || return 1
 332         else
 333             run_osd $dir $id || return 1
 334         fi
 335     done
 336     wait_for_clean || return 1
 337
 338     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 339
 340     add_something $dir $poolname || return 1
 341
 342     local primary=$(get_primary $poolname SOMETHING)
 343     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 344     local not_primary_first=${osds[0]}
 345     local not_primary_second=${osds[1]}
 346     local not_primary_third=${osds[2]}
 347
 348     #
 349     # 1) remove the corresponding file from the OSDs
 350     #
 351     pids=""
 352     run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove
 353     run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove
 354     run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove
 355     wait_background pids
 356     return_code=$?
 357     if [ $return_code -ne 0 ]; then return $return_code; fi
 358
 359     #
 360     # 2) repair the PG
 361     #
 362     local pg=$(get_pg $poolname SOMETHING)
 363     repair $pg
 364     #
 365     # 3) check pg state
 366     #
 367     # it may take a bit to appear due to mon/mgr asynchrony
 368     for f in `seq 1 60`; do
 369         ceph -s | grep "1/1 unfound" && break
 370         sleep 1
 371     done
 372     ceph -s|grep "4 osds: 4 up, 4 in" || return 1
 373     ceph -s|grep "1/1 unfound" || return 1
 374
 375     teardown $dir || return 1
 376 }
 377
 378 function TEST_unfound_erasure_coded_appends() {
 379     unfound_erasure_coded $1
 380 }
 381
 382 function TEST_unfound_erasure_coded_overwrites() {
 383     if [ "$use_ec_overwrite" = "true" ]; then
 384         unfound_erasure_coded $1 true
 385     fi
 386 }
 387
 388 #
 389 # list_missing for EC pool
 390 #
 391 function list_missing_erasure_coded() {
 392     local dir=$1
 393     local allow_overwrites=$2
 394     local poolname=ecpool
 395
 396     setup $dir || return 1
 397     run_mon $dir a || return 1
 398     run_mgr $dir x || return 1
 399     for id in $(seq 0 2) ; do
 400         if [ "$allow_overwrites" = "true" ]; then
 401             run_osd_bluestore $dir $id || return 1
 402         else
 403             run_osd $dir $id || return 1
 404         fi
 405     done
 406     wait_for_clean || return 1
 407
 408     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 409
 410     # Put an object and remove the two shards (including primary)
 411     add_something $dir $poolname MOBJ0 || return 1
 412     local -a osds0=($(get_osds $poolname MOBJ0))
 413
 414     # Put another object and remove two shards (excluding primary)
 415     add_something $dir $poolname MOBJ1 || return 1
 416     local -a osds1=($(get_osds $poolname MOBJ1))
 417
 418     # Stop all osd daemons
 419     for id in $(seq 0 2) ; do
 420         kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1
 421     done
 422
 423     id=${osds0[0]}
 424     ceph-objectstore-tool --data-path $dir/$id \
 425         MOBJ0 remove || return 1
 426     id=${osds0[1]}
 427     ceph-objectstore-tool --data-path $dir/$id \
 428         MOBJ0 remove || return 1
 429
 430     id=${osds1[1]}
 431     ceph-objectstore-tool --data-path $dir/$id \
 432         MOBJ1 remove || return 1
 433     id=${osds1[2]}
 434     ceph-objectstore-tool --data-path $dir/$id \
 435         MOBJ1 remove || return 1
 436
 437     for id in $(seq 0 2) ; do
 438         activate_osd $dir $id >&2 || return 1
 439     done
 440     wait_for_clean || return 1
 441
 442     # Get get - both objects should in the same PG
 443     local pg=$(get_pg $poolname MOBJ0)
 444
 445     # Repair the PG, which triggers the recovering,
 446     # and should mark the object as unfound
 447     repair $pg
 448
 449     for i in $(seq 0 120) ; do
 450         [ $i -lt 60 ] || return 1
 451         matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l)
 452         [ $matches -eq 2 ] && break
 453     done
 454
 455     teardown $dir || return 1
 456 }
 457
 458 function TEST_list_missing_erasure_coded_appends() {
 459     list_missing_erasure_coded $1 false
 460 }
 461
 462 function TEST_list_missing_erasure_coded_overwrites() {
 463     if [ "$use_ec_overwrite" = "true" ]; then
 464         list_missing_erasure_coded $1 true
 465     fi
 466 }
 467
 468 #
 469 # Corrupt one copy of a replicated pool
 470 #
 471 function TEST_corrupt_scrub_replicated() {
 472     local dir=$1
 473     local poolname=csr_pool
 474     local total_objs=15
 475
 476     setup $dir || return 1
 477     run_mon $dir a --osd_pool_default_size=2 || return 1
 478     run_mgr $dir x || return 1
 479     run_osd $dir 0 || return 1
 480     run_osd $dir 1 || return 1
 481     wait_for_clean || return 1
 482
 483     ceph osd pool create $poolname 1 1 || return 1
 484     wait_for_clean || return 1
 485
 486     for i in $(seq 1 $total_objs) ; do
 487         objname=ROBJ${i}
 488         add_something $dir $poolname $objname || return 1
 489
 490         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
 491         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
 492     done
 493
 494     local pg=$(get_pg $poolname ROBJ0)
 495
 496     # Compute an old omap digest and save oi
 497     CEPH_ARGS='' ceph daemon $dir//ceph-osd.0.asok \
 498         config set osd_deep_scrub_update_digest_min_age 0
 499     CEPH_ARGS='' ceph daemon $dir//ceph-osd.1.asok \
 500         config set osd_deep_scrub_update_digest_min_age 0
 501     pg_deep_scrub $pg
 502
 503     for i in $(seq 1 $total_objs) ; do
 504         objname=ROBJ${i}
 505
 506         # Alternate corruption between osd.0 and osd.1
 507         local osd=$(expr $i % 2)
 508
 509         case $i in
 510         1)
 511             # Size (deep scrub data_digest too)
 512             local payload=UVWXYZZZ
 513             echo $payload > $dir/CORRUPT
 514             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 515             ;;
 516
 517         2)
 518             # digest (deep scrub only)
 519             local payload=UVWXYZ
 520             echo $payload > $dir/CORRUPT
 521             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 522             ;;
 523
 524         3)
 525              # missing
 526              objectstore_tool $dir $osd $objname remove || return 1
 527              ;;
 528
 529          4)
 530              # Modify omap value (deep scrub only)
 531              objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1
 532              ;;
 533
 534          5)
 535             # Delete omap key (deep scrub only)
 536             objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1
 537             ;;
 538
 539          6)
 540             # Add extra omap key (deep scrub only)
 541             echo extra > $dir/extra-val
 542             objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1
 543             rm $dir/extra-val
 544             ;;
 545
 546          7)
 547             # Modify omap header (deep scrub only)
 548             echo -n newheader > $dir/hdr
 549             objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1
 550             rm $dir/hdr
 551             ;;
 552
 553          8)
 554             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
 555             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
 556
 557             # Break xattrs
 558             echo -n bad-val > $dir/bad-val
 559             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
 560             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
 561             echo -n val3-$objname > $dir/newval
 562             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
 563             rm $dir/bad-val $dir/newval
 564             ;;
 565
 566         9)
 567             objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi
 568             echo -n D > $dir/change
 569             rados --pool $poolname put $objname $dir/change
 570             objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi
 571             rm $dir/oi $dir/change
 572             ;;
 573
 574           # ROBJ10 must be handled after digests are re-computed by a deep scrub below
 575           # ROBJ11 must be handled with config change before deep scrub
 576           # ROBJ12 must be handled with config change before scrubs
 577           # ROBJ13 must be handled before scrubs
 578
 579         14)
 580             echo -n bad-val > $dir/bad-val
 581             objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1
 582             objectstore_tool $dir 1 $objname rm-attr _ || return 1
 583             rm $dir/bad-val
 584             ;;
 585
 586         15)
 587             objectstore_tool $dir $osd $objname rm-attr _ || return 1
 588
 589         esac
 590     done
 591
 592     local pg=$(get_pg $poolname ROBJ0)
 593
 594     set_config osd 0 filestore_debug_inject_read_err true || return 1
 595     set_config osd 1 filestore_debug_inject_read_err true || return 1
 596     CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \
 597              injectdataerr $poolname ROBJ11 || return 1
 598     CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \
 599              injectmdataerr $poolname ROBJ12 || return 1
 600     CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \
 601              injectmdataerr $poolname ROBJ13 || return 1
 602     CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \
 603              injectdataerr $poolname ROBJ13 || return 1
 604
 605     pg_scrub $pg
 606
 607     rados list-inconsistent-pg $poolname > $dir/json || return 1
 608     # Check pg count
 609     test $(jq '. | length' $dir/json) = "1" || return 1
 610     # Check pgid
 611     test $(jq -r '.[0]' $dir/json) = $pg || return 1
 612
 613     rados list-inconsistent-obj $pg > $dir/json || return 1
 614     # Get epoch for repair-get requests
 615     epoch=$(jq .epoch $dir/json)
 616
 617     jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
 618 {
 619   "inconsistents": [
 620     {
 621       "shards": [
 622         {
 623           "size": 7,
 624           "errors": [],
 625           "osd": 0
 626         },
 627         {
 628           "size": 9,
 629           "errors": [
 630             "size_mismatch_oi"
 631           ],
 632           "osd": 1
 633         }
 634       ],
 635       "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])",
 636       "union_shard_errors": [
 637         "size_mismatch_oi"
 638       ],
 639       "errors": [
 640         "size_mismatch"
 641       ],
 642       "object": {
 643         "version": 3,
 644         "snap": "head",
 645         "locator": "",
 646         "nspace": "",
 647         "name": "ROBJ1"
 648       }
 649     },
 650     {
 651       "shards": [
 652         {
 653           "errors": [
 654             "stat_error"
 655           ],
 656           "osd": 0
 657         },
 658         {
 659           "size": 7,
 660           "errors": [],
 661           "osd": 1
 662         }
 663       ],
 664       "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])",
 665       "union_shard_errors": [
 666         "stat_error"
 667       ],
 668       "errors": [],
 669       "object": {
 670         "version": 36,
 671         "snap": "head",
 672         "locator": "",
 673         "nspace": "",
 674         "name": "ROBJ12"
 675       }
 676     },
 677     {
 678       "shards": [
 679         {
 680           "errors": [
 681             "stat_error"
 682           ],
 683           "osd": 0
 684         },
 685         {
 686           "size": 7,
 687           "errors": [],
 688           "osd": 1
 689         }
 690       ],
 691       "selected_object_info": "3:d60617f9:::ROBJ13:head(47'55 osd.0.0:54 dirty|omap|data_digest|omap_digest s 7 uv 39 dd 2ddbf8f5 od 6441854d alloc_hint [0 0 0])",
 692       "union_shard_errors": [
 693         "stat_error"
 694       ],
 695       "errors": [],
 696       "object": {
 697         "version": 39,
 698         "snap": "head",
 699         "locator": "",
 700         "nspace": "",
 701         "name": "ROBJ13"
 702       }
 703     },
 704     {
 705       "shards": [
 706         {
 707           "size": 7,
 708           "errors": [
 709             "oi_attr_corrupted"
 710           ],
 711           "osd": 0
 712         },
 713         {
 714           "size": 7,
 715           "errors": [
 716             "oi_attr_missing"
 717           ],
 718           "osd": 1
 719         }
 720       ],
 721       "union_shard_errors": [
 722         "oi_attr_missing",
 723         "oi_attr_corrupted"
 724       ],
 725       "errors": [],
 726       "object": {
 727         "version": 0,
 728         "snap": "head",
 729         "locator": "",
 730         "nspace": "",
 731         "name": "ROBJ14"
 732       }
 733     },
 734     {
 735       "shards": [
 736         {
 737           "attrs": [
 738             {
 739               "Base64": true,
 740               "value": "",
 741               "name": "_"
 742             },
 743             {
 744               "Base64": true,
 745               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 746               "name": "snapset"
 747             }
 748           ],
 749           "size": 7,
 750           "errors": [],
 751           "osd": 0
 752         },
 753         {
 754           "attrs": [
 755             {
 756               "Base64": true,
 757               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 758               "name": "snapset"
 759             }
 760           ],
 761           "size": 7,
 762           "errors": [
 763             "oi_attr_missing"
 764           ],
 765           "osd": 1
 766         }
 767       ],
 768       "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])",
 769       "union_shard_errors": [
 770         "oi_attr_missing"
 771       ],
 772       "errors": [
 773         "attr_name_mismatch"
 774       ],
 775       "object": {
 776         "version": 45,
 777         "snap": "head",
 778         "locator": "",
 779         "nspace": "",
 780         "name": "ROBJ15"
 781       }
 782     },
 783     {
 784       "shards": [
 785         {
 786           "size": 7,
 787           "errors": [],
 788           "osd": 0
 789         },
 790         {
 791           "errors": [
 792             "missing"
 793           ],
 794           "osd": 1
 795         }
 796       ],
 797       "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])",
 798       "union_shard_errors": [
 799         "missing"
 800       ],
 801       "errors": [],
 802       "object": {
 803         "version": 9,
 804         "snap": "head",
 805         "locator": "",
 806         "nspace": "",
 807         "name": "ROBJ3"
 808       }
 809     },
 810     {
 811       "shards": [
 812         {
 813           "attrs": [
 814             {
 815               "Base64": true,
 816               "value": "",
 817               "name": "_"
 818             },
 819             {
 820               "Base64": false,
 821               "value": "bad-val",
 822               "name": "_key1-ROBJ8"
 823             },
 824             {
 825               "Base64": false,
 826               "value": "val3-ROBJ8",
 827               "name": "_key3-ROBJ8"
 828             },
 829             {
 830               "Base64": true,
 831               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 832               "name": "snapset"
 833             }
 834           ],
 835           "size": 7,
 836           "errors": [],
 837           "osd": 0
 838         },
 839         {
 840           "attrs": [
 841             {
 842               "Base64": true,
 843               "value": "",
 844               "name": "_"
 845             },
 846             {
 847               "Base64": false,
 848               "value": "val1-ROBJ8",
 849               "name": "_key1-ROBJ8"
 850             },
 851             {
 852               "Base64": false,
 853               "value": "val2-ROBJ8",
 854               "name": "_key2-ROBJ8"
 855             },
 856             {
 857               "Base64": true,
 858               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 859               "name": "snapset"
 860             }
 861           ],
 862           "size": 7,
 863           "errors": [],
 864           "osd": 1
 865         }
 866       ],
 867       "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 62 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
 868       "union_shard_errors": [],
 869       "errors": [
 870         "attr_value_mismatch",
 871         "attr_name_mismatch"
 872       ],
 873       "object": {
 874         "version": 62,
 875         "snap": "head",
 876         "locator": "",
 877         "nspace": "",
 878         "name": "ROBJ8"
 879       }
 880     },
 881     {
 882       "shards": [
 883         {
 884           "attrs": [
 885             {
 886               "Base64": true,
 887               "value": "",
 888               "name": "_"
 889             },
 890             {
 891               "Base64": true,
 892               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 893               "name": "snapset"
 894             }
 895           ],
 896           "object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 63 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
 897           "size": 1,
 898           "errors": [],
 899           "osd": 0
 900         },
 901         {
 902           "attrs": [
 903             {
 904               "Base64": true,
 905               "value": "",
 906               "name": "_"
 907             },
 908             {
 909               "Base64": true,
 910               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
 911               "name": "snapset"
 912             }
 913           ],
 914           "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])",
 915           "size": 1,
 916           "errors": [],
 917           "osd": 1
 918         }
 919       ],
 920       "selected_object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 63 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
 921       "union_shard_errors": [],
 922       "errors": [
 923         "object_info_inconsistency",
 924         "attr_value_mismatch"
 925       ],
 926       "object": {
 927         "version": 63,
 928         "snap": "head",
 929         "locator": "",
 930         "nspace": "",
 931         "name": "ROBJ9"
 932       }
 933     }
 934   ],
 935   "epoch": 0
 936 }
 937 EOF
 938
 939     jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
 940     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
 941     if test $getjson = "yes"
 942     then
 943         jq '.' $dir/json > save1.json
 944     fi
 945
 946     if which jsonschema > /dev/null;
 947     then
 948       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
 949     fi
 950
 951     objname=ROBJ9
 952     # Change data and size again because digest was recomputed
 953     echo -n ZZZ > $dir/change
 954     rados --pool $poolname put $objname $dir/change
 955     # Set one to an even older value
 956     objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi
 957     rm $dir/oi $dir/change
 958
 959     objname=ROBJ10
 960     objectstore_tool $dir 1 $objname get-attr _ > $dir/oi
 961     rados --pool $poolname setomapval $objname key2-$objname val2-$objname
 962     objectstore_tool $dir 0 $objname set-attr _ $dir/oi
 963     objectstore_tool $dir 1 $objname set-attr _ $dir/oi
 964     rm $dir/oi
 965
 966     set_config osd 0 filestore_debug_inject_read_err true || return 1
 967     set_config osd 1 filestore_debug_inject_read_err true || return 1
 968     CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \
 969              injectdataerr $poolname ROBJ11 || return 1
 970     CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \
 971              injectmdataerr $poolname ROBJ12 || return 1
 972     CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \
 973              injectmdataerr $poolname ROBJ13 || return 1
 974     CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \
 975              injectdataerr $poolname ROBJ13 || return 1
 976     pg_deep_scrub $pg
 977
 978     rados list-inconsistent-pg $poolname > $dir/json || return 1
 979     # Check pg count
 980     test $(jq '. | length' $dir/json) = "1" || return 1
 981     # Check pgid
 982     test $(jq -r '.[0]' $dir/json) = $pg || return 1
 983
 984     rados list-inconsistent-obj $pg > $dir/json || return 1
 985     # Get epoch for repair-get requests
 986     epoch=$(jq .epoch $dir/json)
 987
 988     jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
 989 {
 990   "inconsistents": [
 991     {
 992       "shards": [
 993         {
 994           "data_digest": "0x2ddbf8f5",
 995           "omap_digest": "0xf5fba2c6",
 996           "size": 7,
 997           "errors": [],
 998           "osd": 0
 999         },
1000         {
1001           "data_digest": "0x2d4a11c2",
1002           "omap_digest": "0xf5fba2c6",
1003           "size": 9,
1004           "errors": [
1005             "data_digest_mismatch_oi",
1006             "size_mismatch_oi"
1007           ],
1008           "osd": 1
1009         }
1010       ],
1011       "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])",
1012       "union_shard_errors": [
1013         "data_digest_mismatch_oi",
1014         "size_mismatch_oi"
1015       ],
1016       "errors": [
1017         "data_digest_mismatch",
1018         "size_mismatch"
1019       ],
1020       "object": {
1021         "version": 3,
1022         "snap": "head",
1023         "locator": "",
1024         "nspace": "",
1025         "name": "ROBJ1"
1026       }
1027     },
1028     {
1029       "shards": [
1030         {
1031           "data_digest": "0x2ddbf8f5",
1032           "omap_digest": "0xa8dd5adc",
1033           "size": 7,
1034           "errors": [
1035             "omap_digest_mismatch_oi"
1036           ],
1037           "osd": 0
1038         },
1039         {
1040           "data_digest": "0x2ddbf8f5",
1041           "omap_digest": "0xa8dd5adc",
1042           "size": 7,
1043           "errors": [
1044             "omap_digest_mismatch_oi"
1045           ],
1046           "osd": 1
1047         }
1048       ],
1049       "selected_object_info": "3:b1f19cbd:::ROBJ10:head(47'51 osd.0.0:50 dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [0 0 0])",
1050       "union_shard_errors": [
1051         "omap_digest_mismatch_oi"
1052       ],
1053       "errors": [],
1054       "object": {
1055         "version": 30,
1056         "snap": "head",
1057         "locator": "",
1058         "nspace": "",
1059         "name": "ROBJ10"
1060       }
1061     },
1062     {
1063       "shards": [
1064         {
1065           "data_digest": "0x2ddbf8f5",
1066           "omap_digest": "0xa03cef03",
1067           "size": 7,
1068           "errors": [],
1069           "osd": 0
1070         },
1071         {
1072           "size": 7,
1073           "errors": [
1074             "read_error"
1075           ],
1076           "osd": 1
1077         }
1078       ],
1079       "selected_object_info": "3:87abbf36:::ROBJ11:head(47'48 osd.0.0:47 dirty|omap|data_digest|omap_digest s 7 uv 33 dd 2ddbf8f5 od a03cef03 alloc_hint [0 0 0])",
1080       "union_shard_errors": [
1081         "read_error"
1082       ],
1083       "errors": [],
1084       "object": {
1085         "version": 33,
1086         "snap": "head",
1087         "locator": "",
1088         "nspace": "",
1089         "name": "ROBJ11"
1090       }
1091     },
1092     {
1093       "shards": [
1094         {
1095           "errors": [
1096             "stat_error"
1097           ],
1098           "osd": 0
1099         },
1100         {
1101           "data_digest": "0x2ddbf8f5",
1102           "omap_digest": "0x067f306a",
1103           "size": 7,
1104           "errors": [],
1105           "osd": 1
1106         }
1107       ],
1108       "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])",
1109       "union_shard_errors": [
1110         "stat_error"
1111       ],
1112       "errors": [],
1113       "object": {
1114         "version": 36,
1115         "snap": "head",
1116         "locator": "",
1117         "nspace": "",
1118         "name": "ROBJ12"
1119       }
1120     },
1121     {
1122       "shards": [
1123         {
1124           "errors": [
1125             "stat_error"
1126           ],
1127           "osd": 0
1128         },
1129         {
1130           "size": 7,
1131           "errors": [
1132             "read_error"
1133           ],
1134           "osd": 1
1135         }
1136       ],
1137       "union_shard_errors": [
1138         "stat_error",
1139         "read_error"
1140       ],
1141       "errors": [],
1142       "object": {
1143         "version": 0,
1144         "snap": "head",
1145         "locator": "",
1146         "nspace": "",
1147         "name": "ROBJ13"
1148       }
1149     },
1150     {
1151       "shards": [
1152         {
1153           "data_digest": "0x2ddbf8f5",
1154           "omap_digest": "0x4f14f849",
1155           "size": 7,
1156           "errors": [
1157             "oi_attr_corrupted"
1158           ],
1159           "osd": 0
1160         },
1161         {
1162           "data_digest": "0x2ddbf8f5",
1163           "omap_digest": "0x4f14f849",
1164           "size": 7,
1165           "errors": [
1166             "oi_attr_missing"
1167           ],
1168           "osd": 1
1169         }
1170       ],
1171       "union_shard_errors": [
1172         "oi_attr_missing",
1173         "oi_attr_corrupted"
1174       ],
1175       "errors": [],
1176       "object": {
1177         "version": 0,
1178         "snap": "head",
1179         "locator": "",
1180         "nspace": "",
1181         "name": "ROBJ14"
1182       }
1183     },
1184     {
1185       "shards": [
1186         {
1187           "attrs": [
1188             {
1189               "Base64": true,
1190               "value": "",
1191               "name": "_"
1192             },
1193             {
1194               "Base64": true,
1195               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1196               "name": "snapset"
1197             }
1198           ],
1199           "data_digest": "0x2ddbf8f5",
1200           "omap_digest": "0x2d2a4d6e",
1201           "size": 7,
1202           "errors": [],
1203           "osd": 0
1204         },
1205         {
1206           "attrs": [
1207             {
1208               "Base64": true,
1209               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1210               "name": "snapset"
1211             }
1212           ],
1213           "data_digest": "0x2ddbf8f5",
1214           "omap_digest": "0x2d2a4d6e",
1215           "size": 7,
1216           "errors": [
1217             "oi_attr_missing"
1218           ],
1219           "osd": 1
1220         }
1221       ],
1222       "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])",
1223       "union_shard_errors": [
1224         "oi_attr_missing"
1225       ],
1226       "errors": [
1227         "attr_name_mismatch"
1228       ],
1229       "object": {
1230         "version": 45,
1231         "snap": "head",
1232         "locator": "",
1233         "nspace": "",
1234         "name": "ROBJ15"
1235       }
1236     },
1237     {
1238       "shards": [
1239         {
1240           "data_digest": "0x578a4830",
1241           "omap_digest": "0xf8e11918",
1242           "size": 7,
1243           "errors": [
1244             "data_digest_mismatch_oi"
1245           ],
1246           "osd": 0
1247         },
1248         {
1249           "data_digest": "0x2ddbf8f5",
1250           "omap_digest": "0xf8e11918",
1251           "size": 7,
1252           "errors": [],
1253           "osd": 1
1254         }
1255       ],
1256       "selected_object_info": "3:e97ce31e:::ROBJ2:head(47'56 osd.0.0:55 dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [0 0 0])",
1257       "union_shard_errors": [
1258         "data_digest_mismatch_oi"
1259       ],
1260       "errors": [
1261         "data_digest_mismatch"
1262       ],
1263       "object": {
1264         "version": 6,
1265         "snap": "head",
1266         "locator": "",
1267         "nspace": "",
1268         "name": "ROBJ2"
1269       }
1270     },
1271     {
1272       "shards": [
1273         {
1274           "data_digest": "0x2ddbf8f5",
1275           "omap_digest": "0x00b35dfd",
1276           "size": 7,
1277           "errors": [],
1278           "osd": 0
1279         },
1280         {
1281           "errors": [
1282             "missing"
1283           ],
1284           "osd": 1
1285         }
1286       ],
1287       "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])",
1288       "union_shard_errors": [
1289         "missing"
1290       ],
1291       "errors": [],
1292       "object": {
1293         "version": 9,
1294         "snap": "head",
1295         "locator": "",
1296         "nspace": "",
1297         "name": "ROBJ3"
1298       }
1299     },
1300     {
1301       "shards": [
1302         {
1303           "data_digest": "0x2ddbf8f5",
1304           "omap_digest": "0xd7178dfe",
1305           "size": 7,
1306           "errors": [
1307             "omap_digest_mismatch_oi"
1308           ],
1309           "osd": 0
1310         },
1311         {
1312           "data_digest": "0x2ddbf8f5",
1313           "omap_digest": "0xe2d46ea4",
1314           "size": 7,
1315           "errors": [],
1316           "osd": 1
1317         }
1318       ],
1319       "selected_object_info": "3:f4981d31:::ROBJ4:head(47'58 osd.0.0:57 dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [0 0 0])",
1320       "union_shard_errors": [
1321         "omap_digest_mismatch_oi"
1322       ],
1323       "errors": [
1324         "omap_digest_mismatch"
1325       ],
1326       "object": {
1327         "version": 12,
1328         "snap": "head",
1329         "locator": "",
1330         "nspace": "",
1331         "name": "ROBJ4"
1332       }
1333     },
1334     {
1335       "shards": [
1336         {
1337           "data_digest": "0x2ddbf8f5",
1338           "omap_digest": "0x1a862a41",
1339           "size": 7,
1340           "errors": [],
1341           "osd": 0
1342         },
1343         {
1344           "data_digest": "0x2ddbf8f5",
1345           "omap_digest": "0x06cac8f6",
1346           "size": 7,
1347           "errors": [
1348             "omap_digest_mismatch_oi"
1349           ],
1350           "osd": 1
1351         }
1352       ],
1353       "selected_object_info": "3:f4bfd4d1:::ROBJ5:head(47'59 osd.0.0:58 dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [0 0 0])",
1354       "union_shard_errors": [
1355         "omap_digest_mismatch_oi"
1356       ],
1357       "errors": [
1358         "omap_digest_mismatch"
1359       ],
1360       "object": {
1361         "version": 15,
1362         "snap": "head",
1363         "locator": "",
1364         "nspace": "",
1365         "name": "ROBJ5"
1366       }
1367     },
1368     {
1369       "shards": [
1370         {
1371           "data_digest": "0x2ddbf8f5",
1372           "omap_digest": "0x689ee887",
1373           "size": 7,
1374           "errors": [
1375             "omap_digest_mismatch_oi"
1376           ],
1377           "osd": 0
1378         },
1379         {
1380           "data_digest": "0x2ddbf8f5",
1381           "omap_digest": "0x179c919f",
1382           "size": 7,
1383           "errors": [],
1384           "osd": 1
1385         }
1386       ],
1387       "selected_object_info": "3:a53c12e8:::ROBJ6:head(47'50 osd.0.0:49 dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [0 0 0])",
1388       "union_shard_errors": [
1389         "omap_digest_mismatch_oi"
1390       ],
1391       "errors": [
1392         "omap_digest_mismatch"
1393       ],
1394       "object": {
1395         "version": 18,
1396         "snap": "head",
1397         "locator": "",
1398         "nspace": "",
1399         "name": "ROBJ6"
1400       }
1401     },
1402     {
1403       "shards": [
1404         {
1405           "data_digest": "0x2ddbf8f5",
1406           "omap_digest": "0xefced57a",
1407           "size": 7,
1408           "errors": [],
1409           "osd": 0
1410         },
1411         {
1412           "data_digest": "0x2ddbf8f5",
1413           "omap_digest": "0x6a73cc07",
1414           "size": 7,
1415           "errors": [
1416             "omap_digest_mismatch_oi"
1417           ],
1418           "osd": 1
1419         }
1420       ],
1421       "selected_object_info": "3:8b55fa4b:::ROBJ7:head(47'49 osd.0.0:48 dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [0 0 0])",
1422       "union_shard_errors": [
1423         "omap_digest_mismatch_oi"
1424       ],
1425       "errors": [
1426         "omap_digest_mismatch"
1427       ],
1428       "object": {
1429         "version": 21,
1430         "snap": "head",
1431         "locator": "",
1432         "nspace": "",
1433         "name": "ROBJ7"
1434       }
1435     },
1436     {
1437       "shards": [
1438         {
1439           "attrs": [
1440             {
1441               "Base64": true,
1442               "value": "",
1443               "name": "_"
1444             },
1445             {
1446               "Base64": false,
1447               "value": "bad-val",
1448               "name": "_key1-ROBJ8"
1449             },
1450             {
1451               "Base64": false,
1452               "value": "val3-ROBJ8",
1453               "name": "_key3-ROBJ8"
1454             },
1455             {
1456               "Base64": true,
1457               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1458               "name": "snapset"
1459             }
1460           ],
1461           "data_digest": "0x2ddbf8f5",
1462           "omap_digest": "0xd6be81dc",
1463           "size": 7,
1464           "errors": [],
1465           "osd": 0
1466         },
1467         {
1468           "attrs": [
1469             {
1470               "Base64": true,
1471               "value": "",
1472               "name": "_"
1473             },
1474             {
1475               "Base64": false,
1476               "value": "val1-ROBJ8",
1477               "name": "_key1-ROBJ8"
1478             },
1479             {
1480               "Base64": false,
1481               "value": "val2-ROBJ8",
1482               "name": "_key2-ROBJ8"
1483             },
1484             {
1485               "Base64": true,
1486               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1487               "name": "snapset"
1488             }
1489           ],
1490           "data_digest": "0x2ddbf8f5",
1491           "omap_digest": "0xd6be81dc",
1492           "size": 7,
1493           "errors": [],
1494           "osd": 1
1495         }
1496       ],
1497       "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 62 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
1498       "union_shard_errors": [],
1499       "errors": [
1500         "attr_value_mismatch",
1501         "attr_name_mismatch"
1502       ],
1503       "object": {
1504         "version": 62,
1505         "snap": "head",
1506         "locator": "",
1507         "nspace": "",
1508         "name": "ROBJ8"
1509       }
1510     },
1511     {
1512       "shards": [
1513         {
1514           "attrs": [
1515             {
1516               "Base64": true,
1517               "value": "",
1518               "name": "_"
1519             },
1520             {
1521               "Base64": true,
1522               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1523               "name": "snapset"
1524             }
1525           ],
1526           "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])",
1527           "data_digest": "0x1f26fb26",
1528           "omap_digest": "0x2eecc539",
1529           "size": 3,
1530           "errors": [],
1531           "osd": 0
1532         },
1533         {
1534           "attrs": [
1535             {
1536               "Base64": true,
1537               "value": "",
1538               "name": "_"
1539             },
1540             {
1541               "Base64": true,
1542               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1543               "name": "snapset"
1544             }
1545           ],
1546           "object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 64 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
1547           "data_digest": "0x1f26fb26",
1548           "omap_digest": "0x2eecc539",
1549           "size": 3,
1550           "errors": [],
1551           "osd": 1
1552         }
1553       ],
1554       "selected_object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 64 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
1555       "union_shard_errors": [],
1556       "errors": [
1557         "object_info_inconsistency",
1558         "attr_value_mismatch"
1559       ],
1560       "object": {
1561         "version": 64,
1562         "snap": "head",
1563         "locator": "",
1564         "nspace": "",
1565         "name": "ROBJ9"
1566       }
1567     }
1568   ],
1569   "epoch": 0
1570 }
1571 EOF
1572
1573     jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
1574     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
1575     if test $getjson = "yes"
1576     then
1577         jq '.' $dir/json > save2.json
1578     fi
1579
1580     if which jsonschema > /dev/null;
1581     then
1582       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
1583     fi
1584
1585     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
1586     teardown $dir || return 1
1587 }
1588
1589
1590 #
1591 # Test scrub errors for an erasure coded pool
1592 #
1593 function corrupt_scrub_erasure() {
1594     local dir=$1
1595     local allow_overwrites=$2
1596     local poolname=ecpool
1597     local total_objs=5
1598
1599     setup $dir || return 1
1600     run_mon $dir a || return 1
1601     run_mgr $dir x || return 1
1602     for id in $(seq 0 2) ; do
1603         if [ "$allow_overwrites" = "true" ]; then
1604             run_osd_bluestore $dir $id || return 1
1605         else
1606             run_osd $dir $id || return 1
1607         fi
1608     done
1609     wait_for_clean || return 1
1610
1611     create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1
1612
1613     for i in $(seq 1 $total_objs) ; do
1614         objname=EOBJ${i}
1615         add_something $dir $poolname $objname || return 1
1616
1617         local osd=$(expr $i % 2)
1618
1619         case $i in
1620         1)
1621             # Size (deep scrub data_digest too)
1622             local payload=UVWXYZZZ
1623             echo $payload > $dir/CORRUPT
1624             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1625             ;;
1626
1627         2)
1628             # Corrupt EC shard
1629             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1
1630             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1631             ;;
1632
1633         3)
1634              # missing
1635              objectstore_tool $dir $osd $objname remove || return 1
1636              ;;
1637
1638         4)
1639             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
1640             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
1641
1642             # Break xattrs
1643             echo -n bad-val > $dir/bad-val
1644             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
1645             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
1646             echo -n val3-$objname > $dir/newval
1647             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
1648             rm $dir/bad-val $dir/newval
1649             ;;
1650
1651         5)
1652             # Corrupt EC shard
1653             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2
1654             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1655             ;;
1656
1657         esac
1658     done
1659
1660     local pg=$(get_pg $poolname EOBJ0)
1661
1662     pg_scrub $pg
1663
1664     rados list-inconsistent-pg $poolname > $dir/json || return 1
1665     # Check pg count
1666     test $(jq '. | length' $dir/json) = "1" || return 1
1667     # Check pgid
1668     test $(jq -r '.[0]' $dir/json) = $pg || return 1
1669
1670     rados list-inconsistent-obj $pg > $dir/json || return 1
1671     # Get epoch for repair-get requests
1672     epoch=$(jq .epoch $dir/json)
1673
1674     jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
1675 {
1676   "inconsistents": [
1677     {
1678       "shards": [
1679         {
1680           "size": 2048,
1681           "errors": [],
1682           "shard": 2,
1683           "osd": 0
1684         },
1685         {
1686           "size": 9,
1687           "shard": 0,
1688           "errors": [
1689             "size_mismatch_oi"
1690           ],
1691           "osd": 1
1692         },
1693         {
1694           "size": 2048,
1695           "shard": 1,
1696           "errors": [],
1697           "osd": 2
1698         }
1699       ],
1700       "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1701       "union_shard_errors": [
1702         "size_mismatch_oi"
1703       ],
1704       "errors": [
1705         "size_mismatch"
1706       ],
1707       "object": {
1708         "version": 1,
1709         "snap": "head",
1710         "locator": "",
1711         "nspace": "",
1712         "name": "EOBJ1"
1713       }
1714     },
1715     {
1716       "shards": [
1717         {
1718           "size": 2048,
1719           "errors": [],
1720           "shard": 2,
1721           "osd": 0
1722         },
1723         {
1724           "shard": 0,
1725           "errors": [
1726             "missing"
1727           ],
1728           "osd": 1
1729         },
1730         {
1731           "size": 2048,
1732           "shard": 1,
1733           "errors": [],
1734           "osd": 2
1735         }
1736       ],
1737       "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1738       "union_shard_errors": [
1739         "missing"
1740       ],
1741       "errors": [],
1742       "object": {
1743         "version": 3,
1744         "snap": "head",
1745         "locator": "",
1746         "nspace": "",
1747         "name": "EOBJ3"
1748       }
1749     },
1750     {
1751       "shards": [
1752         {
1753           "attrs": [
1754             {
1755               "Base64": true,
1756               "value": "",
1757               "name": "_"
1758             },
1759             {
1760               "Base64": false,
1761               "value": "bad-val",
1762               "name": "_key1-EOBJ4"
1763             },
1764             {
1765               "Base64": false,
1766               "value": "val3-EOBJ4",
1767               "name": "_key3-EOBJ4"
1768             },
1769             {
1770               "Base64": true,
1771               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1772               "name": "hinfo_key"
1773             },
1774             {
1775               "Base64": true,
1776               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1777               "name": "snapset"
1778             }
1779           ],
1780           "size": 2048,
1781           "errors": [],
1782           "shard": 2,
1783           "osd": 0
1784         },
1785         {
1786           "osd": 1,
1787           "shard": 0,
1788           "errors": [],
1789           "size": 2048,
1790           "attrs": [
1791             {
1792               "Base64": true,
1793               "value": "",
1794               "name": "_"
1795             },
1796             {
1797               "Base64": false,
1798               "value": "val1-EOBJ4",
1799               "name": "_key1-EOBJ4"
1800             },
1801             {
1802               "Base64": false,
1803               "value": "val2-EOBJ4",
1804               "name": "_key2-EOBJ4"
1805             },
1806             {
1807               "Base64": true,
1808               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1809               "name": "hinfo_key"
1810             },
1811             {
1812               "Base64": true,
1813               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1814               "name": "snapset"
1815             }
1816           ]
1817         },
1818         {
1819           "osd": 2,
1820           "shard": 1,
1821           "errors": [],
1822           "size": 2048,
1823           "attrs": [
1824             {
1825               "Base64": true,
1826               "value": "",
1827               "name": "_"
1828             },
1829             {
1830               "Base64": false,
1831               "value": "val1-EOBJ4",
1832               "name": "_key1-EOBJ4"
1833             },
1834             {
1835               "Base64": false,
1836               "value": "val2-EOBJ4",
1837               "name": "_key2-EOBJ4"
1838             },
1839             {
1840               "Base64": true,
1841               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1842               "name": "hinfo_key"
1843             },
1844             {
1845               "Base64": true,
1846               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1847               "name": "snapset"
1848             }
1849           ]
1850         }
1851       ],
1852       "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1853       "union_shard_errors": [],
1854       "errors": [
1855         "attr_value_mismatch",
1856         "attr_name_mismatch"
1857       ],
1858       "object": {
1859         "version": 6,
1860         "snap": "head",
1861         "locator": "",
1862         "nspace": "",
1863         "name": "EOBJ4"
1864       }
1865     },
1866     {
1867       "shards": [
1868         {
1869           "size": 2048,
1870           "errors": [],
1871           "shard": 2,
1872           "osd": 0
1873         },
1874         {
1875           "size": 4096,
1876           "shard": 0,
1877           "errors": [
1878             "size_mismatch_oi"
1879           ],
1880           "osd": 1
1881         },
1882         {
1883           "size": 2048,
1884           "shard": 1,
1885           "errors": [],
1886           "osd": 2
1887         }
1888       ],
1889       "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1890       "union_shard_errors": [
1891         "size_mismatch_oi"
1892       ],
1893       "errors": [
1894         "size_mismatch"
1895       ],
1896       "object": {
1897         "version": 7,
1898         "snap": "head",
1899         "locator": "",
1900         "nspace": "",
1901         "name": "EOBJ5"
1902       }
1903     }
1904   ],
1905   "epoch": 0
1906 }
1907 EOF
1908
1909     jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
1910     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
1911     if test $getjson = "yes"
1912     then
1913         jq '.' $dir/json > save3.json
1914     fi
1915
1916     if which jsonschema > /dev/null;
1917     then
1918       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
1919     fi
1920
1921     pg_deep_scrub $pg
1922
1923     rados list-inconsistent-pg $poolname > $dir/json || return 1
1924     # Check pg count
1925     test $(jq '. | length' $dir/json) = "1" || return 1
1926     # Check pgid
1927     test $(jq -r '.[0]' $dir/json) = $pg || return 1
1928
1929     rados list-inconsistent-obj $pg > $dir/json || return 1
1930     # Get epoch for repair-get requests
1931     epoch=$(jq .epoch $dir/json)
1932
1933     if [ "$allow_overwrites" = "true" ]
1934     then
1935       jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
1936 {
1937   "inconsistents": [
1938     {
1939       "shards": [
1940         {
1941           "data_digest": "0x00000000",
1942           "omap_digest": "0xffffffff",
1943           "size": 2048,
1944           "errors": [],
1945           "shard": 2,
1946           "osd": 0
1947         },
1948         {
1949           "size": 9,
1950           "shard": 0,
1951           "errors": [
1952             "read_error",
1953             "size_mismatch_oi"
1954           ],
1955           "osd": 1
1956         },
1957         {
1958           "data_digest": "0x00000000",
1959           "omap_digest": "0xffffffff",
1960           "size": 2048,
1961           "shard": 1,
1962           "errors": [],
1963           "osd": 2
1964         }
1965       ],
1966       "selected_object_info": "3:9175b684:::EOBJ1:head(27'1 client.4155.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1967       "union_shard_errors": [
1968         "read_error",
1969         "size_mismatch_oi"
1970       ],
1971       "errors": [
1972         "size_mismatch"
1973       ],
1974       "object": {
1975         "version": 1,
1976         "snap": "head",
1977         "locator": "",
1978         "nspace": "",
1979         "name": "EOBJ1"
1980       }
1981     },
1982     {
1983       "shards": [
1984         {
1985           "data_digest": "0x00000000",
1986           "omap_digest": "0xffffffff",
1987           "size": 2048,
1988           "errors": [],
1989           "shard": 2,
1990           "osd": 0
1991         },
1992         {
1993           "shard": 0,
1994           "errors": [
1995             "missing"
1996           ],
1997           "osd": 1
1998         },
1999         {
2000           "data_digest": "0x00000000",
2001           "omap_digest": "0xffffffff",
2002           "size": 2048,
2003           "shard": 1,
2004           "errors": [],
2005           "osd": 2
2006         }
2007       ],
2008       "selected_object_info": "3:b197b25d:::EOBJ3:head(41'3 client.4199.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2009       "union_shard_errors": [
2010         "missing"
2011       ],
2012       "errors": [],
2013       "object": {
2014         "version": 3,
2015         "snap": "head",
2016         "locator": "",
2017         "nspace": "",
2018         "name": "EOBJ3"
2019       }
2020     },
2021     {
2022       "shards": [
2023         {
2024           "attrs": [
2025             {
2026               "Base64": true,
2027               "value": "",
2028               "name": "_"
2029             },
2030             {
2031               "Base64": false,
2032               "value": "bad-val",
2033               "name": "_key1-EOBJ4"
2034             },
2035             {
2036               "Base64": false,
2037               "value": "val3-EOBJ4",
2038               "name": "_key3-EOBJ4"
2039             },
2040             {
2041               "Base64": true,
2042               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2043               "name": "hinfo_key"
2044             },
2045             {
2046               "Base64": true,
2047               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2048               "name": "snapset"
2049             }
2050           ],
2051           "data_digest": "0x00000000",
2052           "omap_digest": "0xffffffff",
2053           "size": 2048,
2054           "errors": [],
2055           "shard": 2,
2056           "osd": 0
2057         },
2058         {
2059           "attrs": [
2060             {
2061               "Base64": true,
2062               "value": "",
2063               "name": "_"
2064             },
2065             {
2066               "Base64": false,
2067               "value": "val1-EOBJ4",
2068               "name": "_key1-EOBJ4"
2069             },
2070             {
2071               "Base64": false,
2072               "value": "val2-EOBJ4",
2073               "name": "_key2-EOBJ4"
2074             },
2075             {
2076               "Base64": true,
2077               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2078               "name": "hinfo_key"
2079             },
2080             {
2081               "Base64": true,
2082               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2083               "name": "snapset"
2084             }
2085           ],
2086           "data_digest": "0x00000000",
2087           "omap_digest": "0xffffffff",
2088           "size": 2048,
2089           "errors": [],
2090           "shard": 0,
2091           "osd": 1
2092         },
2093         {
2094           "attrs": [
2095             {
2096               "Base64": true,
2097               "value": "",
2098               "name": "_"
2099             },
2100             {
2101               "Base64": false,
2102               "value": "val1-EOBJ4",
2103               "name": "_key1-EOBJ4"
2104             },
2105             {
2106               "Base64": false,
2107               "value": "val2-EOBJ4",
2108               "name": "_key2-EOBJ4"
2109             },
2110             {
2111               "Base64": true,
2112               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2113               "name": "hinfo_key"
2114             },
2115             {
2116               "Base64": true,
2117               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2118               "name": "snapset"
2119             }
2120           ],
2121           "data_digest": "0x00000000",
2122           "omap_digest": "0xffffffff",
2123           "size": 2048,
2124           "errors": [],
2125           "shard": 1,
2126           "osd": 2
2127         }
2128       ],
2129       "selected_object_info": "3:5e723e06:::EOBJ4:head(48'6 client.4223.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2130       "union_shard_errors": [],
2131       "errors": [
2132         "attr_value_mismatch",
2133         "attr_name_mismatch"
2134       ],
2135       "object": {
2136         "version": 6,
2137         "snap": "head",
2138         "locator": "",
2139         "nspace": "",
2140         "name": "EOBJ4"
2141       }
2142     },
2143     {
2144       "shards": [
2145         {
2146           "data_digest": "0x00000000",
2147           "omap_digest": "0xffffffff",
2148           "size": 2048,
2149           "errors": [],
2150           "shard": 2,
2151           "osd": 0
2152         },
2153         {
2154           "data_digest": "0x00000000",
2155           "omap_digest": "0xffffffff",
2156           "size": 4096,
2157           "errors": [
2158             "size_mismatch_oi"
2159           ],
2160           "shard": 0,
2161           "osd": 1
2162         },
2163         {
2164           "data_digest": "0x00000000",
2165           "omap_digest": "0xffffffff",
2166           "size": 2048,
2167           "errors": [],
2168           "shard": 1,
2169           "osd": 2
2170         }
2171       ],
2172       "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4288.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2173       "union_shard_errors": [
2174         "size_mismatch_oi"
2175       ],
2176       "errors": [
2177         "size_mismatch"
2178       ],
2179       "object": {
2180         "version": 7,
2181         "snap": "head",
2182         "locator": "",
2183         "nspace": "",
2184         "name": "EOBJ5"
2185       }
2186     }
2187   ],
2188   "epoch": 0
2189 }
2190 EOF
2191
2192     else
2193
2194       jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
2195 {
2196   "inconsistents": [
2197     {
2198       "shards": [
2199         {
2200           "data_digest": "0x04cfa72f",
2201           "omap_digest": "0xffffffff",
2202           "size": 2048,
2203           "errors": [],
2204           "shard": 2,
2205           "osd": 0
2206         },
2207         {
2208           "size": 9,
2209           "shard": 0,
2210           "errors": [
2211             "read_error",
2212             "size_mismatch_oi"
2213           ],
2214           "osd": 1
2215         },
2216         {
2217           "data_digest": "0x04cfa72f",
2218           "omap_digest": "0xffffffff",
2219           "size": 2048,
2220           "shard": 1,
2221           "errors": [],
2222           "osd": 2
2223         }
2224       ],
2225       "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2226       "union_shard_errors": [
2227         "read_error",
2228         "size_mismatch_oi"
2229       ],
2230       "errors": [
2231         "size_mismatch"
2232       ],
2233       "object": {
2234         "version": 1,
2235         "snap": "head",
2236         "locator": "",
2237         "nspace": "",
2238         "name": "EOBJ1"
2239       }
2240     },
2241     {
2242       "shards": [
2243         {
2244           "size": 2048,
2245           "errors": [
2246             "ec_hash_error"
2247           ],
2248           "shard": 2,
2249           "osd": 0
2250         },
2251         {
2252           "data_digest": "0x04cfa72f",
2253           "omap_digest": "0xffffffff",
2254           "size": 2048,
2255           "errors": [],
2256           "shard": 0,
2257           "osd": 1
2258         },
2259         {
2260           "data_digest": "0x04cfa72f",
2261           "omap_digest": "0xffffffff",
2262           "size": 2048,
2263           "errors": [],
2264           "shard": 1,
2265           "osd": 2
2266         }
2267       ],
2268       "selected_object_info": "3:9babd184:::EOBJ2:head(29'2 client.4217.0:1 dirty|data_digest|omap_digest s 7 uv 2 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2269       "union_shard_errors": [
2270         "ec_hash_error"
2271       ],
2272       "errors": [],
2273       "object": {
2274         "version": 2,
2275         "snap": "head",
2276         "locator": "",
2277         "nspace": "",
2278         "name": "EOBJ2"
2279       }
2280     },
2281     {
2282       "shards": [
2283         {
2284           "data_digest": "0x04cfa72f",
2285           "omap_digest": "0xffffffff",
2286           "size": 2048,
2287           "errors": [],
2288           "shard": 2,
2289           "osd": 0
2290         },
2291         {
2292           "osd": 1,
2293           "shard": 0,
2294           "errors": [
2295             "missing"
2296           ]
2297         },
2298         {
2299           "data_digest": "0x04cfa72f",
2300           "omap_digest": "0xffffffff",
2301           "size": 2048,
2302           "shard": 1,
2303           "errors": [],
2304           "osd": 2
2305         }
2306       ],
2307       "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2308       "union_shard_errors": [
2309         "missing"
2310       ],
2311       "errors": [],
2312       "object": {
2313         "version": 3,
2314         "snap": "head",
2315         "locator": "",
2316         "nspace": "",
2317         "name": "EOBJ3"
2318       }
2319     },
2320     {
2321       "shards": [
2322         {
2323           "attrs": [
2324             {
2325               "Base64": true,
2326               "value": "",
2327               "name": "_"
2328             },
2329             {
2330               "Base64": false,
2331               "value": "bad-val",
2332               "name": "_key1-EOBJ4"
2333             },
2334             {
2335               "Base64": false,
2336               "value": "val3-EOBJ4",
2337               "name": "_key3-EOBJ4"
2338             },
2339             {
2340               "Base64": true,
2341               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2342               "name": "hinfo_key"
2343             },
2344             {
2345               "Base64": true,
2346               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2347               "name": "snapset"
2348             }
2349           ],
2350           "data_digest": "0x04cfa72f",
2351           "omap_digest": "0xffffffff",
2352           "size": 2048,
2353           "errors": [],
2354           "shard": 2,
2355           "osd": 0
2356         },
2357         {
2358           "osd": 1,
2359           "shard": 0,
2360           "errors": [],
2361           "size": 2048,
2362           "omap_digest": "0xffffffff",
2363           "data_digest": "0x04cfa72f",
2364           "attrs": [
2365             {
2366               "Base64": true,
2367               "value": "",
2368               "name": "_"
2369             },
2370             {
2371               "Base64": false,
2372               "value": "val1-EOBJ4",
2373               "name": "_key1-EOBJ4"
2374             },
2375             {
2376               "Base64": false,
2377               "value": "val2-EOBJ4",
2378               "name": "_key2-EOBJ4"
2379             },
2380             {
2381               "Base64": true,
2382               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2383               "name": "hinfo_key"
2384             },
2385             {
2386               "Base64": true,
2387               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2388               "name": "snapset"
2389             }
2390           ]
2391         },
2392         {
2393           "osd": 2,
2394           "shard": 1,
2395           "errors": [],
2396           "size": 2048,
2397           "omap_digest": "0xffffffff",
2398           "data_digest": "0x04cfa72f",
2399           "attrs": [
2400             {
2401               "Base64": true,
2402               "value": "",
2403               "name": "_"
2404             },
2405             {
2406               "Base64": false,
2407               "value": "val1-EOBJ4",
2408               "name": "_key1-EOBJ4"
2409             },
2410             {
2411               "Base64": false,
2412               "value": "val2-EOBJ4",
2413               "name": "_key2-EOBJ4"
2414             },
2415             {
2416               "Base64": true,
2417               "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2418               "name": "hinfo_key"
2419             },
2420             {
2421               "Base64": true,
2422               "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2423               "name": "snapset"
2424             }
2425           ]
2426         }
2427       ],
2428       "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2429       "union_shard_errors": [],
2430       "errors": [
2431         "attr_value_mismatch",
2432         "attr_name_mismatch"
2433       ],
2434       "object": {
2435         "version": 6,
2436         "snap": "head",
2437         "locator": "",
2438         "nspace": "",
2439         "name": "EOBJ4"
2440       }
2441     },
2442     {
2443       "shards": [
2444         {
2445           "data_digest": "0x04cfa72f",
2446           "omap_digest": "0xffffffff",
2447           "size": 2048,
2448           "errors": [],
2449           "shard": 2,
2450           "osd": 0
2451         },
2452         {
2453           "size": 4096,
2454           "shard": 0,
2455           "errors": [
2456             "size_mismatch_oi",
2457             "ec_size_error"
2458           ],
2459           "osd": 1
2460         },
2461         {
2462           "data_digest": "0x04cfa72f",
2463           "omap_digest": "0xffffffff",
2464           "size": 2048,
2465           "shard": 1,
2466           "errors": [],
2467           "osd": 2
2468         }
2469       ],
2470       "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2471       "union_shard_errors": [
2472         "size_mismatch_oi",
2473         "ec_size_error"
2474       ],
2475       "errors": [
2476         "size_mismatch"
2477       ],
2478       "object": {
2479         "version": 7,
2480         "snap": "head",
2481         "locator": "",
2482         "nspace": "",
2483         "name": "EOBJ5"
2484       }
2485     }
2486   ],
2487   "epoch": 0
2488 }
2489 EOF
2490
2491     fi
2492
2493     jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
2494     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
2495     if test $getjson = "yes"
2496     then
2497       if [ "$allow_overwrites" = "true" ]
2498       then
2499         num=4
2500       else
2501         num=5
2502       fi
2503       jq '.' $dir/json > save${num}.json
2504     fi
2505
2506     if which jsonschema > /dev/null;
2507     then
2508       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
2509     fi
2510
2511     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
2512     teardown $dir || return 1
2513 }
2514
2515 function TEST_corrupt_scrub_erasure_appends() {
2516     corrupt_scrub_erasure $1 false
2517 }
2518
2519 function TEST_corrupt_scrub_erasure_overwrites() {
2520     if [ "$use_ec_overwrite" = "true" ]; then
2521         corrupt_scrub_erasure $1 true
2522     fi
2523 }
2524
2525 #
2526 # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost
2527 #
2528 function TEST_periodic_scrub_replicated() {
2529     local dir=$1
2530     local poolname=psr_pool
2531     local objname=POBJ
2532
2533     setup $dir || return 1
2534     run_mon $dir a --osd_pool_default_size=2 || return 1
2535     run_mgr $dir x || return 1
2536     local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0"
2537     run_osd $dir 0 $ceph_osd_args || return 1
2538     run_osd $dir 1 $ceph_osd_args || return 1
2539     wait_for_clean || return 1
2540
2541     ceph osd pool create $poolname 1 1 || return 1
2542     wait_for_clean || return 1
2543
2544     local osd=0
2545     add_something $dir $poolname $objname scrub || return 1
2546     local primary=$(get_primary $poolname $objname)
2547     local pg=$(get_pg $poolname $objname)
2548
2549     # Add deep-scrub only error
2550     local payload=UVWXYZ
2551     echo $payload > $dir/CORRUPT
2552     # Uses $ceph_osd_args for osd restart
2553     objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2554
2555     # No scrub information available, so expect failure
2556     set -o pipefail
2557     !  rados list-inconsistent-obj $pg | jq '.' || return 1
2558     set +o pipefail
2559
2560     pg_deep_scrub $pg || return 1
2561
2562     # Make sure bad object found
2563     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
2564
2565     local last_scrub=$(get_last_scrub_stamp $pg)
2566     # Fake a schedule scrub
2567     CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.${primary}.asok \
2568              trigger_scrub $pg || return 1
2569     # Wait for schedule regular scrub
2570     wait_for_scrub $pg "$last_scrub"
2571
2572     # It needed to be upgraded
2573     grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1
2574
2575     # Bad object still known
2576     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
2577
2578     # Can't upgrade with this set
2579     ceph osd set nodeep-scrub
2580     # Let map change propagate to OSDs
2581     sleep 2
2582
2583     # Fake a schedule scrub
2584     local last_scrub=$(get_last_scrub_stamp $pg)
2585     CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.${primary}.asok \
2586              trigger_scrub $pg || return 1
2587     # Wait for schedule regular scrub
2588     # to notice scrub and skip it
2589     local found=false
2590     for i in $(seq 14 -1 0)
2591     do
2592       sleep 1
2593       ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; }
2594       echo Time left: $i seconds
2595     done
2596     test $found = "true" || return 1
2597
2598     # Bad object still known
2599     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
2600
2601     # Request a regular scrub and it will be done
2602     local scrub_backoff_ratio=$(get_config osd ${primary} osd_scrub_backoff_ratio)
2603     set_config osd ${primary} osd_scrub_backoff_ratio 0
2604     pg_scrub $pg
2605     sleep 1
2606     set_config osd ${primary} osd_scrub_backoff_ratio $scrub_backoff_ratio
2607     grep -q "Regular scrub request, losing deep-scrub details" $dir/osd.${primary}.log || return 1
2608
2609     # deep-scrub error is no longer present
2610     rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1
2611 }
2612
2613
2614 main osd-scrub-repair "$@"
2615
2616 # Local Variables:
2617 # compile-command: "cd ../.. ; make -j4 && \
2618 #    test/osd/osd-scrub-repair.sh # TEST_corrupt_and_repair_replicated"
2619 # End: