ceph/qa/standalone/scrub/osd-scrub-repair.sh

   1 #!/bin/bash -x
   2 #
   3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
   4 #
   5 # Author: Loic Dachary <loic@dachary.org>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  18
  19 if [ `uname` = FreeBSD ]; then
  20     # erasure coding overwrites are only tested on Bluestore
  21     # erasure coding on filestore is unsafe
  22     # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites
  23     use_ec_overwrite=false
  24 else
  25     use_ec_overwrite=true
  26 fi
  27
  28 # Test development and debugging
  29 # Set to "yes" in order to ignore diff errors and save results to update test
  30 getjson="no"
  31
  32 # Filter out mtime and local_mtime dates, version, prior_version and last_reqid (client) from any object_info.
  33 jqfilter='def walk(f):
  34   . as $in
  35   | if type == "object" then
  36       reduce keys[] as $key
  37         ( {}; . + { ($key):  ($in[$key] | walk(f)) } ) | f
  38     elif type == "array" then map( walk(f) ) | f
  39     else f
  40     end;
  41 walk(if type == "object" then del(.mtime) else . end)
  42 | walk(if type == "object" then del(.local_mtime) else . end)
  43 | walk(if type == "object" then del(.last_reqid) else . end)
  44 | walk(if type == "object" then del(.version) else . end)
  45 | walk(if type == "object" then del(.prior_version) else . end)
  46 | walk(if type == "object" then del(.redirect_target) else . end)
  47 | walk(if type == "object" then del(.legacy_snaps) else . end)'
  48
  49 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
  50
  51 function run() {
  52     local dir=$1
  53     shift
  54
  55     export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
  56     export CEPH_ARGS
  57     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  58     CEPH_ARGS+="--mon-host=$CEPH_MON "
  59     CEPH_ARGS+="--osd-skip-data-digest=false "
  60
  61     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  62     for func in $funcs ; do
  63         $func $dir || return 1
  64     done
  65 }
  66
  67 function add_something() {
  68     local dir=$1
  69     local poolname=$2
  70     local obj=${3:-SOMETHING}
  71     local scrub=${4:-noscrub}
  72
  73     if [ "$scrub" = "noscrub" ];
  74     then
  75         ceph osd set noscrub || return 1
  76         ceph osd set nodeep-scrub || return 1
  77     else
  78         ceph osd unset noscrub || return 1
  79         ceph osd unset nodeep-scrub || return 1
  80     fi
  81
  82     local payload=ABCDEF
  83     echo $payload > $dir/ORIGINAL
  84     rados --pool $poolname put $obj $dir/ORIGINAL || return 1
  85 }
  86
  87 #
  88 # Corrupt one copy of a replicated pool
  89 #
  90 function TEST_corrupt_and_repair_replicated() {
  91     local dir=$1
  92     local poolname=rbd
  93
  94     setup $dir || return 1
  95     run_mon $dir a --osd_pool_default_size=2 || return 1
  96     run_mgr $dir x || return 1
  97     run_osd $dir 0 || return 1
  98     run_osd $dir 1 || return 1
  99     create_rbd_pool || return 1
 100     wait_for_clean || return 1
 101
 102     add_something $dir $poolname || return 1
 103     corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
 104     # Reproduces http://tracker.ceph.com/issues/8914
 105     corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1
 106
 107     teardown $dir || return 1
 108 }
 109
 110 function corrupt_and_repair_two() {
 111     local dir=$1
 112     local poolname=$2
 113     local first=$3
 114     local second=$4
 115
 116     #
 117     # 1) remove the corresponding file from the OSDs
 118     #
 119     pids=""
 120     run_in_background pids objectstore_tool $dir $first SOMETHING remove
 121     run_in_background pids objectstore_tool $dir $second SOMETHING remove
 122     wait_background pids
 123     return_code=$?
 124     if [ $return_code -ne 0 ]; then return $return_code; fi
 125
 126     #
 127     # 2) repair the PG
 128     #
 129     local pg=$(get_pg $poolname SOMETHING)
 130     repair $pg
 131     #
 132     # 3) The files must be back
 133     #
 134     pids=""
 135     run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs
 136     run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs
 137     wait_background pids
 138     return_code=$?
 139     if [ $return_code -ne 0 ]; then return $return_code; fi
 140
 141     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 142     diff $dir/ORIGINAL $dir/COPY || return 1
 143 }
 144
 145 #
 146 # 1) add an object
 147 # 2) remove the corresponding file from a designated OSD
 148 # 3) repair the PG
 149 # 4) check that the file has been restored in the designated OSD
 150 #
 151 function corrupt_and_repair_one() {
 152     local dir=$1
 153     local poolname=$2
 154     local osd=$3
 155
 156     #
 157     # 1) remove the corresponding file from the OSD
 158     #
 159     objectstore_tool $dir $osd SOMETHING remove || return 1
 160     #
 161     # 2) repair the PG
 162     #
 163     local pg=$(get_pg $poolname SOMETHING)
 164     repair $pg
 165     #
 166     # 3) The file must be back
 167     #
 168     objectstore_tool $dir $osd SOMETHING list-attrs || return 1
 169     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 170     diff $dir/ORIGINAL $dir/COPY || return 1
 171 }
 172
 173 function corrupt_and_repair_erasure_coded() {
 174     local dir=$1
 175     local poolname=$2
 176
 177     add_something $dir $poolname || return 1
 178
 179     local primary=$(get_primary $poolname SOMETHING)
 180     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 181     local not_primary_first=${osds[0]}
 182     local not_primary_second=${osds[1]}
 183
 184     # Reproduces http://tracker.ceph.com/issues/10017
 185     corrupt_and_repair_one $dir $poolname $primary  || return 1
 186     # Reproduces http://tracker.ceph.com/issues/10409
 187     corrupt_and_repair_one $dir $poolname $not_primary_first || return 1
 188     corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1
 189     corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1
 190
 191 }
 192
 193 function create_ec_pool() {
 194     local pool_name=$1
 195     local allow_overwrites=$2
 196
 197     ceph osd erasure-code-profile set myprofile crush-failure-domain=osd $3 $4 $5 $6 $7 || return 1
 198
 199     create_pool "$poolname" 1 1 erasure myprofile || return 1
 200
 201     if [ "$allow_overwrites" = "true" ]; then
 202         ceph osd pool set "$poolname" allow_ec_overwrites true || return 1
 203     fi
 204
 205     wait_for_clean || return 1
 206     return 0
 207 }
 208
 209 function auto_repair_erasure_coded() {
 210     local dir=$1
 211     local allow_overwrites=$2
 212     local poolname=ecpool
 213
 214     # Launch a cluster with 5 seconds scrub interval
 215     setup $dir || return 1
 216     run_mon $dir a || return 1
 217     run_mgr $dir x || return 1
 218     local ceph_osd_args="--osd-scrub-auto-repair=true \
 219             --osd-deep-scrub-interval=5 \
 220             --osd-scrub-max-interval=5 \
 221             --osd-scrub-min-interval=5 \
 222             --osd-scrub-interval-randomize-ratio=0"
 223     for id in $(seq 0 2) ; do
 224         if [ "$allow_overwrites" = "true" ]; then
 225             run_osd_bluestore $dir $id $ceph_osd_args || return 1
 226         else
 227             run_osd $dir $id $ceph_osd_args || return 1
 228         fi
 229     done
 230     create_rbd_pool || return 1
 231     wait_for_clean || return 1
 232
 233     # Create an EC pool
 234     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 235
 236     # Put an object
 237     local payload=ABCDEF
 238     echo $payload > $dir/ORIGINAL
 239     rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
 240
 241     # Remove the object from one shard physically
 242     # Restarted osd get $ceph_osd_args passed
 243     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
 244     # Wait for auto repair
 245     local pgid=$(get_pg $poolname SOMETHING)
 246     wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)"
 247     wait_for_clean || return 1
 248     # Verify - the file should be back
 249     # Restarted osd get $ceph_osd_args passed
 250     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
 251     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 252     diff $dir/ORIGINAL $dir/COPY || return 1
 253
 254     # Tear down
 255     teardown $dir || return 1
 256 }
 257
 258 function TEST_auto_repair_erasure_coded_appends() {
 259     auto_repair_erasure_coded $1 false
 260 }
 261
 262 function TEST_auto_repair_erasure_coded_overwrites() {
 263     if [ "$use_ec_overwrite" = "true" ]; then
 264         auto_repair_erasure_coded $1 true
 265     fi
 266 }
 267
 268 function corrupt_and_repair_jerasure() {
 269     local dir=$1
 270     local allow_overwrites=$2
 271     local poolname=ecpool
 272
 273     setup $dir || return 1
 274     run_mon $dir a || return 1
 275     run_mgr $dir x || return 1
 276     for id in $(seq 0 3) ; do
 277         if [ "$allow_overwrites" = "true" ]; then
 278             run_osd_bluestore $dir $id || return 1
 279         else
 280             run_osd $dir $id || return 1
 281         fi
 282     done
 283     create_rbd_pool || return 1
 284     wait_for_clean || return 1
 285
 286     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 287     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 288
 289     teardown $dir || return 1
 290 }
 291
 292 function TEST_corrupt_and_repair_jerasure_appends() {
 293     corrupt_and_repair_jerasure $1
 294 }
 295
 296 function TEST_corrupt_and_repair_jerasure_overwrites() {
 297     if [ "$use_ec_overwrite" = "true" ]; then
 298         corrupt_and_repair_jerasure $1 true
 299     fi
 300 }
 301
 302 function corrupt_and_repair_lrc() {
 303     local dir=$1
 304     local allow_overwrites=$2
 305     local poolname=ecpool
 306
 307     setup $dir || return 1
 308     run_mon $dir a || return 1
 309     run_mgr $dir x || return 1
 310     for id in $(seq 0 9) ; do
 311         if [ "$allow_overwrites" = "true" ]; then
 312             run_osd_bluestore $dir $id || return 1
 313         else
 314             run_osd $dir $id || return 1
 315         fi
 316     done
 317     create_rbd_pool || return 1
 318     wait_for_clean || return 1
 319
 320     create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1
 321     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 322
 323     teardown $dir || return 1
 324 }
 325
 326 function TEST_corrupt_and_repair_lrc_appends() {
 327     corrupt_and_repair_jerasure $1
 328 }
 329
 330 function TEST_corrupt_and_repair_lrc_overwrites() {
 331     if [ "$use_ec_overwrite" = "true" ]; then
 332         corrupt_and_repair_jerasure $1 true
 333     fi
 334 }
 335
 336 function unfound_erasure_coded() {
 337     local dir=$1
 338     local allow_overwrites=$2
 339     local poolname=ecpool
 340     local payload=ABCDEF
 341
 342     setup $dir || return 1
 343     run_mon $dir a || return 1
 344     run_mgr $dir x || return 1
 345     for id in $(seq 0 3) ; do
 346         if [ "$allow_overwrites" = "true" ]; then
 347             run_osd_bluestore $dir $id || return 1
 348         else
 349             run_osd $dir $id || return 1
 350         fi
 351     done
 352     create_rbd_pool || return 1
 353     wait_for_clean || return 1
 354
 355     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 356
 357     add_something $dir $poolname || return 1
 358
 359     local primary=$(get_primary $poolname SOMETHING)
 360     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 361     local not_primary_first=${osds[0]}
 362     local not_primary_second=${osds[1]}
 363     local not_primary_third=${osds[2]}
 364
 365     #
 366     # 1) remove the corresponding file from the OSDs
 367     #
 368     pids=""
 369     run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove
 370     run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove
 371     run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove
 372     wait_background pids
 373     return_code=$?
 374     if [ $return_code -ne 0 ]; then return $return_code; fi
 375
 376     #
 377     # 2) repair the PG
 378     #
 379     local pg=$(get_pg $poolname SOMETHING)
 380     repair $pg
 381     #
 382     # 3) check pg state
 383     #
 384     # it may take a bit to appear due to mon/mgr asynchrony
 385     for f in `seq 1 60`; do
 386         ceph -s | grep "1/1 objects unfound" && break
 387         sleep 1
 388     done
 389     ceph -s|grep "4 osds: 4 up, 4 in" || return 1
 390     ceph -s|grep "1/1 objects unfound" || return 1
 391
 392     teardown $dir || return 1
 393 }
 394
 395 function TEST_unfound_erasure_coded_appends() {
 396     unfound_erasure_coded $1
 397 }
 398
 399 function TEST_unfound_erasure_coded_overwrites() {
 400     if [ "$use_ec_overwrite" = "true" ]; then
 401         unfound_erasure_coded $1 true
 402     fi
 403 }
 404
 405 #
 406 # list_missing for EC pool
 407 #
 408 function list_missing_erasure_coded() {
 409     local dir=$1
 410     local allow_overwrites=$2
 411     local poolname=ecpool
 412
 413     setup $dir || return 1
 414     run_mon $dir a || return 1
 415     run_mgr $dir x || return 1
 416     for id in $(seq 0 2) ; do
 417         if [ "$allow_overwrites" = "true" ]; then
 418             run_osd_bluestore $dir $id || return 1
 419         else
 420             run_osd $dir $id || return 1
 421         fi
 422     done
 423     create_rbd_pool || return 1
 424     wait_for_clean || return 1
 425
 426     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 427
 428     # Put an object and remove the two shards (including primary)
 429     add_something $dir $poolname MOBJ0 || return 1
 430     local -a osds0=($(get_osds $poolname MOBJ0))
 431
 432     # Put another object and remove two shards (excluding primary)
 433     add_something $dir $poolname MOBJ1 || return 1
 434     local -a osds1=($(get_osds $poolname MOBJ1))
 435
 436     # Stop all osd daemons
 437     for id in $(seq 0 2) ; do
 438         kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1
 439     done
 440
 441     id=${osds0[0]}
 442     ceph-objectstore-tool --data-path $dir/$id \
 443         MOBJ0 remove || return 1
 444     id=${osds0[1]}
 445     ceph-objectstore-tool --data-path $dir/$id \
 446         MOBJ0 remove || return 1
 447
 448     id=${osds1[1]}
 449     ceph-objectstore-tool --data-path $dir/$id \
 450         MOBJ1 remove || return 1
 451     id=${osds1[2]}
 452     ceph-objectstore-tool --data-path $dir/$id \
 453         MOBJ1 remove || return 1
 454
 455     for id in $(seq 0 2) ; do
 456         activate_osd $dir $id >&2 || return 1
 457     done
 458     create_rbd_pool || return 1
 459     wait_for_clean || return 1
 460
 461     # Get get - both objects should in the same PG
 462     local pg=$(get_pg $poolname MOBJ0)
 463
 464     # Repair the PG, which triggers the recovering,
 465     # and should mark the object as unfound
 466     repair $pg
 467
 468     for i in $(seq 0 120) ; do
 469         [ $i -lt 60 ] || return 1
 470         matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l)
 471         [ $matches -eq 2 ] && break
 472     done
 473
 474     teardown $dir || return 1
 475 }
 476
 477 function TEST_list_missing_erasure_coded_appends() {
 478     list_missing_erasure_coded $1 false
 479 }
 480
 481 function TEST_list_missing_erasure_coded_overwrites() {
 482     if [ "$use_ec_overwrite" = "true" ]; then
 483         list_missing_erasure_coded $1 true
 484     fi
 485 }
 486
 487 #
 488 # Corrupt one copy of a replicated pool
 489 #
 490 function TEST_corrupt_scrub_replicated() {
 491     local dir=$1
 492     local poolname=csr_pool
 493     local total_objs=16
 494
 495     setup $dir || return 1
 496     run_mon $dir a --osd_pool_default_size=2 || return 1
 497     run_mgr $dir x || return 1
 498     run_osd $dir 0 || return 1
 499     run_osd $dir 1 || return 1
 500     create_rbd_pool || return 1
 501     wait_for_clean || return 1
 502
 503     create_pool foo 1 || return 1
 504     create_pool $poolname 1 1 || return 1
 505     wait_for_clean || return 1
 506
 507     for i in $(seq 1 $total_objs) ; do
 508         objname=ROBJ${i}
 509         add_something $dir $poolname $objname || return 1
 510
 511         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
 512         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
 513     done
 514
 515     local pg=$(get_pg $poolname ROBJ0)
 516
 517     # Compute an old omap digest and save oi
 518     CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \
 519         config set osd_deep_scrub_update_digest_min_age 0
 520     CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \
 521         config set osd_deep_scrub_update_digest_min_age 0
 522     pg_deep_scrub $pg
 523
 524     for i in $(seq 1 $total_objs) ; do
 525         objname=ROBJ${i}
 526
 527         # Alternate corruption between osd.0 and osd.1
 528         local osd=$(expr $i % 2)
 529
 530         case $i in
 531         1)
 532             # Size (deep scrub data_digest too)
 533             local payload=UVWXYZZZ
 534             echo $payload > $dir/CORRUPT
 535             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 536             ;;
 537
 538         2)
 539             # digest (deep scrub only)
 540             local payload=UVWXYZ
 541             echo $payload > $dir/CORRUPT
 542             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 543             ;;
 544
 545         3)
 546              # missing
 547              objectstore_tool $dir $osd $objname remove || return 1
 548              ;;
 549
 550          4)
 551              # Modify omap value (deep scrub only)
 552              objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1
 553              ;;
 554
 555          5)
 556             # Delete omap key (deep scrub only)
 557             objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1
 558             ;;
 559
 560          6)
 561             # Add extra omap key (deep scrub only)
 562             echo extra > $dir/extra-val
 563             objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1
 564             rm $dir/extra-val
 565             ;;
 566
 567          7)
 568             # Modify omap header (deep scrub only)
 569             echo -n newheader > $dir/hdr
 570             objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1
 571             rm $dir/hdr
 572             ;;
 573
 574          8)
 575             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
 576             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
 577
 578             # Break xattrs
 579             echo -n bad-val > $dir/bad-val
 580             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
 581             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
 582             echo -n val3-$objname > $dir/newval
 583             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
 584             rm $dir/bad-val $dir/newval
 585             ;;
 586
 587         9)
 588             objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi
 589             echo -n D > $dir/change
 590             rados --pool $poolname put $objname $dir/change
 591             objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi
 592             rm $dir/oi $dir/change
 593             ;;
 594
 595           # ROBJ10 must be handled after digests are re-computed by a deep scrub below
 596           # ROBJ11 must be handled with config change before deep scrub
 597           # ROBJ12 must be handled with config change before scrubs
 598           # ROBJ13 must be handled before scrubs
 599
 600         14)
 601             echo -n bad-val > $dir/bad-val
 602             objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1
 603             objectstore_tool $dir 1 $objname rm-attr _ || return 1
 604             rm $dir/bad-val
 605             ;;
 606
 607         15)
 608             objectstore_tool $dir $osd $objname rm-attr _ || return 1
 609             ;;
 610
 611         16)
 612             objectstore_tool $dir 0 $objname rm-attr snapset || return 1
 613             echo -n bad-val > $dir/bad-val
 614             objectstore_tool $dir 1 $objname set-attr snapset $dir/bad-val || return 1
 615
 616         esac
 617     done
 618
 619     local pg=$(get_pg $poolname ROBJ0)
 620
 621     inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
 622     inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
 623     inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
 624     inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
 625
 626     pg_scrub $pg
 627
 628     rados list-inconsistent-pg $poolname > $dir/json || return 1
 629     # Check pg count
 630     test $(jq '. | length' $dir/json) = "1" || return 1
 631     # Check pgid
 632     test $(jq -r '.[0]' $dir/json) = $pg || return 1
 633
 634     rados list-inconsistent-obj $pg > $dir/json || return 1
 635     # Get epoch for repair-get requests
 636     epoch=$(jq .epoch $dir/json)
 637
 638     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
 639 {
 640   "inconsistents": [
 641     {
 642       "shards": [
 643         {
 644           "size": 7,
 645           "errors": [],
 646           "osd": 0,
 647           "primary": false
 648         },
 649         {
 650           "object_info": {
 651             "oid": {
 652               "oid": "ROBJ1",
 653               "key": "",
 654               "snapid": -2,
 655               "hash": 1454963827,
 656               "max": 0,
 657               "pool": 3,
 658               "namespace": ""
 659             },
 660             "version": "51'58",
 661             "prior_version": "21'3",
 662             "last_reqid": "osd.1.0:57",
 663             "user_version": 3,
 664             "size": 7,
 665             "mtime": "",
 666             "local_mtime": "",
 667             "lost": 0,
 668             "flags": [
 669               "dirty",
 670               "omap",
 671               "data_digest",
 672               "omap_digest"
 673             ],
 674             "truncate_seq": 0,
 675             "truncate_size": 0,
 676             "data_digest": "0x2ddbf8f5",
 677             "omap_digest": "0xf5fba2c6",
 678             "expected_object_size": 0,
 679             "expected_write_size": 0,
 680             "alloc_hint_flags": 0,
 681             "manifest": {
 682               "type": 0
 683             },
 684             "watchers": {}
 685           },
 686           "size": 9,
 687           "errors": [
 688             "size_mismatch_info",
 689             "obj_size_info_mismatch"
 690           ],
 691           "osd": 1,
 692           "primary": true
 693         }
 694       ],
 695       "selected_object_info": {
 696         "oid": {
 697           "oid": "ROBJ1",
 698           "key": "",
 699           "snapid": -2,
 700           "hash": 1454963827,
 701           "max": 0,
 702           "pool": 3,
 703           "namespace": ""
 704         },
 705         "version": "51'58",
 706         "prior_version": "21'3",
 707         "last_reqid": "osd.1.0:57",
 708         "user_version": 3,
 709         "size": 7,
 710         "mtime": "2018-04-05 14:33:19.804040",
 711         "local_mtime": "2018-04-05 14:33:19.804839",
 712         "lost": 0,
 713         "flags": [
 714           "dirty",
 715           "omap",
 716           "data_digest",
 717           "omap_digest"
 718         ],
 719         "truncate_seq": 0,
 720         "truncate_size": 0,
 721         "data_digest": "0x2ddbf8f5",
 722         "omap_digest": "0xf5fba2c6",
 723         "expected_object_size": 0,
 724         "expected_write_size": 0,
 725         "alloc_hint_flags": 0,
 726         "manifest": {
 727           "type": 0
 728         },
 729         "watchers": {}
 730       },
 731       "union_shard_errors": [
 732         "size_mismatch_info",
 733         "obj_size_info_mismatch"
 734       ],
 735       "errors": [
 736         "size_mismatch"
 737       ],
 738       "object": {
 739         "version": 3,
 740         "snap": "head",
 741         "locator": "",
 742         "nspace": "",
 743         "name": "ROBJ1"
 744       }
 745     },
 746     {
 747       "shards": [
 748         {
 749           "errors": [
 750             "stat_error"
 751           ],
 752           "osd": 0,
 753           "primary": false
 754         },
 755         {
 756           "size": 7,
 757           "errors": [],
 758           "osd": 1,
 759           "primary": true
 760         }
 761       ],
 762       "selected_object_info": {
 763         "oid": {
 764           "oid": "ROBJ12",
 765           "key": "",
 766           "snapid": -2,
 767           "hash": 3920199997,
 768           "max": 0,
 769           "pool": 3,
 770           "namespace": ""
 771         },
 772         "version": "51'56",
 773         "prior_version": "43'36",
 774         "last_reqid": "osd.1.0:55",
 775         "user_version": 36,
 776         "size": 7,
 777         "mtime": "",
 778         "local_mtime": "",
 779         "lost": 0,
 780         "flags": [
 781           "dirty",
 782           "omap",
 783           "data_digest",
 784           "omap_digest"
 785         ],
 786         "truncate_seq": 0,
 787         "truncate_size": 0,
 788         "data_digest": "0x2ddbf8f5",
 789         "omap_digest": "0x067f306a",
 790         "expected_object_size": 0,
 791         "expected_write_size": 0,
 792         "alloc_hint_flags": 0,
 793         "manifest": {
 794           "type": 0
 795         },
 796         "watchers": {}
 797       },
 798       "union_shard_errors": [
 799         "stat_error"
 800       ],
 801       "errors": [],
 802       "object": {
 803         "version": 36,
 804         "snap": "head",
 805         "locator": "",
 806         "nspace": "",
 807         "name": "ROBJ12"
 808       }
 809     },
 810     {
 811       "shards": [
 812         {
 813           "errors": [
 814             "stat_error"
 815           ],
 816           "osd": 0,
 817           "primary": false
 818         },
 819         {
 820           "size": 7,
 821           "errors": [],
 822           "osd": 1,
 823           "primary": true
 824         }
 825       ],
 826       "selected_object_info": {
 827         "oid": {
 828           "oid": "ROBJ13",
 829           "key": "",
 830           "snapid": -2,
 831           "hash": 2682806379,
 832           "max": 0,
 833           "pool": 3,
 834           "namespace": ""
 835         },
 836         "version": "51'59",
 837         "prior_version": "45'39",
 838         "last_reqid": "osd.1.0:58",
 839         "user_version": 39,
 840         "size": 7,
 841         "mtime": "",
 842         "local_mtime": "",
 843         "lost": 0,
 844         "flags": [
 845           "dirty",
 846           "omap",
 847           "data_digest",
 848           "omap_digest"
 849         ],
 850         "truncate_seq": 0,
 851         "truncate_size": 0,
 852         "data_digest": "0x2ddbf8f5",
 853         "omap_digest": "0x6441854d",
 854         "expected_object_size": 0,
 855         "expected_write_size": 0,
 856         "alloc_hint_flags": 0,
 857         "manifest": {
 858           "type": 0
 859         },
 860         "watchers": {}
 861       },
 862       "union_shard_errors": [
 863         "stat_error"
 864       ],
 865       "errors": [],
 866       "object": {
 867         "version": 39,
 868         "snap": "head",
 869         "locator": "",
 870         "nspace": "",
 871         "name": "ROBJ13"
 872       }
 873     },
 874     {
 875       "shards": [
 876         {
 877           "object_info": "bad-val",
 878           "size": 7,
 879           "errors": [
 880             "info_corrupted"
 881           ],
 882           "osd": 0,
 883           "primary": false
 884         },
 885         {
 886           "size": 7,
 887           "errors": [
 888             "info_missing"
 889           ],
 890           "osd": 1,
 891           "primary": true
 892         }
 893       ],
 894       "union_shard_errors": [
 895         "info_missing",
 896         "info_corrupted"
 897       ],
 898       "errors": [],
 899       "object": {
 900         "version": 0,
 901         "snap": "head",
 902         "locator": "",
 903         "nspace": "",
 904         "name": "ROBJ14"
 905       }
 906     },
 907     {
 908       "shards": [
 909         {
 910           "object_info": {
 911             "oid": {
 912               "oid": "ROBJ15",
 913               "key": "",
 914               "snapid": -2,
 915               "hash": 504996876,
 916               "max": 0,
 917               "pool": 3,
 918               "namespace": ""
 919             },
 920             "version": "51'49",
 921             "prior_version": "49'45",
 922             "last_reqid": "osd.1.0:48",
 923             "user_version": 45,
 924             "size": 7,
 925             "mtime": "2018-04-05 14:33:29.498969",
 926             "local_mtime": "2018-04-05 14:33:29.499890",
 927             "lost": 0,
 928             "flags": [
 929               "dirty",
 930               "omap",
 931               "data_digest",
 932               "omap_digest"
 933             ],
 934             "truncate_seq": 0,
 935             "truncate_size": 0,
 936             "data_digest": "0x2ddbf8f5",
 937             "omap_digest": "0x2d2a4d6e",
 938             "expected_object_size": 0,
 939             "expected_write_size": 0,
 940             "alloc_hint_flags": 0,
 941             "manifest": {
 942               "type": 0
 943             },
 944             "watchers": {}
 945           },
 946           "size": 7,
 947           "errors": [],
 948           "osd": 0,
 949           "primary": false
 950         },
 951         {
 952           "size": 7,
 953           "errors": [
 954             "info_missing"
 955           ],
 956           "osd": 1,
 957           "primary": true
 958         }
 959       ],
 960       "selected_object_info": {
 961         "oid": {
 962           "oid": "ROBJ15",
 963           "key": "",
 964           "snapid": -2,
 965           "hash": 504996876,
 966           "max": 0,
 967           "pool": 3,
 968           "namespace": ""
 969         },
 970         "version": "51'49",
 971         "prior_version": "49'45",
 972         "last_reqid": "osd.1.0:48",
 973         "user_version": 45,
 974         "size": 7,
 975         "mtime": "",
 976         "local_mtime": "",
 977         "lost": 0,
 978         "flags": [
 979           "dirty",
 980           "omap",
 981           "data_digest",
 982           "omap_digest"
 983         ],
 984         "truncate_seq": 0,
 985         "truncate_size": 0,
 986         "data_digest": "0x2ddbf8f5",
 987         "omap_digest": "0x2d2a4d6e",
 988         "expected_object_size": 0,
 989         "expected_write_size": 0,
 990         "alloc_hint_flags": 0,
 991         "manifest": {
 992           "type": 0
 993         },
 994         "watchers": {}
 995       },
 996       "union_shard_errors": [
 997         "info_missing"
 998       ],
 999       "errors": [],
1000       "object": {
1001         "version": 45,
1002         "snap": "head",
1003         "locator": "",
1004         "nspace": "",
1005         "name": "ROBJ15"
1006       }
1007     },
1008     {
1009       "errors": [],
1010       "object": {
1011       "locator": "",
1012       "name": "ROBJ16",
1013       "nspace": "",
1014       "snap": "head",
1015       "version": 0
1016        },
1017         "shards": [
1018       {
1019         "errors": [
1020           "snapset_missing"
1021         ],
1022         "osd": 0,
1023         "primary": false,
1024         "size": 7
1025       },
1026       {
1027         "errors": [
1028           "snapset_corrupted"
1029         ],
1030         "osd": 1,
1031         "primary": true,
1032         "snapset": "bad-val",
1033         "size": 7
1034       }
1035       ],
1036       "union_shard_errors": [
1037         "snapset_missing",
1038         "snapset_corrupted"
1039       ]
1040     },
1041     {
1042       "shards": [
1043         {
1044           "size": 7,
1045           "errors": [],
1046           "osd": 0,
1047           "primary": false
1048         },
1049         {
1050           "errors": [
1051             "missing"
1052           ],
1053           "osd": 1,
1054           "primary": true
1055         }
1056       ],
1057       "selected_object_info": {
1058         "oid": {
1059           "oid": "ROBJ3",
1060           "key": "",
1061           "snapid": -2,
1062           "hash": 625845583,
1063           "max": 0,
1064           "pool": 3,
1065           "namespace": ""
1066         },
1067         "version": "51'61",
1068         "prior_version": "25'9",
1069         "last_reqid": "osd.1.0:60",
1070         "user_version": 9,
1071         "size": 7,
1072         "mtime": "",
1073         "local_mtime": "",
1074         "lost": 0,
1075         "flags": [
1076           "dirty",
1077           "omap",
1078           "data_digest",
1079           "omap_digest"
1080         ],
1081         "truncate_seq": 0,
1082         "truncate_size": 0,
1083         "data_digest": "0x2ddbf8f5",
1084         "omap_digest": "0x00b35dfd",
1085         "expected_object_size": 0,
1086         "expected_write_size": 0,
1087         "alloc_hint_flags": 0,
1088         "manifest": {
1089           "type": 0
1090         },
1091         "watchers": {}
1092       },
1093       "union_shard_errors": [
1094         "missing"
1095       ],
1096       "errors": [],
1097       "object": {
1098         "version": 9,
1099         "snap": "head",
1100         "locator": "",
1101         "nspace": "",
1102         "name": "ROBJ3"
1103       }
1104     },
1105     {
1106       "shards": [
1107         {
1108           "attrs": [
1109             {
1110               "Base64": false,
1111               "value": "bad-val",
1112               "name": "key1-ROBJ8"
1113             },
1114             {
1115               "Base64": false,
1116               "value": "val2-ROBJ8",
1117               "name": "key2-ROBJ8"
1118             }
1119           ],
1120           "size": 7,
1121           "errors": [],
1122           "osd": 0,
1123           "primary": false
1124         },
1125         {
1126           "attrs": [
1127             {
1128               "Base64": false,
1129               "value": "val1-ROBJ8",
1130               "name": "key1-ROBJ8"
1131             },
1132             {
1133               "Base64": false,
1134               "value": "val3-ROBJ8",
1135               "name": "key3-ROBJ8"
1136             }
1137           ],
1138           "size": 7,
1139           "errors": [],
1140           "osd": 1,
1141           "primary": true
1142         }
1143       ],
1144       "selected_object_info": {
1145         "oid": {
1146           "oid": "ROBJ8",
1147           "key": "",
1148           "snapid": -2,
1149           "hash": 2359695969,
1150           "max": 0,
1151           "pool": 3,
1152           "namespace": ""
1153         },
1154         "version": "79'66",
1155         "prior_version": "79'65",
1156         "last_reqid": "client.4554.0:1",
1157         "user_version": 66,
1158         "size": 7,
1159         "mtime": "",
1160         "local_mtime": "",
1161         "lost": 0,
1162         "flags": [
1163           "dirty",
1164           "omap",
1165           "data_digest",
1166           "omap_digest"
1167         ],
1168         "truncate_seq": 0,
1169         "truncate_size": 0,
1170         "data_digest": "0x2ddbf8f5",
1171         "omap_digest": "0xd6be81dc",
1172         "expected_object_size": 0,
1173         "expected_write_size": 0,
1174         "alloc_hint_flags": 0,
1175         "manifest": {
1176           "type": 0
1177         },
1178         "watchers": {}
1179       },
1180       "union_shard_errors": [],
1181       "errors": [
1182         "attr_value_mismatch",
1183         "attr_name_mismatch"
1184       ],
1185       "object": {
1186         "version": 66,
1187         "snap": "head",
1188         "locator": "",
1189         "nspace": "",
1190         "name": "ROBJ8"
1191       }
1192     },
1193     {
1194       "shards": [
1195         {
1196           "object_info": {
1197             "oid": {
1198               "oid": "ROBJ9",
1199               "key": "",
1200               "snapid": -2,
1201               "hash": 537189375,
1202               "max": 0,
1203               "pool": 3,
1204               "namespace": ""
1205             },
1206             "version": "95'67",
1207             "prior_version": "51'64",
1208             "last_reqid": "client.4649.0:1",
1209             "user_version": 67,
1210             "size": 1,
1211             "mtime": "",
1212             "local_mtime": "",
1213             "lost": 0,
1214             "flags": [
1215               "dirty",
1216               "omap",
1217               "data_digest",
1218               "omap_digest"
1219             ],
1220             "truncate_seq": 0,
1221             "truncate_size": 0,
1222             "data_digest": "0x2b63260d",
1223             "omap_digest": "0x2eecc539",
1224             "expected_object_size": 0,
1225             "expected_write_size": 0,
1226             "alloc_hint_flags": 0,
1227             "manifest": {
1228               "type": 0
1229             },
1230             "watchers": {}
1231           },
1232           "size": 1,
1233           "errors": [],
1234           "osd": 0,
1235           "primary": false
1236         },
1237         {
1238           "object_info": {
1239             "oid": {
1240               "oid": "ROBJ9",
1241               "key": "",
1242               "snapid": -2,
1243               "hash": 537189375,
1244               "max": 0,
1245               "pool": 3,
1246               "namespace": ""
1247             },
1248             "version": "51'64",
1249             "prior_version": "37'27",
1250             "last_reqid": "osd.1.0:63",
1251             "user_version": 27,
1252             "size": 7,
1253             "mtime": "2018-04-05 14:33:25.352485",
1254             "local_mtime": "2018-04-05 14:33:25.353746",
1255             "lost": 0,
1256             "flags": [
1257               "dirty",
1258               "omap",
1259               "data_digest",
1260               "omap_digest"
1261             ],
1262             "truncate_seq": 0,
1263             "truncate_size": 0,
1264             "data_digest": "0x2ddbf8f5",
1265             "omap_digest": "0x2eecc539",
1266             "expected_object_size": 0,
1267             "expected_write_size": 0,
1268             "alloc_hint_flags": 0,
1269             "manifest": {
1270               "type": 0
1271             },
1272             "watchers": {}
1273           },
1274           "size": 1,
1275           "errors": [
1276             "obj_size_info_mismatch"
1277           ],
1278           "osd": 1,
1279           "primary": true
1280         }
1281       ],
1282       "selected_object_info": {
1283         "oid": {
1284           "oid": "ROBJ9",
1285           "key": "",
1286           "snapid": -2,
1287           "hash": 537189375,
1288           "max": 0,
1289           "pool": 3,
1290           "namespace": ""
1291         },
1292         "version": "95'67",
1293         "prior_version": "51'64",
1294         "last_reqid": "client.4649.0:1",
1295         "user_version": 67,
1296         "size": 1,
1297         "mtime": "",
1298         "local_mtime": "",
1299         "lost": 0,
1300         "flags": [
1301           "dirty",
1302           "omap",
1303           "data_digest",
1304           "omap_digest"
1305         ],
1306         "truncate_seq": 0,
1307         "truncate_size": 0,
1308         "data_digest": "0x2b63260d",
1309         "omap_digest": "0x2eecc539",
1310         "expected_object_size": 0,
1311         "expected_write_size": 0,
1312         "alloc_hint_flags": 0,
1313         "manifest": {
1314           "type": 0
1315         },
1316         "watchers": {}
1317       },
1318       "union_shard_errors": [
1319          "obj_size_info_mismatch"
1320       ],
1321       "errors": [
1322         "object_info_inconsistency"
1323       ],
1324       "object": {
1325         "version": 67,
1326         "snap": "head",
1327         "locator": "",
1328         "nspace": "",
1329         "name": "ROBJ9"
1330       }
1331     }
1332   ],
1333   "epoch": 0
1334 }
1335 EOF
1336
1337     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
1338     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
1339     if test $getjson = "yes"
1340     then
1341         jq '.' $dir/json > save1.json
1342     fi
1343
1344     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
1345     then
1346       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
1347     fi
1348
1349     objname=ROBJ9
1350     # Change data and size again because digest was recomputed
1351     echo -n ZZZ > $dir/change
1352     rados --pool $poolname put $objname $dir/change
1353     # Set one to an even older value
1354     objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi
1355     rm $dir/oi $dir/change
1356
1357     objname=ROBJ10
1358     objectstore_tool $dir 1 $objname get-attr _ > $dir/oi
1359     rados --pool $poolname setomapval $objname key2-$objname val2-$objname
1360     objectstore_tool $dir 0 $objname set-attr _ $dir/oi
1361     objectstore_tool $dir 1 $objname set-attr _ $dir/oi
1362     rm $dir/oi
1363
1364     inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1365     inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1366     inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1367     inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1368     pg_deep_scrub $pg
1369
1370     rados list-inconsistent-pg $poolname > $dir/json || return 1
1371     # Check pg count
1372     test $(jq '. | length' $dir/json) = "1" || return 1
1373     # Check pgid
1374     test $(jq -r '.[0]' $dir/json) = $pg || return 1
1375
1376     rados list-inconsistent-obj $pg > $dir/json || return 1
1377     # Get epoch for repair-get requests
1378     epoch=$(jq .epoch $dir/json)
1379
1380     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
1381 {
1382   "inconsistents": [
1383     {
1384       "shards": [
1385         {
1386           "data_digest": "0x2ddbf8f5",
1387           "omap_digest": "0xf5fba2c6",
1388           "size": 7,
1389           "errors": [],
1390           "osd": 0,
1391           "primary": false
1392         },
1393         {
1394           "object_info": {
1395             "oid": {
1396               "oid": "ROBJ1",
1397               "key": "",
1398               "snapid": -2,
1399               "hash": 1454963827,
1400               "max": 0,
1401               "pool": 3,
1402               "namespace": ""
1403             },
1404             "version": "51'58",
1405             "prior_version": "21'3",
1406             "last_reqid": "osd.1.0:57",
1407             "user_version": 3,
1408             "size": 7,
1409             "mtime": "2018-04-05 14:33:19.804040",
1410             "local_mtime": "2018-04-05 14:33:19.804839",
1411             "lost": 0,
1412             "flags": [
1413               "dirty",
1414               "omap",
1415               "data_digest",
1416               "omap_digest"
1417             ],
1418             "truncate_seq": 0,
1419             "truncate_size": 0,
1420             "data_digest": "0x2ddbf8f5",
1421             "omap_digest": "0xf5fba2c6",
1422             "expected_object_size": 0,
1423             "expected_write_size": 0,
1424             "alloc_hint_flags": 0,
1425             "manifest": {
1426               "type": 0
1427             },
1428             "watchers": {}
1429           },
1430           "data_digest": "0x2d4a11c2",
1431           "omap_digest": "0xf5fba2c6",
1432           "size": 9,
1433           "errors": [
1434             "data_digest_mismatch_info",
1435             "size_mismatch_info",
1436             "obj_size_info_mismatch"
1437           ],
1438           "osd": 1,
1439           "primary": true
1440         }
1441       ],
1442       "selected_object_info": {
1443         "oid": {
1444           "oid": "ROBJ1",
1445           "key": "",
1446           "snapid": -2,
1447           "hash": 1454963827,
1448           "max": 0,
1449           "pool": 3,
1450           "namespace": ""
1451         },
1452         "version": "51'58",
1453         "prior_version": "21'3",
1454         "last_reqid": "osd.1.0:57",
1455         "user_version": 3,
1456         "size": 7,
1457         "mtime": "2018-04-05 14:33:19.804040",
1458         "local_mtime": "2018-04-05 14:33:19.804839",
1459         "lost": 0,
1460         "flags": [
1461           "dirty",
1462           "omap",
1463           "data_digest",
1464           "omap_digest"
1465         ],
1466         "truncate_seq": 0,
1467         "truncate_size": 0,
1468         "data_digest": "0x2ddbf8f5",
1469         "omap_digest": "0xf5fba2c6",
1470         "expected_object_size": 0,
1471         "expected_write_size": 0,
1472         "alloc_hint_flags": 0,
1473         "manifest": {
1474           "type": 0
1475         },
1476         "watchers": {}
1477       },
1478       "union_shard_errors": [
1479         "data_digest_mismatch_info",
1480         "size_mismatch_info",
1481         "obj_size_info_mismatch"
1482       ],
1483       "errors": [
1484         "data_digest_mismatch",
1485         "size_mismatch"
1486       ],
1487       "object": {
1488         "version": 3,
1489         "snap": "head",
1490         "locator": "",
1491         "nspace": "",
1492         "name": "ROBJ1"
1493       }
1494     },
1495     {
1496       "shards": [
1497         {
1498           "data_digest": "0x2ddbf8f5",
1499           "omap_digest": "0xa8dd5adc",
1500           "size": 7,
1501           "errors": [
1502             "omap_digest_mismatch_info"
1503           ],
1504           "osd": 0,
1505           "primary": false
1506         },
1507         {
1508           "data_digest": "0x2ddbf8f5",
1509           "omap_digest": "0xa8dd5adc",
1510           "size": 7,
1511           "errors": [
1512             "omap_digest_mismatch_info"
1513           ],
1514           "osd": 1,
1515           "primary": true
1516         }
1517       ],
1518      "selected_object_info": {
1519        "alloc_hint_flags": 0,
1520        "data_digest": "0x2ddbf8f5",
1521        "expected_object_size": 0,
1522        "expected_write_size": 0,
1523        "flags": [
1524          "dirty",
1525          "omap",
1526          "data_digest",
1527          "omap_digest"
1528        ],
1529        "last_reqid": "osd.0.0:50",
1530        "local_mtime": "2018-04-05 14:33:26.762368",
1531        "lost": 0,
1532       "manifest": {
1533          "type": 0
1534        },
1535        "mtime": "2018-04-05 14:33:26.762368",
1536        "oid": {
1537          "hash": 3174666125,
1538          "key": "",
1539          "max": 0,
1540          "namespace": "",
1541          "oid": "ROBJ10",
1542          "pool": 3,
1543          "snapid": -2
1544        },
1545        "omap_digest": "0xc2025a24",
1546        "prior_version": "41'33",
1547        "size": 7,
1548        "truncate_seq": 0,
1549        "truncate_size": 0,
1550        "user_version": 30,
1551        "version": "47'51",
1552        "watchers": {}
1553       },
1554       "union_shard_errors": [
1555         "omap_digest_mismatch_info"
1556       ],
1557       "errors": [],
1558       "object": {
1559         "version": 30,
1560         "snap": "head",
1561         "locator": "",
1562         "nspace": "",
1563         "name": "ROBJ10"
1564       }
1565     },
1566     {
1567       "shards": [
1568         {
1569           "data_digest": "0x2ddbf8f5",
1570           "omap_digest": "0xa03cef03",
1571           "size": 7,
1572           "errors": [],
1573           "osd": 0,
1574           "primary": false
1575         },
1576         {
1577           "size": 7,
1578           "errors": [
1579             "read_error"
1580           ],
1581           "osd": 1,
1582           "primary": true
1583         }
1584       ],
1585       "selected_object_info": {
1586         "oid": {
1587           "oid": "ROBJ11",
1588           "key": "",
1589           "snapid": -2,
1590           "hash": 1828574689,
1591           "max": 0,
1592           "pool": 3,
1593           "namespace": ""
1594         },
1595         "version": "51'52",
1596         "prior_version": "41'33",
1597         "last_reqid": "osd.1.0:51",
1598         "user_version": 33,
1599         "size": 7,
1600         "mtime": "2018-04-05 14:33:26.761286",
1601         "local_mtime": "2018-04-05 14:33:26.762368",
1602         "lost": 0,
1603         "flags": [
1604           "dirty",
1605           "omap",
1606           "data_digest",
1607           "omap_digest"
1608         ],
1609         "truncate_seq": 0,
1610         "truncate_size": 0,
1611         "data_digest": "0x2ddbf8f5",
1612         "omap_digest": "0xa03cef03",
1613         "expected_object_size": 0,
1614         "expected_write_size": 0,
1615         "alloc_hint_flags": 0,
1616         "manifest": {
1617           "type": 0
1618         },
1619         "watchers": {}
1620       },
1621       "union_shard_errors": [
1622         "read_error"
1623       ],
1624       "errors": [],
1625       "object": {
1626         "version": 33,
1627         "snap": "head",
1628         "locator": "",
1629         "nspace": "",
1630         "name": "ROBJ11"
1631       }
1632     },
1633     {
1634       "shards": [
1635         {
1636           "errors": [
1637             "stat_error"
1638           ],
1639           "osd": 0,
1640           "primary": false
1641         },
1642         {
1643           "data_digest": "0x2ddbf8f5",
1644           "omap_digest": "0x067f306a",
1645           "size": 7,
1646           "errors": [],
1647           "osd": 1,
1648           "primary": true
1649         }
1650       ],
1651       "selected_object_info": {
1652         "oid": {
1653           "oid": "ROBJ12",
1654           "key": "",
1655           "snapid": -2,
1656           "hash": 3920199997,
1657           "max": 0,
1658           "pool": 3,
1659           "namespace": ""
1660         },
1661         "version": "51'56",
1662         "prior_version": "43'36",
1663         "last_reqid": "osd.1.0:55",
1664         "user_version": 36,
1665         "size": 7,
1666         "mtime": "2018-04-05 14:33:27.460958",
1667         "local_mtime": "2018-04-05 14:33:27.462109",
1668         "lost": 0,
1669         "flags": [
1670           "dirty",
1671           "omap",
1672           "data_digest",
1673           "omap_digest"
1674         ],
1675         "truncate_seq": 0,
1676         "truncate_size": 0,
1677         "data_digest": "0x2ddbf8f5",
1678         "omap_digest": "0x067f306a",
1679         "expected_object_size": 0,
1680         "expected_write_size": 0,
1681         "alloc_hint_flags": 0,
1682         "manifest": {
1683           "type": 0
1684         },
1685         "watchers": {}
1686       },
1687       "union_shard_errors": [
1688         "stat_error"
1689       ],
1690       "errors": [],
1691       "object": {
1692         "version": 36,
1693         "snap": "head",
1694         "locator": "",
1695         "nspace": "",
1696         "name": "ROBJ12"
1697       }
1698     },
1699     {
1700       "shards": [
1701         {
1702           "errors": [
1703             "stat_error"
1704           ],
1705           "osd": 0,
1706           "primary": false
1707         },
1708         {
1709           "size": 7,
1710           "errors": [
1711             "read_error"
1712           ],
1713           "osd": 1,
1714           "primary": true
1715         }
1716       ],
1717       "union_shard_errors": [
1718         "stat_error",
1719         "read_error"
1720       ],
1721       "errors": [],
1722       "object": {
1723         "version": 0,
1724         "snap": "head",
1725         "locator": "",
1726         "nspace": "",
1727         "name": "ROBJ13"
1728       }
1729     },
1730     {
1731       "shards": [
1732         {
1733           "object_info": "bad-val",
1734           "data_digest": "0x2ddbf8f5",
1735           "omap_digest": "0x4f14f849",
1736           "size": 7,
1737           "errors": [
1738             "info_corrupted"
1739           ],
1740           "osd": 0,
1741           "primary": false
1742         },
1743         {
1744           "data_digest": "0x2ddbf8f5",
1745           "omap_digest": "0x4f14f849",
1746           "size": 7,
1747           "errors": [
1748             "info_missing"
1749           ],
1750           "osd": 1,
1751           "primary": true
1752         }
1753       ],
1754       "union_shard_errors": [
1755         "info_missing",
1756         "info_corrupted"
1757       ],
1758       "errors": [],
1759       "object": {
1760         "version": 0,
1761         "snap": "head",
1762         "locator": "",
1763         "nspace": "",
1764         "name": "ROBJ14"
1765       }
1766     },
1767     {
1768       "shards": [
1769         {
1770           "object_info": {
1771             "oid": {
1772               "oid": "ROBJ15",
1773               "key": "",
1774               "snapid": -2,
1775               "hash": 504996876,
1776               "max": 0,
1777               "pool": 3,
1778               "namespace": ""
1779             },
1780             "version": "51'49",
1781             "prior_version": "49'45",
1782             "last_reqid": "osd.1.0:48",
1783             "user_version": 45,
1784             "size": 7,
1785             "mtime": "2018-04-05 14:33:29.498969",
1786             "local_mtime": "2018-04-05 14:33:29.499890",
1787             "lost": 0,
1788             "flags": [
1789               "dirty",
1790               "omap",
1791               "data_digest",
1792               "omap_digest"
1793             ],
1794             "truncate_seq": 0,
1795             "truncate_size": 0,
1796             "data_digest": "0x2ddbf8f5",
1797             "omap_digest": "0x2d2a4d6e",
1798             "expected_object_size": 0,
1799             "expected_write_size": 0,
1800             "alloc_hint_flags": 0,
1801             "manifest": {
1802               "type": 0
1803             },
1804             "watchers": {}
1805           },
1806           "data_digest": "0x2ddbf8f5",
1807           "omap_digest": "0x2d2a4d6e",
1808           "size": 7,
1809           "errors": [],
1810           "osd": 0,
1811           "primary": false
1812         },
1813         {
1814           "data_digest": "0x2ddbf8f5",
1815           "omap_digest": "0x2d2a4d6e",
1816           "size": 7,
1817           "errors": [
1818             "info_missing"
1819           ],
1820           "osd": 1,
1821           "primary": true
1822         }
1823       ],
1824       "selected_object_info": {
1825         "oid": {
1826           "oid": "ROBJ15",
1827           "key": "",
1828           "snapid": -2,
1829           "hash": 504996876,
1830           "max": 0,
1831           "pool": 3,
1832           "namespace": ""
1833         },
1834         "version": "51'49",
1835         "prior_version": "49'45",
1836         "last_reqid": "osd.1.0:48",
1837         "user_version": 45,
1838         "size": 7,
1839         "mtime": "2018-04-05 14:33:29.498969",
1840         "local_mtime": "2018-04-05 14:33:29.499890",
1841         "lost": 0,
1842         "flags": [
1843           "dirty",
1844           "omap",
1845           "data_digest",
1846           "omap_digest"
1847         ],
1848         "truncate_seq": 0,
1849         "truncate_size": 0,
1850         "data_digest": "0x2ddbf8f5",
1851         "omap_digest": "0x2d2a4d6e",
1852         "expected_object_size": 0,
1853         "expected_write_size": 0,
1854         "alloc_hint_flags": 0,
1855         "manifest": {
1856           "type": 0
1857         },
1858         "watchers": {}
1859       },
1860       "union_shard_errors": [
1861         "info_missing"
1862       ],
1863       "errors": [],
1864       "object": {
1865         "version": 45,
1866         "snap": "head",
1867         "locator": "",
1868         "nspace": "",
1869         "name": "ROBJ15"
1870       }
1871     },
1872     {
1873       "errors": [],
1874       "object": {
1875       "locator": "",
1876       "name": "ROBJ16",
1877       "nspace": "",
1878       "snap": "head",
1879       "version": 0
1880        },
1881         "shards": [
1882       {
1883         "data_digest": "0x2ddbf8f5",
1884         "errors": [
1885           "snapset_missing"
1886         ],
1887         "omap_digest": "0x8b699207",
1888         "osd": 0,
1889         "primary": false,
1890         "size": 7
1891       },
1892       {
1893         "snapset": "bad-val",
1894         "data_digest": "0x2ddbf8f5",
1895         "errors": [
1896           "snapset_corrupted"
1897         ],
1898         "omap_digest": "0x8b699207",
1899         "osd": 1,
1900         "primary": true,
1901         "size": 7
1902       }
1903       ],
1904       "union_shard_errors": [
1905         "snapset_missing",
1906         "snapset_corrupted"
1907       ]
1908     },
1909     {
1910       "shards": [
1911         {
1912           "data_digest": "0x578a4830",
1913           "omap_digest": "0xf8e11918",
1914           "size": 7,
1915           "errors": [
1916             "data_digest_mismatch_info"
1917           ],
1918           "osd": 0,
1919           "primary": false
1920         },
1921         {
1922           "data_digest": "0x2ddbf8f5",
1923           "omap_digest": "0xf8e11918",
1924           "size": 7,
1925           "errors": [],
1926           "osd": 1,
1927           "primary": true
1928         }
1929       ],
1930       "selected_object_info": {
1931         "oid": {
1932           "oid": "ROBJ2",
1933           "key": "",
1934           "snapid": -2,
1935           "hash": 2026323607,
1936           "max": 0,
1937           "pool": 3,
1938           "namespace": ""
1939         },
1940         "version": "51'60",
1941         "prior_version": "23'6",
1942         "last_reqid": "osd.1.0:59",
1943         "user_version": 6,
1944         "size": 7,
1945         "mtime": "2018-04-05 14:33:20.498756",
1946         "local_mtime": "2018-04-05 14:33:20.499704",
1947         "lost": 0,
1948         "flags": [
1949           "dirty",
1950           "omap",
1951           "data_digest",
1952           "omap_digest"
1953         ],
1954         "truncate_seq": 0,
1955         "truncate_size": 0,
1956         "data_digest": "0x2ddbf8f5",
1957         "omap_digest": "0xf8e11918",
1958         "expected_object_size": 0,
1959         "expected_write_size": 0,
1960         "alloc_hint_flags": 0,
1961         "manifest": {
1962           "type": 0
1963         },
1964         "watchers": {}
1965       },
1966       "union_shard_errors": [
1967         "data_digest_mismatch_info"
1968       ],
1969       "errors": [
1970         "data_digest_mismatch"
1971       ],
1972       "object": {
1973         "version": 6,
1974         "snap": "head",
1975         "locator": "",
1976         "nspace": "",
1977         "name": "ROBJ2"
1978       }
1979     },
1980     {
1981       "shards": [
1982         {
1983           "data_digest": "0x2ddbf8f5",
1984           "omap_digest": "0x00b35dfd",
1985           "size": 7,
1986           "errors": [],
1987           "osd": 0,
1988           "primary": false
1989         },
1990         {
1991           "errors": [
1992             "missing"
1993           ],
1994           "osd": 1,
1995           "primary": true
1996         }
1997       ],
1998       "selected_object_info": {
1999         "oid": {
2000           "oid": "ROBJ3",
2001           "key": "",
2002           "snapid": -2,
2003           "hash": 625845583,
2004           "max": 0,
2005           "pool": 3,
2006           "namespace": ""
2007         },
2008         "version": "51'61",
2009         "prior_version": "25'9",
2010         "last_reqid": "osd.1.0:60",
2011         "user_version": 9,
2012         "size": 7,
2013         "mtime": "2018-04-05 14:33:21.189382",
2014         "local_mtime": "2018-04-05 14:33:21.190446",
2015         "lost": 0,
2016         "flags": [
2017           "dirty",
2018           "omap",
2019           "data_digest",
2020           "omap_digest"
2021         ],
2022         "truncate_seq": 0,
2023         "truncate_size": 0,
2024         "data_digest": "0x2ddbf8f5",
2025         "omap_digest": "0x00b35dfd",
2026         "expected_object_size": 0,
2027         "expected_write_size": 0,
2028         "alloc_hint_flags": 0,
2029         "manifest": {
2030           "type": 0
2031         },
2032         "watchers": {}
2033       },
2034       "union_shard_errors": [
2035         "missing"
2036       ],
2037       "errors": [],
2038       "object": {
2039         "version": 9,
2040         "snap": "head",
2041         "locator": "",
2042         "nspace": "",
2043         "name": "ROBJ3"
2044       }
2045     },
2046     {
2047       "shards": [
2048         {
2049           "data_digest": "0x2ddbf8f5",
2050           "omap_digest": "0xd7178dfe",
2051           "size": 7,
2052           "errors": [
2053             "omap_digest_mismatch_info"
2054           ],
2055           "osd": 0,
2056           "primary": false
2057         },
2058         {
2059           "data_digest": "0x2ddbf8f5",
2060           "omap_digest": "0xe2d46ea4",
2061           "size": 7,
2062           "errors": [],
2063           "osd": 1,
2064           "primary": true
2065         }
2066       ],
2067       "selected_object_info": {
2068         "oid": {
2069           "oid": "ROBJ4",
2070           "key": "",
2071           "snapid": -2,
2072           "hash": 2360875311,
2073           "max": 0,
2074           "pool": 3,
2075           "namespace": ""
2076         },
2077         "version": "51'62",
2078         "prior_version": "27'12",
2079         "last_reqid": "osd.1.0:61",
2080         "user_version": 12,
2081         "size": 7,
2082         "mtime": "2018-04-05 14:33:21.862313",
2083         "local_mtime": "2018-04-05 14:33:21.863261",
2084         "lost": 0,
2085         "flags": [
2086           "dirty",
2087           "omap",
2088           "data_digest",
2089           "omap_digest"
2090         ],
2091         "truncate_seq": 0,
2092         "truncate_size": 0,
2093         "data_digest": "0x2ddbf8f5",
2094         "omap_digest": "0xe2d46ea4",
2095         "expected_object_size": 0,
2096         "expected_write_size": 0,
2097         "alloc_hint_flags": 0,
2098         "manifest": {
2099           "type": 0
2100         },
2101         "watchers": {}
2102       },
2103       "union_shard_errors": [
2104         "omap_digest_mismatch_info"
2105       ],
2106       "errors": [
2107         "omap_digest_mismatch"
2108       ],
2109       "object": {
2110         "version": 12,
2111         "snap": "head",
2112         "locator": "",
2113         "nspace": "",
2114         "name": "ROBJ4"
2115       }
2116     },
2117     {
2118       "shards": [
2119         {
2120           "data_digest": "0x2ddbf8f5",
2121           "omap_digest": "0x1a862a41",
2122           "size": 7,
2123           "errors": [],
2124           "osd": 0,
2125           "primary": false
2126         },
2127         {
2128           "data_digest": "0x2ddbf8f5",
2129           "omap_digest": "0x06cac8f6",
2130           "size": 7,
2131           "errors": [
2132             "omap_digest_mismatch_info"
2133           ],
2134           "osd": 1,
2135           "primary": true
2136         }
2137       ],
2138       "selected_object_info": {
2139         "oid": {
2140           "oid": "ROBJ5",
2141           "key": "",
2142           "snapid": -2,
2143           "hash": 2334915887,
2144           "max": 0,
2145           "pool": 3,
2146           "namespace": ""
2147         },
2148         "version": "51'63",
2149         "prior_version": "29'15",
2150         "last_reqid": "osd.1.0:62",
2151         "user_version": 15,
2152         "size": 7,
2153         "mtime": "2018-04-05 14:33:22.589300",
2154         "local_mtime": "2018-04-05 14:33:22.590376",
2155         "lost": 0,
2156         "flags": [
2157           "dirty",
2158           "omap",
2159           "data_digest",
2160           "omap_digest"
2161         ],
2162         "truncate_seq": 0,
2163         "truncate_size": 0,
2164         "data_digest": "0x2ddbf8f5",
2165         "omap_digest": "0x1a862a41",
2166         "expected_object_size": 0,
2167         "expected_write_size": 0,
2168         "alloc_hint_flags": 0,
2169         "manifest": {
2170           "type": 0
2171         },
2172         "watchers": {}
2173       },
2174       "union_shard_errors": [
2175         "omap_digest_mismatch_info"
2176       ],
2177       "errors": [
2178         "omap_digest_mismatch"
2179       ],
2180       "object": {
2181         "version": 15,
2182         "snap": "head",
2183         "locator": "",
2184         "nspace": "",
2185         "name": "ROBJ5"
2186       }
2187     },
2188     {
2189       "shards": [
2190         {
2191           "data_digest": "0x2ddbf8f5",
2192           "omap_digest": "0x689ee887",
2193           "size": 7,
2194           "errors": [
2195             "omap_digest_mismatch_info"
2196           ],
2197           "osd": 0,
2198           "primary": false
2199         },
2200         {
2201           "data_digest": "0x2ddbf8f5",
2202           "omap_digest": "0x179c919f",
2203           "size": 7,
2204           "errors": [],
2205           "osd": 1,
2206           "primary": true
2207         }
2208       ],
2209       "selected_object_info": {
2210         "oid": {
2211           "oid": "ROBJ6",
2212           "key": "",
2213           "snapid": -2,
2214           "hash": 390610085,
2215           "max": 0,
2216           "pool": 3,
2217           "namespace": ""
2218         },
2219         "version": "51'54",
2220         "prior_version": "31'18",
2221         "last_reqid": "osd.1.0:53",
2222         "user_version": 18,
2223         "size": 7,
2224         "mtime": "2018-04-05 14:33:23.289188",
2225         "local_mtime": "2018-04-05 14:33:23.290130",
2226         "lost": 0,
2227         "flags": [
2228           "dirty",
2229           "omap",
2230           "data_digest",
2231           "omap_digest"
2232         ],
2233         "truncate_seq": 0,
2234         "truncate_size": 0,
2235         "data_digest": "0x2ddbf8f5",
2236         "omap_digest": "0x179c919f",
2237         "expected_object_size": 0,
2238         "expected_write_size": 0,
2239         "alloc_hint_flags": 0,
2240         "manifest": {
2241           "type": 0
2242         },
2243         "watchers": {}
2244       },
2245       "union_shard_errors": [
2246         "omap_digest_mismatch_info"
2247       ],
2248       "errors": [
2249         "omap_digest_mismatch"
2250       ],
2251       "object": {
2252         "version": 18,
2253         "snap": "head",
2254         "locator": "",
2255         "nspace": "",
2256         "name": "ROBJ6"
2257       }
2258     },
2259     {
2260       "shards": [
2261         {
2262           "data_digest": "0x2ddbf8f5",
2263           "omap_digest": "0xefced57a",
2264           "size": 7,
2265           "errors": [],
2266           "osd": 0,
2267           "primary": false
2268         },
2269         {
2270           "data_digest": "0x2ddbf8f5",
2271           "omap_digest": "0x6a73cc07",
2272           "size": 7,
2273           "errors": [
2274             "omap_digest_mismatch_info"
2275           ],
2276           "osd": 1,
2277           "primary": true
2278         }
2279       ],
2280       "selected_object_info": {
2281         "oid": {
2282           "oid": "ROBJ7",
2283           "key": "",
2284           "snapid": -2,
2285           "hash": 3529485009,
2286           "max": 0,
2287           "pool": 3,
2288           "namespace": ""
2289         },
2290         "version": "51'53",
2291         "prior_version": "33'21",
2292         "last_reqid": "osd.1.0:52",
2293         "user_version": 21,
2294         "size": 7,
2295         "mtime": "2018-04-05 14:33:23.979658",
2296         "local_mtime": "2018-04-05 14:33:23.980731",
2297         "lost": 0,
2298         "flags": [
2299           "dirty",
2300           "omap",
2301           "data_digest",
2302           "omap_digest"
2303         ],
2304         "truncate_seq": 0,
2305         "truncate_size": 0,
2306         "data_digest": "0x2ddbf8f5",
2307         "omap_digest": "0xefced57a",
2308         "expected_object_size": 0,
2309         "expected_write_size": 0,
2310         "alloc_hint_flags": 0,
2311         "manifest": {
2312           "type": 0
2313         },
2314         "watchers": {}
2315       },
2316       "union_shard_errors": [
2317         "omap_digest_mismatch_info"
2318       ],
2319       "errors": [
2320         "omap_digest_mismatch"
2321       ],
2322       "object": {
2323         "version": 21,
2324         "snap": "head",
2325         "locator": "",
2326         "nspace": "",
2327         "name": "ROBJ7"
2328       }
2329     },
2330     {
2331       "shards": [
2332         {
2333           "attrs": [
2334             {
2335               "Base64": false,
2336               "value": "bad-val",
2337               "name": "key1-ROBJ8"
2338             },
2339             {
2340               "Base64": false,
2341               "value": "val2-ROBJ8",
2342               "name": "key2-ROBJ8"
2343             }
2344           ],
2345           "data_digest": "0x2ddbf8f5",
2346           "omap_digest": "0xd6be81dc",
2347           "size": 7,
2348           "errors": [],
2349           "osd": 0,
2350           "primary": false
2351         },
2352         {
2353           "attrs": [
2354             {
2355               "Base64": false,
2356               "value": "val1-ROBJ8",
2357               "name": "key1-ROBJ8"
2358             },
2359             {
2360               "Base64": false,
2361               "value": "val3-ROBJ8",
2362               "name": "key3-ROBJ8"
2363             }
2364           ],
2365           "data_digest": "0x2ddbf8f5",
2366           "omap_digest": "0xd6be81dc",
2367           "size": 7,
2368           "errors": [],
2369           "osd": 1,
2370           "primary": true
2371         }
2372       ],
2373       "selected_object_info": {
2374         "oid": {
2375           "oid": "ROBJ8",
2376           "key": "",
2377           "snapid": -2,
2378           "hash": 2359695969,
2379           "max": 0,
2380           "pool": 3,
2381           "namespace": ""
2382         },
2383         "version": "79'66",
2384         "prior_version": "79'65",
2385         "last_reqid": "client.4554.0:1",
2386         "user_version": 66,
2387         "size": 7,
2388         "mtime": "2018-04-05 14:34:05.598688",
2389         "local_mtime": "2018-04-05 14:34:05.599698",
2390         "lost": 0,
2391         "flags": [
2392           "dirty",
2393           "omap",
2394           "data_digest",
2395           "omap_digest"
2396         ],
2397         "truncate_seq": 0,
2398         "truncate_size": 0,
2399         "data_digest": "0x2ddbf8f5",
2400         "omap_digest": "0xd6be81dc",
2401         "expected_object_size": 0,
2402         "expected_write_size": 0,
2403         "alloc_hint_flags": 0,
2404         "manifest": {
2405           "type": 0
2406         },
2407         "watchers": {}
2408       },
2409       "union_shard_errors": [],
2410       "errors": [
2411         "attr_value_mismatch",
2412         "attr_name_mismatch"
2413       ],
2414       "object": {
2415         "version": 66,
2416         "snap": "head",
2417         "locator": "",
2418         "nspace": "",
2419         "name": "ROBJ8"
2420       }
2421     },
2422     {
2423       "shards": [
2424         {
2425           "object_info": {
2426             "oid": {
2427               "oid": "ROBJ9",
2428               "key": "",
2429               "snapid": -2,
2430               "hash": 537189375,
2431               "max": 0,
2432               "pool": 3,
2433               "namespace": ""
2434             },
2435             "version": "51'64",
2436             "prior_version": "37'27",
2437             "last_reqid": "osd.1.0:63",
2438             "user_version": 27,
2439             "size": 7,
2440             "mtime": "2018-04-05 14:33:25.352485",
2441             "local_mtime": "2018-04-05 14:33:25.353746",
2442             "lost": 0,
2443             "flags": [
2444               "dirty",
2445               "omap",
2446               "data_digest",
2447               "omap_digest"
2448             ],
2449             "truncate_seq": 0,
2450             "truncate_size": 0,
2451             "data_digest": "0x2ddbf8f5",
2452            "omap_digest": "0x2eecc539",
2453             "expected_object_size": 0,
2454             "expected_write_size": 0,
2455             "alloc_hint_flags": 0,
2456             "manifest": {
2457               "type": 0
2458             },
2459             "watchers": {}
2460           },
2461           "data_digest": "0x1f26fb26",
2462           "omap_digest": "0x2eecc539",
2463           "size": 3,
2464           "errors": [
2465             "obj_size_info_mismatch"
2466           ],
2467           "osd": 0,
2468           "primary": false
2469         },
2470         {
2471           "object_info": {
2472             "oid": {
2473               "oid": "ROBJ9",
2474               "key": "",
2475               "snapid": -2,
2476               "hash": 537189375,
2477               "max": 0,
2478               "pool": 3,
2479               "namespace": ""
2480             },
2481             "version": "119'68",
2482             "prior_version": "51'64",
2483             "last_reqid": "client.4834.0:1",
2484             "user_version": 68,
2485             "size": 3,
2486             "mtime": "2018-04-05 14:35:01.500659",
2487             "local_mtime": "2018-04-05 14:35:01.502117",
2488             "lost": 0,
2489             "flags": [
2490               "dirty",
2491               "omap",
2492               "data_digest",
2493               "omap_digest"
2494             ],
2495             "truncate_seq": 0,
2496             "truncate_size": 0,
2497             "data_digest": "0x1f26fb26",
2498             "omap_digest": "0x2eecc539",
2499             "expected_object_size": 0,
2500             "expected_write_size": 0,
2501             "alloc_hint_flags": 0,
2502             "manifest": {
2503               "type": 0
2504             },
2505             "watchers": {}
2506           },
2507           "data_digest": "0x1f26fb26",
2508           "omap_digest": "0x2eecc539",
2509           "size": 3,
2510           "errors": [],
2511           "osd": 1,
2512           "primary": true
2513         }
2514       ],
2515       "selected_object_info": {
2516         "oid": {
2517           "oid": "ROBJ9",
2518           "key": "",
2519           "snapid": -2,
2520           "hash": 537189375,
2521           "max": 0,
2522           "pool": 3,
2523           "namespace": ""
2524         },
2525         "version": "119'68",
2526         "prior_version": "51'64",
2527         "last_reqid": "client.4834.0:1",
2528         "user_version": 68,
2529         "size": 3,
2530         "mtime": "2018-04-05 14:35:01.500659",
2531         "local_mtime": "2018-04-05 14:35:01.502117",
2532         "lost": 0,
2533         "flags": [
2534           "dirty",
2535           "omap",
2536           "data_digest",
2537           "omap_digest"
2538         ],
2539         "truncate_seq": 0,
2540         "truncate_size": 0,
2541         "data_digest": "0x1f26fb26",
2542         "omap_digest": "0x2eecc539",
2543         "expected_object_size": 0,
2544         "expected_write_size": 0,
2545         "alloc_hint_flags": 0,
2546         "manifest": {
2547           "type": 0
2548         },
2549         "watchers": {}
2550       },
2551       "union_shard_errors": [
2552         "obj_size_info_mismatch"
2553       ],
2554       "errors": [
2555         "object_info_inconsistency"
2556       ],
2557       "object": {
2558         "version": 68,
2559         "snap": "head",
2560         "locator": "",
2561         "nspace": "",
2562         "name": "ROBJ9"
2563       }
2564     }
2565   ],
2566   "epoch": 0
2567 }
2568 EOF
2569
2570     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
2571     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
2572     if test $getjson = "yes"
2573     then
2574         jq '.' $dir/json > save2.json
2575     fi
2576
2577     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
2578     then
2579       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
2580     fi
2581
2582     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
2583     teardown $dir || return 1
2584 }
2585
2586
2587 #
2588 # Test scrub errors for an erasure coded pool
2589 #
2590 function corrupt_scrub_erasure() {
2591     local dir=$1
2592     local allow_overwrites=$2
2593     local poolname=ecpool
2594     local total_objs=7
2595
2596     setup $dir || return 1
2597     run_mon $dir a || return 1
2598     run_mgr $dir x || return 1
2599     for id in $(seq 0 2) ; do
2600         if [ "$allow_overwrites" = "true" ]; then
2601             run_osd_bluestore $dir $id || return 1
2602         else
2603             run_osd $dir $id || return 1
2604         fi
2605     done
2606     create_rbd_pool || return 1
2607     create_pool foo 1
2608
2609     create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1
2610     wait_for_clean || return 1
2611
2612     for i in $(seq 1 $total_objs) ; do
2613         objname=EOBJ${i}
2614         add_something $dir $poolname $objname || return 1
2615
2616         local osd=$(expr $i % 2)
2617
2618         case $i in
2619         1)
2620             # Size (deep scrub data_digest too)
2621             local payload=UVWXYZZZ
2622             echo $payload > $dir/CORRUPT
2623             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2624             ;;
2625
2626         2)
2627             # Corrupt EC shard
2628             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1
2629             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2630             ;;
2631
2632         3)
2633              # missing
2634              objectstore_tool $dir $osd $objname remove || return 1
2635              ;;
2636
2637         4)
2638             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
2639             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
2640
2641             # Break xattrs
2642             echo -n bad-val > $dir/bad-val
2643             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
2644             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
2645             echo -n val3-$objname > $dir/newval
2646             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
2647             rm $dir/bad-val $dir/newval
2648             ;;
2649
2650         5)
2651             # Corrupt EC shard
2652             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2
2653             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2654             ;;
2655
2656         6)
2657             objectstore_tool $dir 0 $objname rm-attr hinfo_key || return 1
2658             echo -n bad-val > $dir/bad-val
2659             objectstore_tool $dir 1 $objname set-attr hinfo_key $dir/bad-val || return 1
2660             ;;
2661
2662         7)
2663             local payload=MAKETHISDIFFERENTFROMOTHEROBJECTS
2664             echo $payload > $dir/DIFFERENT
2665             rados --pool $poolname put $objname $dir/DIFFERENT || return 1
2666
2667             # Get hinfo_key from EOBJ1
2668             objectstore_tool $dir 0 EOBJ1 get-attr hinfo_key > $dir/hinfo
2669             objectstore_tool $dir 0 $objname set-attr hinfo_key $dir/hinfo || return 1
2670             rm -f $dir/hinfo
2671             ;;
2672
2673         esac
2674     done
2675
2676     local pg=$(get_pg $poolname EOBJ0)
2677
2678     pg_scrub $pg
2679
2680     rados list-inconsistent-pg $poolname > $dir/json || return 1
2681     # Check pg count
2682     test $(jq '. | length' $dir/json) = "1" || return 1
2683     # Check pgid
2684     test $(jq -r '.[0]' $dir/json) = $pg || return 1
2685
2686     rados list-inconsistent-obj $pg > $dir/json || return 1
2687     # Get epoch for repair-get requests
2688     epoch=$(jq .epoch $dir/json)
2689
2690     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
2691 {
2692   "inconsistents": [
2693     {
2694       "shards": [
2695         {
2696           "size": 2048,
2697           "errors": [],
2698           "shard": 2,
2699           "osd": 0,
2700           "primary": false
2701         },
2702         {
2703           "object_info": {
2704             "oid": {
2705               "oid": "EOBJ1",
2706               "key": "",
2707               "snapid": -2,
2708               "hash": 560836233,
2709               "max": 0,
2710               "pool": 3,
2711               "namespace": ""
2712             },
2713             "version": "27'1",
2714             "prior_version": "0'0",
2715             "last_reqid": "client.4184.0:1",
2716             "user_version": 1,
2717             "size": 7,
2718             "mtime": "",
2719             "local_mtime": "",
2720             "lost": 0,
2721             "flags": [
2722               "dirty",
2723               "data_digest"
2724             ],
2725             "truncate_seq": 0,
2726             "truncate_size": 0,
2727             "data_digest": "0x2ddbf8f5",
2728             "omap_digest": "0xffffffff",
2729             "expected_object_size": 0,
2730             "expected_write_size": 0,
2731             "alloc_hint_flags": 0,
2732             "manifest": {
2733               "type": 0
2734             },
2735             "watchers": {}
2736           },
2737           "size": 9,
2738           "shard": 0,
2739           "errors": [
2740             "size_mismatch_info",
2741             "obj_size_info_mismatch"
2742           ],
2743           "osd": 1,
2744           "primary": true
2745         },
2746         {
2747           "size": 2048,
2748           "shard": 1,
2749           "errors": [],
2750           "osd": 2,
2751           "primary": false
2752         }
2753       ],
2754       "selected_object_info": {
2755         "oid": {
2756           "oid": "EOBJ1",
2757           "key": "",
2758           "snapid": -2,
2759           "hash": 560836233,
2760           "max": 0,
2761           "pool": 3,
2762           "namespace": ""
2763         },
2764         "version": "27'1",
2765         "prior_version": "0'0",
2766         "last_reqid": "client.4184.0:1",
2767         "user_version": 1,
2768         "size": 7,
2769         "mtime": "",
2770         "local_mtime": "",
2771         "lost": 0,
2772         "flags": [
2773           "dirty",
2774           "data_digest"
2775         ],
2776         "truncate_seq": 0,
2777         "truncate_size": 0,
2778         "data_digest": "0x2ddbf8f5",
2779         "omap_digest": "0xffffffff",
2780         "expected_object_size": 0,
2781         "expected_write_size": 0,
2782         "alloc_hint_flags": 0,
2783         "manifest": {
2784           "type": 0
2785         },
2786         "watchers": {}
2787       },
2788       "union_shard_errors": [
2789         "size_mismatch_info",
2790         "obj_size_info_mismatch"
2791       ],
2792       "errors": [
2793         "size_mismatch"
2794       ],
2795       "object": {
2796         "version": 1,
2797         "snap": "head",
2798         "locator": "",
2799         "nspace": "",
2800         "name": "EOBJ1"
2801       }
2802     },
2803     {
2804       "shards": [
2805         {
2806           "size": 2048,
2807           "errors": [],
2808           "shard": 2,
2809           "osd": 0,
2810           "primary": false
2811         },
2812         {
2813           "shard": 0,
2814           "errors": [
2815             "missing"
2816           ],
2817           "osd": 1,
2818           "primary": true
2819         },
2820         {
2821           "size": 2048,
2822           "shard": 1,
2823           "errors": [],
2824           "osd": 2,
2825           "primary": false
2826         }
2827       ],
2828       "selected_object_info": {
2829         "oid": {
2830           "oid": "EOBJ3",
2831           "key": "",
2832           "snapid": -2,
2833           "hash": 3125668237,
2834           "max": 0,
2835           "pool": 3,
2836           "namespace": ""
2837         },
2838         "version": "39'3",
2839         "prior_version": "0'0",
2840         "last_reqid": "client.4252.0:1",
2841         "user_version": 3,
2842         "size": 7,
2843         "mtime": "",
2844         "local_mtime": "",
2845         "lost": 0,
2846         "flags": [
2847           "dirty",
2848           "data_digest"
2849         ],
2850         "truncate_seq": 0,
2851         "truncate_size": 0,
2852         "data_digest": "0x2ddbf8f5",
2853         "omap_digest": "0xffffffff",
2854         "expected_object_size": 0,
2855         "expected_write_size": 0,
2856         "alloc_hint_flags": 0,
2857         "manifest": {
2858           "type": 0
2859         },
2860         "watchers": {}
2861       },
2862       "union_shard_errors": [
2863         "missing"
2864       ],
2865       "errors": [],
2866       "object": {
2867         "version": 3,
2868         "snap": "head",
2869         "locator": "",
2870         "nspace": "",
2871         "name": "EOBJ3"
2872       }
2873     },
2874     {
2875       "shards": [
2876         {
2877           "attrs": [
2878             {
2879               "Base64": false,
2880               "value": "bad-val",
2881               "name": "key1-EOBJ4"
2882             },
2883             {
2884               "Base64": false,
2885               "value": "val2-EOBJ4",
2886               "name": "key2-EOBJ4"
2887             }
2888           ],
2889           "size": 2048,
2890           "errors": [],
2891           "shard": 2,
2892           "osd": 0,
2893           "primary": false
2894         },
2895         {
2896           "osd": 1,
2897           "primary": true,
2898           "shard": 0,
2899           "errors": [],
2900           "size": 2048,
2901           "attrs": [
2902             {
2903               "Base64": false,
2904               "value": "val1-EOBJ4",
2905               "name": "key1-EOBJ4"
2906             },
2907             {
2908               "Base64": false,
2909               "value": "val2-EOBJ4",
2910               "name": "key2-EOBJ4"
2911             }
2912           ]
2913         },
2914         {
2915           "osd": 2,
2916           "primary": false,
2917           "shard": 1,
2918           "errors": [],
2919           "size": 2048,
2920           "attrs": [
2921             {
2922               "Base64": false,
2923               "value": "val1-EOBJ4",
2924               "name": "key1-EOBJ4"
2925             },
2926             {
2927               "Base64": false,
2928               "value": "val3-EOBJ4",
2929               "name": "key3-EOBJ4"
2930             }
2931           ]
2932         }
2933       ],
2934       "selected_object_info": {
2935         "oid": {
2936           "oid": "EOBJ4",
2937           "key": "",
2938           "snapid": -2,
2939           "hash": 1618759290,
2940           "max": 0,
2941           "pool": 3,
2942           "namespace": ""
2943         },
2944         "version": "45'6",
2945         "prior_version": "45'5",
2946         "last_reqid": "client.4294.0:1",
2947         "user_version": 6,
2948         "size": 7,
2949         "mtime": "",
2950         "local_mtime": "",
2951         "lost": 0,
2952         "flags": [
2953           "dirty",
2954           "data_digest"
2955         ],
2956         "truncate_seq": 0,
2957         "truncate_size": 0,
2958         "data_digest": "0x2ddbf8f5",
2959         "omap_digest": "0xffffffff",
2960         "expected_object_size": 0,
2961         "expected_write_size": 0,
2962         "alloc_hint_flags": 0,
2963         "manifest": {
2964           "type": 0
2965         },
2966         "watchers": {}
2967       },
2968       "union_shard_errors": [],
2969       "errors": [
2970         "attr_value_mismatch",
2971         "attr_name_mismatch"
2972       ],
2973       "object": {
2974         "version": 6,
2975         "snap": "head",
2976         "locator": "",
2977         "nspace": "",
2978         "name": "EOBJ4"
2979       }
2980     },
2981     {
2982       "shards": [
2983         {
2984           "size": 2048,
2985           "errors": [],
2986           "shard": 2,
2987           "osd": 0,
2988           "primary": false
2989         },
2990         {
2991           "object_info": {
2992             "oid": {
2993               "oid": "EOBJ5",
2994               "key": "",
2995               "snapid": -2,
2996               "hash": 2918945441,
2997               "max": 0,
2998               "pool": 3,
2999               "namespace": ""
3000             },
3001             "version": "59'7",
3002             "prior_version": "0'0",
3003             "last_reqid": "client.4382.0:1",
3004             "user_version": 7,
3005             "size": 7,
3006             "mtime": "",
3007             "local_mtime": "",
3008             "lost": 0,
3009             "flags": [
3010               "dirty",
3011               "data_digest"
3012             ],
3013             "truncate_seq": 0,
3014             "truncate_size": 0,
3015             "data_digest": "0x2ddbf8f5",
3016             "omap_digest": "0xffffffff",
3017             "expected_object_size": 0,
3018             "expected_write_size": 0,
3019             "alloc_hint_flags": 0,
3020             "manifest": {
3021               "type": 0
3022             },
3023             "watchers": {}
3024           },
3025           "size": 4096,
3026           "shard": 0,
3027           "errors": [
3028             "size_mismatch_info",
3029             "obj_size_info_mismatch"
3030           ],
3031           "osd": 1,
3032           "primary": true
3033         },
3034         {
3035           "size": 2048,
3036           "shard": 1,
3037           "errors": [],
3038           "osd": 2,
3039           "primary": false
3040         }
3041       ],
3042       "selected_object_info": {
3043         "oid": {
3044           "oid": "EOBJ5",
3045           "key": "",
3046           "snapid": -2,
3047           "hash": 2918945441,
3048           "max": 0,
3049           "pool": 3,
3050           "namespace": ""
3051         },
3052         "version": "59'7",
3053         "prior_version": "0'0",
3054         "last_reqid": "client.4382.0:1",
3055         "user_version": 7,
3056         "size": 7,
3057         "mtime": "",
3058         "local_mtime": "",
3059         "lost": 0,
3060         "flags": [
3061           "dirty",
3062           "data_digest"
3063         ],
3064         "truncate_seq": 0,
3065         "truncate_size": 0,
3066         "data_digest": "0x2ddbf8f5",
3067         "omap_digest": "0xffffffff",
3068         "expected_object_size": 0,
3069         "expected_write_size": 0,
3070         "alloc_hint_flags": 0,
3071         "manifest": {
3072           "type": 0
3073         },
3074         "watchers": {}
3075       },
3076       "union_shard_errors": [
3077         "size_mismatch_info",
3078         "obj_size_info_mismatch"
3079       ],
3080       "errors": [
3081         "size_mismatch"
3082       ],
3083       "object": {
3084         "version": 7,
3085         "snap": "head",
3086         "locator": "",
3087         "nspace": "",
3088         "name": "EOBJ5"
3089       }
3090    },
3091    {
3092      "errors": [],
3093      "object": {
3094        "locator": "",
3095        "name": "EOBJ6",
3096        "nspace": "",
3097        "snap": "head",
3098        "version": 8
3099      },
3100      "selected_object_info": {
3101         "oid": {
3102           "oid": "EOBJ6",
3103           "key": "",
3104           "snapid": -2,
3105           "hash": 3050890866,
3106           "max": 0,
3107           "pool": 3,
3108           "namespace": ""
3109         },
3110         "version": "65'8",
3111         "prior_version": "0'0",
3112         "last_reqid": "client.4418.0:1",
3113         "user_version": 8,
3114         "size": 7,
3115         "mtime": "",
3116         "local_mtime": "",
3117         "lost": 0,
3118         "flags": [
3119           "dirty",
3120           "data_digest"
3121         ],
3122         "truncate_seq": 0,
3123         "truncate_size": 0,
3124         "data_digest": "0x2ddbf8f5",
3125         "omap_digest": "0xffffffff",
3126         "expected_object_size": 0,
3127         "expected_write_size": 0,
3128         "alloc_hint_flags": 0,
3129         "manifest": {
3130           "type": 0
3131         },
3132         "watchers": {}
3133      },
3134      "shards": [
3135        {
3136          "errors": [
3137            "hinfo_missing"
3138          ],
3139          "osd": 0,
3140          "primary": false,
3141          "shard": 2,
3142          "size": 2048
3143        },
3144        {
3145          "errors": [
3146            "hinfo_corrupted"
3147          ],
3148          "osd": 1,
3149          "primary": true,
3150          "shard": 0,
3151          "hashinfo": "bad-val",
3152          "size": 2048
3153        },
3154        {
3155          "errors": [],
3156          "osd": 2,
3157          "primary": false,
3158          "shard": 1,
3159          "size": 2048,
3160          "hashinfo": {
3161            "cumulative_shard_hashes": [
3162             {
3163               "hash": 80717615,
3164               "shard": 0
3165             },
3166             {
3167               "hash": 1534491824,
3168               "shard": 1
3169             },
3170             {
3171               "hash": 80717615,
3172               "shard": 2
3173             }
3174            ],
3175            "total_chunk_size": 2048
3176          }
3177        }
3178      ],
3179      "union_shard_errors": [
3180        "hinfo_missing",
3181        "hinfo_corrupted"
3182      ]
3183    },
3184    {
3185      "errors": [
3186        "hinfo_inconsistency"
3187      ],
3188      "object": {
3189        "locator": "",
3190        "name": "EOBJ7",
3191        "nspace": "",
3192        "snap": "head",
3193        "version": 10
3194      },
3195      "selected_object_info": {
3196         "oid": {
3197           "oid": "EOBJ7",
3198           "key": "",
3199           "snapid": -2,
3200           "hash": 3258066308,
3201           "max": 0,
3202           "pool": 3,
3203           "namespace": ""
3204         },
3205         "version": "75'10",
3206         "prior_version": "75'9",
3207         "last_reqid": "client.4482.0:1",
3208         "user_version": 10,
3209         "size": 34,
3210         "mtime": "",
3211         "local_mtime": "",
3212         "lost": 0,
3213         "flags": [
3214           "dirty",
3215           "data_digest"
3216         ],
3217         "truncate_seq": 0,
3218         "truncate_size": 0,
3219         "data_digest": "0x136e4e27",
3220         "omap_digest": "0xffffffff",
3221         "expected_object_size": 0,
3222         "expected_write_size": 0,
3223         "alloc_hint_flags": 0,
3224         "manifest": {
3225           "type": 0
3226         },
3227         "watchers": {}
3228      },
3229      "shards": [
3230        {
3231          "hashinfo": {
3232            "cumulative_shard_hashes": [
3233             {
3234               "hash": 80717615,
3235               "shard": 0
3236             },
3237             {
3238               "hash": 1534491824,
3239               "shard": 1
3240             },
3241             {
3242               "hash": 80717615,
3243               "shard": 2
3244             }
3245            ],
3246            "total_chunk_size": 2048
3247          },
3248          "errors": [],
3249          "osd": 0,
3250          "primary": false,
3251          "shard": 2,
3252          "size": 2048
3253        },
3254        {
3255          "hashinfo": {
3256            "cumulative_shard_hashes": [
3257             {
3258               "hash": 1534350760,
3259               "shard": 0
3260             },
3261             {
3262               "hash": 1534491824,
3263               "shard": 1
3264             },
3265             {
3266               "hash": 1534350760,
3267               "shard": 2
3268             }
3269            ],
3270            "total_chunk_size": 2048
3271          },
3272          "errors": [],
3273          "osd": 1,
3274          "primary": true,
3275          "shard": 0,
3276          "size": 2048
3277        },
3278        {
3279          "hashinfo": {
3280            "cumulative_shard_hashes": [
3281             {
3282               "hash": 1534350760,
3283               "shard": 0
3284             },
3285             {
3286               "hash": 1534491824,
3287               "shard": 1
3288             },
3289             {
3290               "hash": 1534350760,
3291               "shard": 2
3292             }
3293            ],
3294            "total_chunk_size": 2048
3295          },
3296          "errors": [],
3297          "osd": 2,
3298          "primary": false,
3299          "shard": 1,
3300          "size": 2048
3301        }
3302      ],
3303      "union_shard_errors": []
3304     }
3305   ],
3306   "epoch": 0
3307 }
3308 EOF
3309
3310     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
3311     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
3312     if test $getjson = "yes"
3313     then
3314         jq '.' $dir/json > save3.json
3315     fi
3316
3317     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
3318     then
3319       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
3320     fi
3321
3322     pg_deep_scrub $pg
3323
3324     rados list-inconsistent-pg $poolname > $dir/json || return 1
3325     # Check pg count
3326     test $(jq '. | length' $dir/json) = "1" || return 1
3327     # Check pgid
3328     test $(jq -r '.[0]' $dir/json) = $pg || return 1
3329
3330     rados list-inconsistent-obj $pg > $dir/json || return 1
3331     # Get epoch for repair-get requests
3332     epoch=$(jq .epoch $dir/json)
3333
3334     if [ "$allow_overwrites" = "true" ]
3335     then
3336       jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
3337 {
3338   "inconsistents": [
3339     {
3340       "shards": [
3341         {
3342           "data_digest": "0x00000000",
3343           "omap_digest": "0xffffffff",
3344           "size": 2048,
3345           "errors": [],
3346           "shard": 2,
3347           "osd": 0,
3348           "primary": false
3349         },
3350         {
3351           "object_info": {
3352             "oid": {
3353               "oid": "EOBJ1",
3354               "key": "",
3355               "snapid": -2,
3356               "hash": 560836233,
3357               "max": 0,
3358               "pool": 3,
3359               "namespace": ""
3360             },
3361             "version": "27'1",
3362             "prior_version": "0'0",
3363             "last_reqid": "client.4184.0:1",
3364             "user_version": 1,
3365             "size": 7,
3366             "mtime": "2018-04-05 14:31:33.837147",
3367             "local_mtime": "2018-04-05 14:31:33.840763",
3368             "lost": 0,
3369             "flags": [
3370               "dirty",
3371               "data_digest"
3372             ],
3373             "truncate_seq": 0,
3374             "truncate_size": 0,
3375             "data_digest": "0x2ddbf8f5",
3376             "omap_digest": "0xffffffff",
3377             "expected_object_size": 0,
3378             "expected_write_size": 0,
3379             "alloc_hint_flags": 0,
3380             "manifest": {
3381               "type": 0
3382             },
3383             "watchers": {}
3384           },
3385           "size": 9,
3386           "shard": 0,
3387           "errors": [
3388             "read_error",
3389             "size_mismatch_info",
3390             "obj_size_info_mismatch"
3391           ],
3392           "osd": 1,
3393           "primary": true
3394         },
3395         {
3396           "data_digest": "0x00000000",
3397           "omap_digest": "0xffffffff",
3398           "size": 2048,
3399           "shard": 1,
3400           "errors": [],
3401           "osd": 2,
3402           "primary": false
3403         }
3404       ],
3405       "selected_object_info": {
3406         "oid": {
3407           "oid": "EOBJ1",
3408           "key": "",
3409           "snapid": -2,
3410           "hash": 560836233,
3411           "max": 0,
3412           "pool": 3,
3413           "namespace": ""
3414         },
3415         "version": "27'1",
3416         "prior_version": "0'0",
3417         "last_reqid": "client.4184.0:1",
3418         "user_version": 1,
3419         "size": 7,
3420         "mtime": "2018-04-05 14:31:33.837147",
3421         "local_mtime": "2018-04-05 14:31:33.840763",
3422         "lost": 0,
3423         "flags": [
3424           "dirty",
3425           "data_digest"
3426         ],
3427         "truncate_seq": 0,
3428         "truncate_size": 0,
3429         "data_digest": "0x2ddbf8f5",
3430         "omap_digest": "0xffffffff",
3431         "expected_object_size": 0,
3432         "expected_write_size": 0,
3433         "alloc_hint_flags": 0,
3434         "manifest": {
3435           "type": 0
3436         },
3437         "watchers": {}
3438       },
3439       "union_shard_errors": [
3440         "read_error",
3441         "size_mismatch_info",
3442         "obj_size_info_mismatch"
3443       ],
3444       "errors": [
3445         "size_mismatch"
3446       ],
3447       "object": {
3448         "version": 1,
3449         "snap": "head",
3450         "locator": "",
3451         "nspace": "",
3452         "name": "EOBJ1"
3453       }
3454     },
3455     {
3456       "shards": [
3457         {
3458           "data_digest": "0x00000000",
3459           "omap_digest": "0xffffffff",
3460           "size": 2048,
3461           "errors": [],
3462           "shard": 2,
3463           "osd": 0,
3464           "primary": false
3465         },
3466         {
3467           "shard": 0,
3468           "errors": [
3469             "missing"
3470           ],
3471           "osd": 1,
3472           "primary": true
3473         },
3474         {
3475           "data_digest": "0x00000000",
3476           "omap_digest": "0xffffffff",
3477           "size": 2048,
3478           "shard": 1,
3479           "errors": [],
3480           "osd": 2,
3481           "primary": false
3482         }
3483       ],
3484       "selected_object_info": {
3485         "oid": {
3486           "oid": "EOBJ3",
3487           "key": "",
3488           "snapid": -2,
3489           "hash": 3125668237,
3490           "max": 0,
3491           "pool": 3,
3492           "namespace": ""
3493         },
3494         "version": "39'3",
3495         "prior_version": "0'0",
3496         "last_reqid": "client.4252.0:1",
3497         "user_version": 3,
3498         "size": 7,
3499         "mtime": "2018-04-05 14:31:46.841145",
3500         "local_mtime": "2018-04-05 14:31:46.844996",
3501         "lost": 0,
3502         "flags": [
3503           "dirty",
3504           "data_digest"
3505         ],
3506         "truncate_seq": 0,
3507         "truncate_size": 0,
3508         "data_digest": "0x2ddbf8f5",
3509         "omap_digest": "0xffffffff",
3510         "expected_object_size": 0,
3511         "expected_write_size": 0,
3512         "alloc_hint_flags": 0,
3513         "manifest": {
3514           "type": 0
3515         },
3516         "watchers": {}
3517       },
3518       "union_shard_errors": [
3519         "missing"
3520       ],
3521       "errors": [],
3522       "object": {
3523         "version": 3,
3524         "snap": "head",
3525         "locator": "",
3526         "nspace": "",
3527         "name": "EOBJ3"
3528       }
3529     },
3530     {
3531       "shards": [
3532         {
3533           "attrs": [
3534             {
3535               "Base64": false,
3536               "value": "bad-val",
3537               "name": "key1-EOBJ4"
3538             },
3539             {
3540               "Base64": false,
3541               "value": "val2-EOBJ4",
3542               "name": "key2-EOBJ4"
3543             }
3544           ],
3545           "data_digest": "0x00000000",
3546           "omap_digest": "0xffffffff",
3547           "size": 2048,
3548           "errors": [],
3549           "shard": 2,
3550           "osd": 0,
3551           "primary": false
3552         },
3553         {
3554           "attrs": [
3555             {
3556               "Base64": false,
3557               "value": "val1-EOBJ4",
3558               "name": "key1-EOBJ4"
3559             },
3560             {
3561               "Base64": false,
3562               "value": "val2-EOBJ4",
3563               "name": "key2-EOBJ4"
3564             }
3565           ],
3566           "data_digest": "0x00000000",
3567           "omap_digest": "0xffffffff",
3568           "size": 2048,
3569           "errors": [],
3570           "shard": 0,
3571           "osd": 1,
3572           "primary": true
3573         },
3574         {
3575           "attrs": [
3576             {
3577               "Base64": false,
3578               "value": "val1-EOBJ4",
3579               "name": "key1-EOBJ4"
3580             },
3581             {
3582               "Base64": false,
3583               "value": "val3-EOBJ4",
3584               "name": "key3-EOBJ4"
3585             }
3586           ],
3587           "data_digest": "0x00000000",
3588           "omap_digest": "0xffffffff",
3589           "size": 2048,
3590           "errors": [],
3591           "shard": 1,
3592           "osd": 2,
3593           "primary": false
3594         }
3595       ],
3596       "selected_object_info": {
3597         "oid": {
3598           "oid": "EOBJ4",
3599           "key": "",
3600           "snapid": -2,
3601           "hash": 1618759290,
3602           "max": 0,
3603           "pool": 3,
3604           "namespace": ""
3605         },
3606         "version": "45'6",
3607         "prior_version": "45'5",
3608         "last_reqid": "client.4294.0:1",
3609         "user_version": 6,
3610         "size": 7,
3611         "mtime": "2018-04-05 14:31:54.663622",
3612         "local_mtime": "2018-04-05 14:31:54.664527",
3613         "lost": 0,
3614         "flags": [
3615           "dirty",
3616           "data_digest"
3617         ],
3618         "truncate_seq": 0,
3619         "truncate_size": 0,
3620         "data_digest": "0x2ddbf8f5",
3621         "omap_digest": "0xffffffff",
3622         "expected_object_size": 0,
3623         "expected_write_size": 0,
3624         "alloc_hint_flags": 0,
3625         "manifest": {
3626           "type": 0
3627         },
3628         "watchers": {}
3629       },
3630       "union_shard_errors": [],
3631       "errors": [
3632         "attr_value_mismatch",
3633         "attr_name_mismatch"
3634       ],
3635       "object": {
3636         "version": 6,
3637         "snap": "head",
3638         "locator": "",
3639         "nspace": "",
3640         "name": "EOBJ4"
3641       }
3642     },
3643     {
3644       "shards": [
3645         {
3646           "data_digest": "0x00000000",
3647           "omap_digest": "0xffffffff",
3648           "size": 2048,
3649           "errors": [],
3650           "shard": 2,
3651           "osd": 0,
3652           "primary": false
3653         },
3654         {
3655           "data_digest": "0x00000000",
3656           "omap_digest": "0xffffffff",
3657           "object_info": {
3658             "oid": {
3659               "oid": "EOBJ5",
3660               "key": "",
3661               "snapid": -2,
3662               "hash": 2918945441,
3663               "max": 0,
3664               "pool": 3,
3665               "namespace": ""
3666             },
3667             "version": "59'7",
3668             "prior_version": "0'0",
3669             "last_reqid": "client.4382.0:1",
3670             "user_version": 7,
3671             "size": 7,
3672             "mtime": "2018-04-05 14:32:12.929161",
3673             "local_mtime": "2018-04-05 14:32:12.934707",
3674             "lost": 0,
3675             "flags": [
3676               "dirty",
3677               "data_digest"
3678             ],
3679             "truncate_seq": 0,
3680             "truncate_size": 0,
3681             "data_digest": "0x2ddbf8f5",
3682             "omap_digest": "0xffffffff",
3683             "expected_object_size": 0,
3684             "expected_write_size": 0,
3685             "alloc_hint_flags": 0,
3686             "manifest": {
3687               "type": 0
3688             },
3689             "watchers": {}
3690           },
3691           "size": 4096,
3692           "errors": [
3693             "size_mismatch_info",
3694             "obj_size_info_mismatch"
3695           ],
3696           "shard": 0,
3697           "osd": 1,
3698           "primary": true
3699         },
3700         {
3701           "data_digest": "0x00000000",
3702           "omap_digest": "0xffffffff",
3703           "size": 2048,
3704           "errors": [],
3705           "shard": 1,
3706           "osd": 2,
3707           "primary": false
3708         }
3709       ],
3710       "selected_object_info": {
3711         "oid": {
3712           "oid": "EOBJ5",
3713           "key": "",
3714           "snapid": -2,
3715           "hash": 2918945441,
3716           "max": 0,
3717           "pool": 3,
3718           "namespace": ""
3719         },
3720         "version": "59'7",
3721         "prior_version": "0'0",
3722         "last_reqid": "client.4382.0:1",
3723         "user_version": 7,
3724         "size": 7,
3725         "mtime": "2018-04-05 14:32:12.929161",
3726         "local_mtime": "2018-04-05 14:32:12.934707",
3727         "lost": 0,
3728         "flags": [
3729           "dirty",
3730           "data_digest"
3731         ],
3732         "truncate_seq": 0,
3733         "truncate_size": 0,
3734         "data_digest": "0x2ddbf8f5",
3735         "omap_digest": "0xffffffff",
3736         "expected_object_size": 0,
3737         "expected_write_size": 0,
3738         "alloc_hint_flags": 0,
3739         "manifest": {
3740           "type": 0
3741         },
3742         "watchers": {}
3743       },
3744       "union_shard_errors": [
3745         "size_mismatch_info",
3746         "obj_size_info_mismatch"
3747       ],
3748       "errors": [
3749         "size_mismatch"
3750       ],
3751       "object": {
3752         "version": 7,
3753         "snap": "head",
3754         "locator": "",
3755         "nspace": "",
3756         "name": "EOBJ5"
3757       }
3758     },
3759     {
3760       "object": {
3761         "name": "EOBJ6",
3762         "nspace": "",
3763         "locator": "",
3764         "snap": "head",
3765         "version": 8
3766       },
3767       "errors": [],
3768       "union_shard_errors": [
3769         "read_error",
3770         "hinfo_missing",
3771         "hinfo_corrupted"
3772       ],
3773       "selected_object_info": {
3774         "oid": {
3775           "oid": "EOBJ6",
3776           "key": "",
3777           "snapid": -2,
3778           "hash": 3050890866,
3779           "max": 0,
3780           "pool": 3,
3781           "namespace": ""
3782         },
3783         "version": "65'8",
3784         "prior_version": "0'0",
3785         "last_reqid": "client.4418.0:1",
3786         "user_version": 8,
3787         "size": 7,
3788         "mtime": "2018-04-05 14:32:20.634116",
3789         "local_mtime": "2018-04-05 14:32:20.637999",
3790         "lost": 0,
3791         "flags": [
3792           "dirty",
3793           "data_digest"
3794         ],
3795         "truncate_seq": 0,
3796         "truncate_size": 0,
3797         "data_digest": "0x2ddbf8f5",
3798         "omap_digest": "0xffffffff",
3799         "expected_object_size": 0,
3800         "expected_write_size": 0,
3801         "alloc_hint_flags": 0,
3802         "manifest": {
3803           "type": 0
3804         },
3805         "watchers": {}
3806       },
3807       "shards": [
3808         {
3809           "osd": 0,
3810           "primary": false,
3811           "shard": 2,
3812           "errors": [
3813             "read_error",
3814             "hinfo_missing"
3815           ],
3816           "size": 2048
3817         },
3818         {
3819           "osd": 1,
3820           "primary": true,
3821           "shard": 0,
3822           "errors": [
3823             "read_error",
3824             "hinfo_corrupted"
3825           ],
3826           "size": 2048,
3827           "hashinfo": "bad-val"
3828         },
3829         {
3830           "osd": 2,
3831           "primary": false,
3832           "shard": 1,
3833           "errors": [],
3834           "size": 2048,
3835           "omap_digest": "0xffffffff",
3836           "data_digest": "0x00000000",
3837           "hashinfo": {
3838             "cumulative_shard_hashes": [
3839             {
3840               "hash": 80717615,
3841               "shard": 0
3842             },
3843             {
3844               "hash": 1534491824,
3845               "shard": 1
3846             },
3847             {
3848               "hash": 80717615,
3849               "shard": 2
3850             }
3851            ],
3852            "total_chunk_size": 2048
3853          }
3854         }
3855       ]
3856     },
3857     {
3858       "object": {
3859         "name": "EOBJ7",
3860         "nspace": "",
3861         "locator": "",
3862         "snap": "head",
3863         "version": 10
3864       },
3865       "errors": [
3866         "hinfo_inconsistency"
3867       ],
3868       "union_shard_errors": [],
3869       "selected_object_info": {
3870         "oid": {
3871           "oid": "EOBJ7",
3872           "key": "",
3873           "snapid": -2,
3874           "hash": 3258066308,
3875           "max": 0,
3876           "pool": 3,
3877           "namespace": ""
3878         },
3879         "version": "75'10",
3880         "prior_version": "75'9",
3881         "last_reqid": "client.4482.0:1",
3882         "user_version": 10,
3883         "size": 34,
3884         "mtime": "2018-04-05 14:32:33.058782",
3885         "local_mtime": "2018-04-05 14:32:33.059679",
3886         "lost": 0,
3887         "flags": [
3888           "dirty",
3889           "data_digest"
3890         ],
3891         "truncate_seq": 0,
3892         "truncate_size": 0,
3893         "data_digest": "0x136e4e27",
3894         "omap_digest": "0xffffffff",
3895         "expected_object_size": 0,
3896         "expected_write_size": 0,
3897         "alloc_hint_flags": 0,
3898         "manifest": {
3899           "type": 0
3900         },
3901         "watchers": {}
3902       },
3903       "shards": [
3904         {
3905           "osd": 0,
3906           "primary": false,
3907           "shard": 2,
3908           "errors": [],
3909           "size": 2048,
3910           "omap_digest": "0xffffffff",
3911           "data_digest": "0x00000000",
3912           "hashinfo": {
3913            "cumulative_shard_hashes": [
3914             {
3915               "hash": 80717615,
3916               "shard": 0
3917             },
3918             {
3919               "hash": 1534491824,
3920               "shard": 1
3921             },
3922             {
3923               "hash": 80717615,
3924               "shard": 2
3925             }
3926            ],
3927            "total_chunk_size": 2048
3928           }
3929         },
3930         {
3931           "osd": 1,
3932           "primary": true,
3933           "shard": 0,
3934           "errors": [],
3935           "size": 2048,
3936           "omap_digest": "0xffffffff",
3937           "data_digest": "0x00000000",
3938           "hashinfo": {
3939            "cumulative_shard_hashes": [
3940             {
3941               "hash": 1534350760,
3942               "shard": 0
3943             },
3944             {
3945               "hash": 1534491824,
3946               "shard": 1
3947             },
3948             {
3949               "hash": 1534350760,
3950               "shard": 2
3951             }
3952            ],
3953            "total_chunk_size": 2048
3954           }
3955         },
3956         {
3957           "osd": 2,
3958           "primary": false,
3959           "shard": 1,
3960           "errors": [],
3961           "size": 2048,
3962           "omap_digest": "0xffffffff",
3963           "data_digest": "0x00000000",
3964           "hashinfo": {
3965            "cumulative_shard_hashes": [
3966             {
3967               "hash": 1534350760,
3968               "shard": 0
3969             },
3970             {
3971               "hash": 1534491824,
3972               "shard": 1
3973             },
3974             {
3975               "hash": 1534350760,
3976               "shard": 2
3977             }
3978            ],
3979            "total_chunk_size": 2048
3980           }
3981         }
3982       ]
3983     }
3984   ],
3985   "epoch": 0
3986 }
3987 EOF
3988
3989     else
3990
3991       jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
3992 {
3993   "inconsistents": [
3994     {
3995       "shards": [
3996         {
3997           "data_digest": "0x04cfa72f",
3998           "omap_digest": "0xffffffff",
3999           "size": 2048,
4000           "errors": [],
4001           "shard": 2,
4002           "osd": 0,
4003           "primary": false
4004         },
4005         {
4006           "object_info": {
4007             "oid": {
4008               "oid": "EOBJ1",
4009               "key": "",
4010               "snapid": -2,
4011               "hash": 560836233,
4012               "max": 0,
4013               "pool": 3,
4014               "namespace": ""
4015             },
4016             "version": "27'1",
4017             "prior_version": "0'0",
4018             "last_reqid": "client.4192.0:1",
4019             "user_version": 1,
4020             "size": 7,
4021             "mtime": "2018-04-05 14:30:10.688009",
4022             "local_mtime": "2018-04-05 14:30:10.691774",
4023             "lost": 0,
4024             "flags": [
4025               "dirty",
4026               "data_digest"
4027             ],
4028             "truncate_seq": 0,
4029             "truncate_size": 0,
4030             "data_digest": "0x2ddbf8f5",
4031             "omap_digest": "0xffffffff",
4032             "expected_object_size": 0,
4033             "expected_write_size": 0,
4034             "alloc_hint_flags": 0,
4035             "manifest": {
4036               "type": 0
4037             },
4038             "watchers": {}
4039           },
4040           "size": 9,
4041           "shard": 0,
4042           "errors": [
4043             "read_error",
4044             "size_mismatch_info",
4045             "obj_size_info_mismatch"
4046           ],
4047           "osd": 1,
4048           "primary": true
4049         },
4050         {
4051           "data_digest": "0x04cfa72f",
4052           "omap_digest": "0xffffffff",
4053           "size": 2048,
4054           "shard": 1,
4055           "errors": [],
4056           "osd": 2,
4057           "primary": false
4058         }
4059       ],
4060       "selected_object_info": {
4061         "oid": {
4062           "oid": "EOBJ1",
4063           "key": "",
4064           "snapid": -2,
4065           "hash": 560836233,
4066           "max": 0,
4067           "pool": 3,
4068           "namespace": ""
4069         },
4070         "version": "27'1",
4071         "prior_version": "0'0",
4072         "last_reqid": "client.4192.0:1",
4073         "user_version": 1,
4074         "size": 7,
4075         "mtime": "2018-04-05 14:30:10.688009",
4076         "local_mtime": "2018-04-05 14:30:10.691774",
4077         "lost": 0,
4078         "flags": [
4079           "dirty",
4080           "data_digest"
4081         ],
4082         "truncate_seq": 0,
4083         "truncate_size": 0,
4084         "data_digest": "0x2ddbf8f5",
4085         "omap_digest": "0xffffffff",
4086         "expected_object_size": 0,
4087         "expected_write_size": 0,
4088         "alloc_hint_flags": 0,
4089         "manifest": {
4090           "type": 0
4091         },
4092         "watchers": {}
4093       },
4094       "union_shard_errors": [
4095         "read_error",
4096         "size_mismatch_info",
4097         "obj_size_info_mismatch"
4098       ],
4099       "errors": [
4100         "size_mismatch"
4101       ],
4102       "object": {
4103         "version": 1,
4104         "snap": "head",
4105         "locator": "",
4106         "nspace": "",
4107         "name": "EOBJ1"
4108       }
4109     },
4110     {
4111       "shards": [
4112         {
4113           "size": 2048,
4114           "errors": [
4115             "ec_hash_error"
4116           ],
4117           "shard": 2,
4118           "osd": 0,
4119           "primary": false
4120         },
4121         {
4122           "data_digest": "0x04cfa72f",
4123           "omap_digest": "0xffffffff",
4124           "size": 2048,
4125           "errors": [],
4126           "shard": 0,
4127           "osd": 1,
4128           "primary": true
4129         },
4130         {
4131           "data_digest": "0x04cfa72f",
4132           "omap_digest": "0xffffffff",
4133           "size": 2048,
4134           "errors": [],
4135           "shard": 1,
4136           "osd": 2,
4137           "primary": false
4138         }
4139       ],
4140       "selected_object_info": {
4141         "oid": {
4142           "oid": "EOBJ2",
4143           "key": "",
4144           "snapid": -2,
4145           "hash": 562812377,
4146           "max": 0,
4147           "pool": 3,
4148           "namespace": ""
4149         },
4150         "version": "33'2",
4151         "prior_version": "0'0",
4152         "last_reqid": "client.4224.0:1",
4153         "user_version": 2,
4154         "size": 7,
4155         "mtime": "2018-04-05 14:30:14.152945",
4156         "local_mtime": "2018-04-05 14:30:14.154014",
4157         "lost": 0,
4158         "flags": [
4159           "dirty",
4160           "data_digest"
4161         ],
4162         "truncate_seq": 0,
4163         "truncate_size": 0,
4164         "data_digest": "0x2ddbf8f5",
4165         "omap_digest": "0xffffffff",
4166         "expected_object_size": 0,
4167         "expected_write_size": 0,
4168         "alloc_hint_flags": 0,
4169         "manifest": {
4170           "type": 0
4171         },
4172         "watchers": {}
4173       },
4174       "union_shard_errors": [
4175         "ec_hash_error"
4176       ],
4177       "errors": [],
4178       "object": {
4179         "version": 2,
4180         "snap": "head",
4181         "locator": "",
4182         "nspace": "",
4183         "name": "EOBJ2"
4184       }
4185     },
4186     {
4187       "shards": [
4188         {
4189           "data_digest": "0x04cfa72f",
4190           "omap_digest": "0xffffffff",
4191           "size": 2048,
4192           "errors": [],
4193           "shard": 2,
4194           "osd": 0,
4195           "primary": false
4196         },
4197         {
4198           "osd": 1,
4199           "primary": true,
4200           "shard": 0,
4201           "errors": [
4202             "missing"
4203           ]
4204         },
4205         {
4206           "data_digest": "0x04cfa72f",
4207           "omap_digest": "0xffffffff",
4208           "size": 2048,
4209           "shard": 1,
4210           "errors": [],
4211           "osd": 2,
4212           "primary": false
4213         }
4214       ],
4215       "selected_object_info": {
4216         "oid": {
4217           "oid": "EOBJ3",
4218           "key": "",
4219           "snapid": -2,
4220           "hash": 3125668237,
4221           "max": 0,
4222           "pool": 3,
4223           "namespace": ""
4224         },
4225         "version": "39'3",
4226         "prior_version": "0'0",
4227         "last_reqid": "client.4258.0:1",
4228         "user_version": 3,
4229         "size": 7,
4230         "mtime": "2018-04-05 14:30:18.875544",
4231         "local_mtime": "2018-04-05 14:30:18.880153",
4232         "lost": 0,
4233         "flags": [
4234           "dirty",
4235           "data_digest"
4236         ],
4237         "truncate_seq": 0,
4238         "truncate_size": 0,
4239         "data_digest": "0x2ddbf8f5",
4240         "omap_digest": "0xffffffff",
4241         "expected_object_size": 0,
4242         "expected_write_size": 0,
4243         "alloc_hint_flags": 0,
4244         "manifest": {
4245           "type": 0
4246         },
4247         "watchers": {}
4248       },
4249       "union_shard_errors": [
4250         "missing"
4251       ],
4252       "errors": [],
4253       "object": {
4254         "version": 3,
4255         "snap": "head",
4256         "locator": "",
4257         "nspace": "",
4258         "name": "EOBJ3"
4259       }
4260     },
4261     {
4262       "shards": [
4263         {
4264           "attrs": [
4265             {
4266               "Base64": false,
4267               "value": "bad-val",
4268               "name": "key1-EOBJ4"
4269             },
4270             {
4271               "Base64": false,
4272               "value": "val2-EOBJ4",
4273               "name": "key2-EOBJ4"
4274             }
4275           ],
4276           "data_digest": "0x04cfa72f",
4277           "omap_digest": "0xffffffff",
4278           "size": 2048,
4279           "errors": [],
4280           "shard": 2,
4281           "osd": 0,
4282           "primary": false
4283         },
4284         {
4285           "osd": 1,
4286           "primary": true,
4287           "shard": 0,
4288           "errors": [],
4289           "size": 2048,
4290           "omap_digest": "0xffffffff",
4291           "data_digest": "0x04cfa72f",
4292           "attrs": [
4293             {
4294               "Base64": false,
4295               "value": "val1-EOBJ4",
4296               "name": "key1-EOBJ4"
4297             },
4298             {
4299               "Base64": false,
4300               "value": "val2-EOBJ4",
4301               "name": "key2-EOBJ4"
4302             }
4303           ]
4304         },
4305         {
4306           "osd": 2,
4307           "primary": false,
4308           "shard": 1,
4309           "errors": [],
4310           "size": 2048,
4311           "omap_digest": "0xffffffff",
4312           "data_digest": "0x04cfa72f",
4313           "attrs": [
4314             {
4315               "Base64": false,
4316               "value": "val1-EOBJ4",
4317               "name": "key1-EOBJ4"
4318             },
4319             {
4320               "Base64": false,
4321               "value": "val3-EOBJ4",
4322               "name": "key3-EOBJ4"
4323             }
4324           ]
4325         }
4326       ],
4327       "selected_object_info": {
4328         "oid": {
4329           "oid": "EOBJ4",
4330           "key": "",
4331           "snapid": -2,
4332           "hash": 1618759290,
4333           "max": 0,
4334           "pool": 3,
4335           "namespace": ""
4336         },
4337         "version": "45'6",
4338         "prior_version": "45'5",
4339         "last_reqid": "client.4296.0:1",
4340         "user_version": 6,
4341         "size": 7,
4342         "mtime": "2018-04-05 14:30:22.271983",
4343         "local_mtime": "2018-04-05 14:30:22.272840",
4344         "lost": 0,
4345         "flags": [
4346           "dirty",
4347           "data_digest"
4348         ],
4349         "truncate_seq": 0,
4350         "truncate_size": 0,
4351         "data_digest": "0x2ddbf8f5",
4352         "omap_digest": "0xffffffff",
4353         "expected_object_size": 0,
4354         "expected_write_size": 0,
4355         "alloc_hint_flags": 0,
4356         "manifest": {
4357           "type": 0
4358         },
4359         "watchers": {}
4360       },
4361       "union_shard_errors": [],
4362       "errors": [
4363         "attr_value_mismatch",
4364         "attr_name_mismatch"
4365       ],
4366       "object": {
4367         "version": 6,
4368         "snap": "head",
4369         "locator": "",
4370         "nspace": "",
4371         "name": "EOBJ4"
4372       }
4373     },
4374     {
4375       "shards": [
4376         {
4377           "data_digest": "0x04cfa72f",
4378           "omap_digest": "0xffffffff",
4379           "size": 2048,
4380           "errors": [],
4381           "shard": 2,
4382           "osd": 0,
4383           "primary": false
4384         },
4385         {
4386           "object_info": {
4387             "oid": {
4388               "oid": "EOBJ5",
4389               "key": "",
4390               "snapid": -2,
4391               "hash": 2918945441,
4392               "max": 0,
4393               "pool": 3,
4394               "namespace": ""
4395             },
4396             "version": "59'7",
4397             "prior_version": "0'0",
4398             "last_reqid": "client.4384.0:1",
4399             "user_version": 7,
4400             "size": 7,
4401             "mtime": "2018-04-05 14:30:35.162395",
4402             "local_mtime": "2018-04-05 14:30:35.166390",
4403             "lost": 0,
4404             "flags": [
4405               "dirty",
4406               "data_digest"
4407             ],
4408             "truncate_seq": 0,
4409             "truncate_size": 0,
4410             "data_digest": "0x2ddbf8f5",
4411             "omap_digest": "0xffffffff",
4412             "expected_object_size": 0,
4413             "expected_write_size": 0,
4414             "alloc_hint_flags": 0,
4415             "manifest": {
4416               "type": 0
4417             },
4418             "watchers": {}
4419           },
4420           "size": 4096,
4421           "shard": 0,
4422           "errors": [
4423             "size_mismatch_info",
4424             "ec_size_error",
4425             "obj_size_info_mismatch"
4426           ],
4427           "osd": 1,
4428           "primary": true
4429         },
4430         {
4431           "data_digest": "0x04cfa72f",
4432           "omap_digest": "0xffffffff",
4433           "size": 2048,
4434           "shard": 1,
4435           "errors": [],
4436           "osd": 2,
4437           "primary": false
4438         }
4439       ],
4440       "selected_object_info": {
4441         "oid": {
4442           "oid": "EOBJ5",
4443           "key": "",
4444           "snapid": -2,
4445           "hash": 2918945441,
4446           "max": 0,
4447           "pool": 3,
4448           "namespace": ""
4449         },
4450         "version": "59'7",
4451         "prior_version": "0'0",
4452         "last_reqid": "client.4384.0:1",
4453         "user_version": 7,
4454         "size": 7,
4455         "mtime": "2018-04-05 14:30:35.162395",
4456         "local_mtime": "2018-04-05 14:30:35.166390",
4457         "lost": 0,
4458         "flags": [
4459           "dirty",
4460           "data_digest"
4461         ],
4462         "truncate_seq": 0,
4463         "truncate_size": 0,
4464         "data_digest": "0x2ddbf8f5",
4465         "omap_digest": "0xffffffff",
4466         "expected_object_size": 0,
4467         "expected_write_size": 0,
4468         "alloc_hint_flags": 0,
4469         "manifest": {
4470           "type": 0
4471         },
4472         "watchers": {}
4473       },
4474       "union_shard_errors": [
4475         "size_mismatch_info",
4476         "ec_size_error",
4477         "obj_size_info_mismatch"
4478       ],
4479       "errors": [
4480         "size_mismatch"
4481       ],
4482       "object": {
4483         "version": 7,
4484         "snap": "head",
4485         "locator": "",
4486         "nspace": "",
4487         "name": "EOBJ5"
4488       }
4489     },
4490     {
4491       "object": {
4492         "name": "EOBJ6",
4493         "nspace": "",
4494         "locator": "",
4495         "snap": "head",
4496         "version": 8
4497       },
4498       "errors": [],
4499       "union_shard_errors": [
4500         "read_error",
4501         "hinfo_missing",
4502         "hinfo_corrupted"
4503       ],
4504       "selected_object_info": {
4505         "oid": {
4506           "oid": "EOBJ6",
4507           "key": "",
4508           "snapid": -2,
4509           "hash": 3050890866,
4510           "max": 0,
4511           "pool": 3,
4512           "namespace": ""
4513         },
4514         "version": "65'8",
4515         "prior_version": "0'0",
4516         "last_reqid": "client.4420.0:1",
4517         "user_version": 8,
4518         "size": 7,
4519         "mtime": "2018-04-05 14:30:40.914673",
4520         "local_mtime": "2018-04-05 14:30:40.917705",
4521         "lost": 0,
4522         "flags": [
4523           "dirty",
4524           "data_digest"
4525         ],
4526         "truncate_seq": 0,
4527         "truncate_size": 0,
4528         "data_digest": "0x2ddbf8f5",
4529         "omap_digest": "0xffffffff",
4530         "expected_object_size": 0,
4531         "expected_write_size": 0,
4532         "alloc_hint_flags": 0,
4533         "manifest": {
4534           "type": 0
4535         },
4536         "watchers": {}
4537       },
4538       "shards": [
4539         {
4540           "osd": 0,
4541           "primary": false,
4542           "shard": 2,
4543           "errors": [
4544             "read_error",
4545             "hinfo_missing"
4546           ],
4547           "size": 2048
4548         },
4549         {
4550           "osd": 1,
4551           "primary": true,
4552           "shard": 0,
4553           "errors": [
4554             "read_error",
4555             "hinfo_corrupted"
4556           ],
4557           "size": 2048,
4558           "hashinfo": "bad-val"
4559         },
4560         {
4561           "osd": 2,
4562           "primary": false,
4563           "shard": 1,
4564           "errors": [],
4565           "size": 2048,
4566           "omap_digest": "0xffffffff",
4567           "data_digest": "0x04cfa72f",
4568           "hashinfo": {
4569            "cumulative_shard_hashes": [
4570             {
4571               "hash": 80717615,
4572               "shard": 0
4573             },
4574             {
4575               "hash": 1534491824,
4576               "shard": 1
4577             },
4578             {
4579               "hash": 80717615,
4580               "shard": 2
4581             }
4582            ],
4583            "total_chunk_size": 2048
4584           }
4585         }
4586       ]
4587     },
4588     {
4589       "object": {
4590         "name": "EOBJ7",
4591         "nspace": "",
4592         "locator": "",
4593         "snap": "head",
4594         "version": 10
4595       },
4596       "errors": [
4597         "hinfo_inconsistency"
4598       ],
4599       "union_shard_errors": [
4600         "ec_hash_error"
4601       ],
4602       "selected_object_info": {
4603         "oid": {
4604           "oid": "EOBJ7",
4605           "key": "",
4606           "snapid": -2,
4607           "hash": 3258066308,
4608           "max": 0,
4609           "pool": 3,
4610           "namespace": ""
4611         },
4612         "version": "75'10",
4613         "prior_version": "75'9",
4614         "last_reqid": "client.4486.0:1",
4615         "user_version": 10,
4616         "size": 34,
4617         "mtime": "2018-04-05 14:30:50.995009",
4618         "local_mtime": "2018-04-05 14:30:50.996112",
4619         "lost": 0,
4620         "flags": [
4621           "dirty",
4622           "data_digest"
4623         ],
4624         "truncate_seq": 0,
4625         "truncate_size": 0,
4626         "data_digest": "0x136e4e27",
4627         "omap_digest": "0xffffffff",
4628         "expected_object_size": 0,
4629         "expected_write_size": 0,
4630         "alloc_hint_flags": 0,
4631         "manifest": {
4632           "type": 0
4633         },
4634         "watchers": {}
4635       },
4636       "shards": [
4637         {
4638           "osd": 0,
4639           "primary": false,
4640           "shard": 2,
4641           "errors": [
4642             "ec_hash_error"
4643           ],
4644           "size": 2048,
4645           "hashinfo": {
4646            "cumulative_shard_hashes": [
4647             {
4648               "hash": 80717615,
4649               "shard": 0
4650             },
4651             {
4652               "hash": 1534491824,
4653               "shard": 1
4654             },
4655             {
4656               "hash": 80717615,
4657               "shard": 2
4658             }
4659            ],
4660            "total_chunk_size": 2048
4661          }
4662         },
4663         {
4664           "osd": 1,
4665           "primary": true,
4666           "shard": 0,
4667           "errors": [],
4668           "size": 2048,
4669           "omap_digest": "0xffffffff",
4670           "data_digest": "0x5b7455a8",
4671           "hashinfo": {
4672            "cumulative_shard_hashes": [
4673             {
4674               "hash": 1534350760,
4675               "shard": 0
4676             },
4677             {
4678               "hash": 1534491824,
4679               "shard": 1
4680             },
4681             {
4682               "hash": 1534350760,
4683               "shard": 2
4684             }
4685            ],
4686            "total_chunk_size": 2048
4687           }
4688         },
4689         {
4690           "osd": 2,
4691           "primary": false,
4692           "shard": 1,
4693           "errors": [],
4694           "size": 2048,
4695           "omap_digest": "0xffffffff",
4696           "data_digest": "0x5b7455a8",
4697           "hashinfo": {
4698            "cumulative_shard_hashes": [
4699             {
4700               "hash": 1534350760,
4701               "shard": 0
4702             },
4703             {
4704               "hash": 1534491824,
4705               "shard": 1
4706             },
4707             {
4708               "hash": 1534350760,
4709               "shard": 2
4710             }
4711            ],
4712            "total_chunk_size": 2048
4713           }
4714         }
4715       ]
4716     }
4717   ],
4718   "epoch": 0
4719 }
4720 EOF
4721
4722     fi
4723
4724     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
4725     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
4726     if test $getjson = "yes"
4727     then
4728       if [ "$allow_overwrites" = "true" ]
4729       then
4730         num=4
4731       else
4732         num=5
4733       fi
4734       jq '.' $dir/json > save${num}.json
4735     fi
4736
4737     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
4738     then
4739       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
4740     fi
4741
4742     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
4743     teardown $dir || return 1
4744 }
4745
4746 function TEST_corrupt_scrub_erasure_appends() {
4747     corrupt_scrub_erasure $1 false
4748 }
4749
4750 function TEST_corrupt_scrub_erasure_overwrites() {
4751     if [ "$use_ec_overwrite" = "true" ]; then
4752         corrupt_scrub_erasure $1 true
4753     fi
4754 }
4755
4756 #
4757 # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost
4758 #
4759 function TEST_periodic_scrub_replicated() {
4760     local dir=$1
4761     local poolname=psr_pool
4762     local objname=POBJ
4763
4764     setup $dir || return 1
4765     run_mon $dir a --osd_pool_default_size=2 || return 1
4766     run_mgr $dir x || return 1
4767     local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
4768     ceph_osd_args+="--osd_scrub_backoff_ratio=0"
4769     run_osd $dir 0 $ceph_osd_args || return 1
4770     run_osd $dir 1 $ceph_osd_args || return 1
4771     create_rbd_pool || return 1
4772     wait_for_clean || return 1
4773
4774     create_pool $poolname 1 1 || return 1
4775     wait_for_clean || return 1
4776
4777     local osd=0
4778     add_something $dir $poolname $objname scrub || return 1
4779     local primary=$(get_primary $poolname $objname)
4780     local pg=$(get_pg $poolname $objname)
4781
4782     # Add deep-scrub only error
4783     local payload=UVWXYZ
4784     echo $payload > $dir/CORRUPT
4785     # Uses $ceph_osd_args for osd restart
4786     objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
4787
4788     # No scrub information available, so expect failure
4789     set -o pipefail
4790     !  rados list-inconsistent-obj $pg | jq '.' || return 1
4791     set +o pipefail
4792
4793     pg_deep_scrub $pg || return 1
4794
4795     # Make sure bad object found
4796     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
4797
4798     flush_pg_stats
4799     local last_scrub=$(get_last_scrub_stamp $pg)
4800     # Fake a schedule scrub
4801     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
4802              trigger_scrub $pg || return 1
4803     # Wait for schedule regular scrub
4804     wait_for_scrub $pg "$last_scrub"
4805
4806     # It needed to be upgraded
4807     grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1
4808
4809     # Bad object still known
4810     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
4811
4812     # Can't upgrade with this set
4813     ceph osd set nodeep-scrub
4814     # Let map change propagate to OSDs
4815     flush pg_stats
4816     sleep 5
4817
4818     # Fake a schedule scrub
4819     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
4820              trigger_scrub $pg || return 1
4821     # Wait for schedule regular scrub
4822     # to notice scrub and skip it
4823     local found=false
4824     for i in $(seq 14 -1 0)
4825     do
4826       sleep 1
4827       ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; }
4828       echo Time left: $i seconds
4829     done
4830     test $found = "true" || return 1
4831
4832     # Bad object still known
4833     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
4834
4835     flush_pg_stats
4836     # Request a regular scrub and it will be done
4837     pg_scrub $pg
4838     grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1
4839
4840     # deep-scrub error is no longer present
4841     rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1
4842 }
4843
4844 #
4845 # Corrupt snapset in replicated pool
4846 #
4847 function TEST_corrupt_snapset_scrub_rep() {
4848     local dir=$1
4849     local poolname=csr_pool
4850     local total_objs=2
4851
4852     setup $dir || return 1
4853     run_mon $dir a --osd_pool_default_size=2 || return 1
4854     run_mgr $dir x || return 1
4855     run_osd $dir 0 || return 1
4856     run_osd $dir 1 || return 1
4857     create_rbd_pool || return 1
4858     wait_for_clean || return 1
4859
4860     create_pool foo 1 || return 1
4861     create_pool $poolname 1 1 || return 1
4862     wait_for_clean || return 1
4863
4864     for i in $(seq 1 $total_objs) ; do
4865         objname=ROBJ${i}
4866         add_something $dir $poolname $objname || return 1
4867
4868         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
4869         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
4870     done
4871
4872     local pg=$(get_pg $poolname ROBJ0)
4873
4874     for i in $(seq 1 $total_objs) ; do
4875         objname=ROBJ${i}
4876
4877         # Alternate corruption between osd.0 and osd.1
4878         local osd=$(expr $i % 2)
4879
4880         rados -p $poolname mksnap snap1
4881         echo -n head_of_snapshot_data > $dir/change
4882
4883         case $i in
4884         1)
4885           rados --pool $poolname put $objname $dir/change
4886           objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
4887           ;;
4888
4889         2)
4890           rados --pool $poolname put $objname $dir/change
4891           objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
4892           ;;
4893
4894         esac
4895     done
4896     rm $dir/change
4897
4898     pg_scrub $pg
4899
4900     rados list-inconsistent-pg $poolname > $dir/json || return 1
4901     # Check pg count
4902     test $(jq '. | length' $dir/json) = "1" || return 1
4903     # Check pgid
4904     test $(jq -r '.[0]' $dir/json) = $pg || return 1
4905
4906     rados list-inconsistent-obj $pg > $dir/json || return 1
4907
4908     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
4909 {
4910   "epoch": 34,
4911   "inconsistents": [
4912     {
4913       "object": {
4914         "name": "ROBJ1",
4915         "nspace": "",
4916         "locator": "",
4917         "snap": "head",
4918         "version": 8
4919       },
4920       "errors": [
4921         "snapset_inconsistency"
4922       ],
4923       "union_shard_errors": [],
4924       "selected_object_info": {
4925         "oid": {
4926           "oid": "ROBJ1",
4927           "key": "",
4928           "snapid": -2,
4929           "hash": 1454963827,
4930           "max": 0,
4931           "pool": 3,
4932           "namespace": ""
4933         },
4934         "version": "24'8",
4935         "prior_version": "21'3",
4936         "last_reqid": "client.4195.0:1",
4937         "user_version": 8,
4938         "size": 21,
4939         "mtime": "2018-04-05 14:35:43.286117",
4940         "local_mtime": "2018-04-05 14:35:43.288990",
4941         "lost": 0,
4942         "flags": [
4943           "dirty",
4944           "omap",
4945           "data_digest"
4946         ],
4947         "truncate_seq": 0,
4948         "truncate_size": 0,
4949         "data_digest": "0x53acb008",
4950         "omap_digest": "0xffffffff",
4951         "expected_object_size": 0,
4952         "expected_write_size": 0,
4953         "alloc_hint_flags": 0,
4954         "manifest": {
4955           "type": 0
4956         },
4957         "watchers": {}
4958       },
4959       "shards": [
4960         {
4961           "osd": 0,
4962           "primary": false,
4963           "errors": [],
4964           "size": 21,
4965           "snapset": {
4966             "clones": [
4967               {
4968                 "overlap": "[]",
4969                 "size": 7,
4970                 "snap": 1,
4971                 "snaps": [
4972                   1
4973                 ]
4974               }
4975             ],
4976             "head_exists": 1,
4977             "snap_context": {
4978               "seq": 1,
4979               "snaps": [
4980                 1
4981               ]
4982             }
4983           }
4984         },
4985         {
4986           "osd": 1,
4987           "primary": true,
4988           "errors": [],
4989           "size": 21,
4990           "snapset": {
4991             "clones": [],
4992             "head_exists": 0,
4993             "snap_context": {
4994               "seq": 0,
4995               "snaps": []
4996             }
4997           }
4998         }
4999       ]
5000     },
5001     {
5002       "object": {
5003         "name": "ROBJ2",
5004         "nspace": "",
5005         "locator": "",
5006         "snap": "head",
5007         "version": 10
5008       },
5009       "errors": [
5010         "snapset_inconsistency"
5011       ],
5012       "union_shard_errors": [],
5013       "selected_object_info": {
5014         "oid": {
5015           "oid": "ROBJ2",
5016           "key": "",
5017           "snapid": -2,
5018           "hash": 2026323607,
5019           "max": 0,
5020           "pool": 3,
5021           "namespace": ""
5022         },
5023         "version": "28'10",
5024         "prior_version": "23'6",
5025         "last_reqid": "client.4223.0:1",
5026         "user_version": 10,
5027         "size": 21,
5028         "mtime": "2018-04-05 14:35:48.326856",
5029         "local_mtime": "2018-04-05 14:35:48.328097",
5030         "lost": 0,
5031         "flags": [
5032           "dirty",
5033           "omap",
5034           "data_digest"
5035         ],
5036         "truncate_seq": 0,
5037         "truncate_size": 0,
5038         "data_digest": "0x53acb008",
5039         "omap_digest": "0xffffffff",
5040         "expected_object_size": 0,
5041         "expected_write_size": 0,
5042         "alloc_hint_flags": 0,
5043         "manifest": {
5044           "type": 0
5045         },
5046         "watchers": {}
5047       },
5048       "shards": [
5049         {
5050           "osd": 0,
5051           "primary": false,
5052           "errors": [],
5053           "size": 21,
5054           "snapset": {
5055             "clones": [],
5056             "head_exists": 0,
5057             "snap_context": {
5058               "seq": 0,
5059               "snaps": []
5060             }
5061           }
5062         },
5063         {
5064           "osd": 1,
5065           "primary": true,
5066           "errors": [],
5067           "size": 21,
5068           "snapset": {
5069             "clones": [
5070               {
5071                 "overlap": "[]",
5072                 "size": 7,
5073                 "snap": 1,
5074                 "snaps": [
5075                   1
5076                 ]
5077               }
5078             ],
5079             "head_exists": 1,
5080             "snap_context": {
5081               "seq": 1,
5082               "snaps": [
5083                 1
5084               ]
5085             }
5086           }
5087         }
5088       ]
5089     }
5090   ]
5091 }
5092 EOF
5093
5094     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
5095     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
5096     if test $getjson = "yes"
5097     then
5098         jq '.' $dir/json > save6.json
5099     fi
5100
5101     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
5102     then
5103       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
5104     fi
5105
5106     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
5107     teardown $dir || return 1
5108 }
5109
5110 main osd-scrub-repair "$@"
5111
5112 # Local Variables:
5113 # compile-command: "cd build ; make -j4 && \
5114 #    ../qa/run-standalone.sh osd-scrub-repair.sh"
5115 # End: