ceph/qa/standalone/scrub/osd-scrub-repair.sh

   1 #!/bin/bash -x
   2 #
   3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
   4 #
   5 # Author: Loic Dachary <loic@dachary.org>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  18
  19 if [ `uname` = FreeBSD ]; then
  20     # erasure coding overwrites are only tested on Bluestore
  21     # erasure coding on filestore is unsafe
  22     # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites
  23     use_ec_overwrite=false
  24 else
  25     use_ec_overwrite=true
  26 fi
  27
  28 # Test development and debugging
  29 # Set to "yes" in order to ignore diff errors and save results to update test
  30 getjson="no"
  31
  32 # Filter out mtime and local_mtime dates, version, prior_version and last_reqid (client) from any object_info.
  33 jqfilter='def walk(f):
  34   . as $in
  35   | if type == "object" then
  36       reduce keys[] as $key
  37         ( {}; . + { ($key):  ($in[$key] | walk(f)) } ) | f
  38     elif type == "array" then map( walk(f) ) | f
  39     else f
  40     end;
  41 walk(if type == "object" then del(.mtime) else . end)
  42 | walk(if type == "object" then del(.local_mtime) else . end)
  43 | walk(if type == "object" then del(.last_reqid) else . end)
  44 | walk(if type == "object" then del(.version) else . end)
  45 | walk(if type == "object" then del(.prior_version) else . end)
  46 | walk(if type == "object" then del(.redirect_target) else . end)
  47 | walk(if type == "object" then del(.legacy_snaps) else . end)'
  48
  49 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
  50
  51 function run() {
  52     local dir=$1
  53     shift
  54
  55     export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
  56     export CEPH_ARGS
  57     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  58     CEPH_ARGS+="--mon-host=$CEPH_MON "
  59
  60     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  61     for func in $funcs ; do
  62         $func $dir || return 1
  63     done
  64 }
  65
  66 function add_something() {
  67     local dir=$1
  68     local poolname=$2
  69     local obj=${3:-SOMETHING}
  70     local scrub=${4:-noscrub}
  71
  72     if [ "$scrub" = "noscrub" ];
  73     then
  74         ceph osd set noscrub || return 1
  75         ceph osd set nodeep-scrub || return 1
  76     else
  77         ceph osd unset noscrub || return 1
  78         ceph osd unset nodeep-scrub || return 1
  79     fi
  80
  81     local payload=ABCDEF
  82     echo $payload > $dir/ORIGINAL
  83     rados --pool $poolname put $obj $dir/ORIGINAL || return 1
  84 }
  85
  86 #
  87 # Corrupt one copy of a replicated pool
  88 #
  89 function TEST_corrupt_and_repair_replicated() {
  90     local dir=$1
  91     local poolname=rbd
  92
  93     setup $dir || return 1
  94     run_mon $dir a --osd_pool_default_size=2 || return 1
  95     run_mgr $dir x || return 1
  96     run_osd $dir 0 || return 1
  97     run_osd $dir 1 || return 1
  98     create_rbd_pool || return 1
  99     wait_for_clean || return 1
 100
 101     add_something $dir $poolname || return 1
 102     corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
 103     # Reproduces http://tracker.ceph.com/issues/8914
 104     corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1
 105
 106     teardown $dir || return 1
 107 }
 108
 109 function corrupt_and_repair_two() {
 110     local dir=$1
 111     local poolname=$2
 112     local first=$3
 113     local second=$4
 114
 115     #
 116     # 1) remove the corresponding file from the OSDs
 117     #
 118     pids=""
 119     run_in_background pids objectstore_tool $dir $first SOMETHING remove
 120     run_in_background pids objectstore_tool $dir $second SOMETHING remove
 121     wait_background pids
 122     return_code=$?
 123     if [ $return_code -ne 0 ]; then return $return_code; fi
 124
 125     #
 126     # 2) repair the PG
 127     #
 128     local pg=$(get_pg $poolname SOMETHING)
 129     repair $pg
 130     #
 131     # 3) The files must be back
 132     #
 133     pids=""
 134     run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs
 135     run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs
 136     wait_background pids
 137     return_code=$?
 138     if [ $return_code -ne 0 ]; then return $return_code; fi
 139
 140     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 141     diff $dir/ORIGINAL $dir/COPY || return 1
 142 }
 143
 144 #
 145 # 1) add an object
 146 # 2) remove the corresponding file from a designated OSD
 147 # 3) repair the PG
 148 # 4) check that the file has been restored in the designated OSD
 149 #
 150 function corrupt_and_repair_one() {
 151     local dir=$1
 152     local poolname=$2
 153     local osd=$3
 154
 155     #
 156     # 1) remove the corresponding file from the OSD
 157     #
 158     objectstore_tool $dir $osd SOMETHING remove || return 1
 159     #
 160     # 2) repair the PG
 161     #
 162     local pg=$(get_pg $poolname SOMETHING)
 163     repair $pg
 164     #
 165     # 3) The file must be back
 166     #
 167     objectstore_tool $dir $osd SOMETHING list-attrs || return 1
 168     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 169     diff $dir/ORIGINAL $dir/COPY || return 1
 170 }
 171
 172 function corrupt_and_repair_erasure_coded() {
 173     local dir=$1
 174     local poolname=$2
 175
 176     add_something $dir $poolname || return 1
 177
 178     local primary=$(get_primary $poolname SOMETHING)
 179     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 180     local not_primary_first=${osds[0]}
 181     local not_primary_second=${osds[1]}
 182
 183     # Reproduces http://tracker.ceph.com/issues/10017
 184     corrupt_and_repair_one $dir $poolname $primary  || return 1
 185     # Reproduces http://tracker.ceph.com/issues/10409
 186     corrupt_and_repair_one $dir $poolname $not_primary_first || return 1
 187     corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1
 188     corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1
 189
 190 }
 191
 192 function create_ec_pool() {
 193     local pool_name=$1
 194     local allow_overwrites=$2
 195
 196     ceph osd erasure-code-profile set myprofile crush-failure-domain=osd $3 $4 $5 $6 $7 || return 1
 197
 198     create_pool "$poolname" 1 1 erasure myprofile || return 1
 199
 200     if [ "$allow_overwrites" = "true" ]; then
 201         ceph osd pool set "$poolname" allow_ec_overwrites true || return 1
 202     fi
 203
 204     wait_for_clean || return 1
 205     return 0
 206 }
 207
 208 function auto_repair_erasure_coded() {
 209     local dir=$1
 210     local allow_overwrites=$2
 211     local poolname=ecpool
 212
 213     # Launch a cluster with 5 seconds scrub interval
 214     setup $dir || return 1
 215     run_mon $dir a || return 1
 216     run_mgr $dir x || return 1
 217     local ceph_osd_args="--osd-scrub-auto-repair=true \
 218             --osd-deep-scrub-interval=5 \
 219             --osd-scrub-max-interval=5 \
 220             --osd-scrub-min-interval=5 \
 221             --osd-scrub-interval-randomize-ratio=0"
 222     for id in $(seq 0 2) ; do
 223         if [ "$allow_overwrites" = "true" ]; then
 224             run_osd_bluestore $dir $id $ceph_osd_args || return 1
 225         else
 226             run_osd $dir $id $ceph_osd_args || return 1
 227         fi
 228     done
 229     create_rbd_pool || return 1
 230     wait_for_clean || return 1
 231
 232     # Create an EC pool
 233     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 234
 235     # Put an object
 236     local payload=ABCDEF
 237     echo $payload > $dir/ORIGINAL
 238     rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
 239
 240     # Remove the object from one shard physically
 241     # Restarted osd get $ceph_osd_args passed
 242     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
 243     # Wait for auto repair
 244     local pgid=$(get_pg $poolname SOMETHING)
 245     wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)"
 246     wait_for_clean || return 1
 247     # Verify - the file should be back
 248     # Restarted osd get $ceph_osd_args passed
 249     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
 250     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 251     diff $dir/ORIGINAL $dir/COPY || return 1
 252
 253     # Tear down
 254     teardown $dir || return 1
 255 }
 256
 257 function TEST_auto_repair_erasure_coded_appends() {
 258     auto_repair_erasure_coded $1 false
 259 }
 260
 261 function TEST_auto_repair_erasure_coded_overwrites() {
 262     if [ "$use_ec_overwrite" = "true" ]; then
 263         auto_repair_erasure_coded $1 true
 264     fi
 265 }
 266
 267 function corrupt_and_repair_jerasure() {
 268     local dir=$1
 269     local allow_overwrites=$2
 270     local poolname=ecpool
 271
 272     setup $dir || return 1
 273     run_mon $dir a || return 1
 274     run_mgr $dir x || return 1
 275     for id in $(seq 0 3) ; do
 276         if [ "$allow_overwrites" = "true" ]; then
 277             run_osd_bluestore $dir $id || return 1
 278         else
 279             run_osd $dir $id || return 1
 280         fi
 281     done
 282     create_rbd_pool || return 1
 283     wait_for_clean || return 1
 284
 285     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 286     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 287
 288     teardown $dir || return 1
 289 }
 290
 291 function TEST_corrupt_and_repair_jerasure_appends() {
 292     corrupt_and_repair_jerasure $1
 293 }
 294
 295 function TEST_corrupt_and_repair_jerasure_overwrites() {
 296     if [ "$use_ec_overwrite" = "true" ]; then
 297         corrupt_and_repair_jerasure $1 true
 298     fi
 299 }
 300
 301 function corrupt_and_repair_lrc() {
 302     local dir=$1
 303     local allow_overwrites=$2
 304     local poolname=ecpool
 305
 306     setup $dir || return 1
 307     run_mon $dir a || return 1
 308     run_mgr $dir x || return 1
 309     for id in $(seq 0 9) ; do
 310         if [ "$allow_overwrites" = "true" ]; then
 311             run_osd_bluestore $dir $id || return 1
 312         else
 313             run_osd $dir $id || return 1
 314         fi
 315     done
 316     create_rbd_pool || return 1
 317     wait_for_clean || return 1
 318
 319     create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1
 320     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 321
 322     teardown $dir || return 1
 323 }
 324
 325 function TEST_corrupt_and_repair_lrc_appends() {
 326     corrupt_and_repair_jerasure $1
 327 }
 328
 329 function TEST_corrupt_and_repair_lrc_overwrites() {
 330     if [ "$use_ec_overwrite" = "true" ]; then
 331         corrupt_and_repair_jerasure $1 true
 332     fi
 333 }
 334
 335 function unfound_erasure_coded() {
 336     local dir=$1
 337     local allow_overwrites=$2
 338     local poolname=ecpool
 339     local payload=ABCDEF
 340
 341     setup $dir || return 1
 342     run_mon $dir a || return 1
 343     run_mgr $dir x || return 1
 344     for id in $(seq 0 3) ; do
 345         if [ "$allow_overwrites" = "true" ]; then
 346             run_osd_bluestore $dir $id || return 1
 347         else
 348             run_osd $dir $id || return 1
 349         fi
 350     done
 351     create_rbd_pool || return 1
 352     wait_for_clean || return 1
 353
 354     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 355
 356     add_something $dir $poolname || return 1
 357
 358     local primary=$(get_primary $poolname SOMETHING)
 359     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 360     local not_primary_first=${osds[0]}
 361     local not_primary_second=${osds[1]}
 362     local not_primary_third=${osds[2]}
 363
 364     #
 365     # 1) remove the corresponding file from the OSDs
 366     #
 367     pids=""
 368     run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove
 369     run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove
 370     run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove
 371     wait_background pids
 372     return_code=$?
 373     if [ $return_code -ne 0 ]; then return $return_code; fi
 374
 375     #
 376     # 2) repair the PG
 377     #
 378     local pg=$(get_pg $poolname SOMETHING)
 379     repair $pg
 380     #
 381     # 3) check pg state
 382     #
 383     # it may take a bit to appear due to mon/mgr asynchrony
 384     for f in `seq 1 60`; do
 385         ceph -s | grep "1/1 objects unfound" && break
 386         sleep 1
 387     done
 388     ceph -s|grep "4 osds: 4 up, 4 in" || return 1
 389     ceph -s|grep "1/1 objects unfound" || return 1
 390
 391     teardown $dir || return 1
 392 }
 393
 394 function TEST_unfound_erasure_coded_appends() {
 395     unfound_erasure_coded $1
 396 }
 397
 398 function TEST_unfound_erasure_coded_overwrites() {
 399     if [ "$use_ec_overwrite" = "true" ]; then
 400         unfound_erasure_coded $1 true
 401     fi
 402 }
 403
 404 #
 405 # list_missing for EC pool
 406 #
 407 function list_missing_erasure_coded() {
 408     local dir=$1
 409     local allow_overwrites=$2
 410     local poolname=ecpool
 411
 412     setup $dir || return 1
 413     run_mon $dir a || return 1
 414     run_mgr $dir x || return 1
 415     for id in $(seq 0 2) ; do
 416         if [ "$allow_overwrites" = "true" ]; then
 417             run_osd_bluestore $dir $id || return 1
 418         else
 419             run_osd $dir $id || return 1
 420         fi
 421     done
 422     create_rbd_pool || return 1
 423     wait_for_clean || return 1
 424
 425     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 426
 427     # Put an object and remove the two shards (including primary)
 428     add_something $dir $poolname MOBJ0 || return 1
 429     local -a osds0=($(get_osds $poolname MOBJ0))
 430
 431     # Put another object and remove two shards (excluding primary)
 432     add_something $dir $poolname MOBJ1 || return 1
 433     local -a osds1=($(get_osds $poolname MOBJ1))
 434
 435     # Stop all osd daemons
 436     for id in $(seq 0 2) ; do
 437         kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1
 438     done
 439
 440     id=${osds0[0]}
 441     ceph-objectstore-tool --data-path $dir/$id \
 442         MOBJ0 remove || return 1
 443     id=${osds0[1]}
 444     ceph-objectstore-tool --data-path $dir/$id \
 445         MOBJ0 remove || return 1
 446
 447     id=${osds1[1]}
 448     ceph-objectstore-tool --data-path $dir/$id \
 449         MOBJ1 remove || return 1
 450     id=${osds1[2]}
 451     ceph-objectstore-tool --data-path $dir/$id \
 452         MOBJ1 remove || return 1
 453
 454     for id in $(seq 0 2) ; do
 455         activate_osd $dir $id >&2 || return 1
 456     done
 457     create_rbd_pool || return 1
 458     wait_for_clean || return 1
 459
 460     # Get get - both objects should in the same PG
 461     local pg=$(get_pg $poolname MOBJ0)
 462
 463     # Repair the PG, which triggers the recovering,
 464     # and should mark the object as unfound
 465     repair $pg
 466
 467     for i in $(seq 0 120) ; do
 468         [ $i -lt 60 ] || return 1
 469         matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l)
 470         [ $matches -eq 2 ] && break
 471     done
 472
 473     teardown $dir || return 1
 474 }
 475
 476 function TEST_list_missing_erasure_coded_appends() {
 477     list_missing_erasure_coded $1 false
 478 }
 479
 480 function TEST_list_missing_erasure_coded_overwrites() {
 481     if [ "$use_ec_overwrite" = "true" ]; then
 482         list_missing_erasure_coded $1 true
 483     fi
 484 }
 485
 486 #
 487 # Corrupt one copy of a replicated pool
 488 #
 489 function TEST_corrupt_scrub_replicated() {
 490     local dir=$1
 491     local poolname=csr_pool
 492     local total_objs=16
 493
 494     setup $dir || return 1
 495     run_mon $dir a --osd_pool_default_size=2 || return 1
 496     run_mgr $dir x || return 1
 497     run_osd $dir 0 || return 1
 498     run_osd $dir 1 || return 1
 499     create_rbd_pool || return 1
 500     wait_for_clean || return 1
 501
 502     create_pool foo 1 || return 1
 503     create_pool $poolname 1 1 || return 1
 504     wait_for_clean || return 1
 505
 506     for i in $(seq 1 $total_objs) ; do
 507         objname=ROBJ${i}
 508         add_something $dir $poolname $objname || return 1
 509
 510         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
 511         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
 512     done
 513
 514     local pg=$(get_pg $poolname ROBJ0)
 515
 516     # Compute an old omap digest and save oi
 517     CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \
 518         config set osd_deep_scrub_update_digest_min_age 0
 519     CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \
 520         config set osd_deep_scrub_update_digest_min_age 0
 521     pg_deep_scrub $pg
 522
 523     for i in $(seq 1 $total_objs) ; do
 524         objname=ROBJ${i}
 525
 526         # Alternate corruption between osd.0 and osd.1
 527         local osd=$(expr $i % 2)
 528
 529         case $i in
 530         1)
 531             # Size (deep scrub data_digest too)
 532             local payload=UVWXYZZZ
 533             echo $payload > $dir/CORRUPT
 534             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 535             ;;
 536
 537         2)
 538             # digest (deep scrub only)
 539             local payload=UVWXYZ
 540             echo $payload > $dir/CORRUPT
 541             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 542             ;;
 543
 544         3)
 545              # missing
 546              objectstore_tool $dir $osd $objname remove || return 1
 547              ;;
 548
 549          4)
 550              # Modify omap value (deep scrub only)
 551              objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1
 552              ;;
 553
 554          5)
 555             # Delete omap key (deep scrub only)
 556             objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1
 557             ;;
 558
 559          6)
 560             # Add extra omap key (deep scrub only)
 561             echo extra > $dir/extra-val
 562             objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1
 563             rm $dir/extra-val
 564             ;;
 565
 566          7)
 567             # Modify omap header (deep scrub only)
 568             echo -n newheader > $dir/hdr
 569             objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1
 570             rm $dir/hdr
 571             ;;
 572
 573          8)
 574             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
 575             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
 576
 577             # Break xattrs
 578             echo -n bad-val > $dir/bad-val
 579             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
 580             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
 581             echo -n val3-$objname > $dir/newval
 582             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
 583             rm $dir/bad-val $dir/newval
 584             ;;
 585
 586         9)
 587             objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi
 588             echo -n D > $dir/change
 589             rados --pool $poolname put $objname $dir/change
 590             objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi
 591             rm $dir/oi $dir/change
 592             ;;
 593
 594           # ROBJ10 must be handled after digests are re-computed by a deep scrub below
 595           # ROBJ11 must be handled with config change before deep scrub
 596           # ROBJ12 must be handled with config change before scrubs
 597           # ROBJ13 must be handled before scrubs
 598
 599         14)
 600             echo -n bad-val > $dir/bad-val
 601             objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1
 602             objectstore_tool $dir 1 $objname rm-attr _ || return 1
 603             rm $dir/bad-val
 604             ;;
 605
 606         15)
 607             objectstore_tool $dir $osd $objname rm-attr _ || return 1
 608             ;;
 609
 610         16)
 611             objectstore_tool $dir 0 $objname rm-attr snapset || return 1
 612             echo -n bad-val > $dir/bad-val
 613             objectstore_tool $dir 1 $objname set-attr snapset $dir/bad-val || return 1
 614
 615         esac
 616     done
 617
 618     local pg=$(get_pg $poolname ROBJ0)
 619
 620     inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
 621     inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
 622     inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
 623     inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
 624
 625     pg_scrub $pg
 626
 627     rados list-inconsistent-pg $poolname > $dir/json || return 1
 628     # Check pg count
 629     test $(jq '. | length' $dir/json) = "1" || return 1
 630     # Check pgid
 631     test $(jq -r '.[0]' $dir/json) = $pg || return 1
 632
 633     rados list-inconsistent-obj $pg > $dir/json || return 1
 634     # Get epoch for repair-get requests
 635     epoch=$(jq .epoch $dir/json)
 636
 637     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
 638 {
 639   "inconsistents": [
 640     {
 641       "shards": [
 642         {
 643           "size": 7,
 644           "errors": [],
 645           "osd": 0,
 646           "primary": false
 647         },
 648         {
 649           "object_info": {
 650             "oid": {
 651               "oid": "ROBJ1",
 652               "key": "",
 653               "snapid": -2,
 654               "hash": 1454963827,
 655               "max": 0,
 656               "pool": 3,
 657               "namespace": ""
 658             },
 659             "version": "51'58",
 660             "prior_version": "21'3",
 661             "last_reqid": "osd.1.0:57",
 662             "user_version": 3,
 663             "size": 7,
 664             "mtime": "",
 665             "local_mtime": "",
 666             "lost": 0,
 667             "flags": [
 668               "dirty",
 669               "omap",
 670               "data_digest",
 671               "omap_digest"
 672             ],
 673             "truncate_seq": 0,
 674             "truncate_size": 0,
 675             "data_digest": "0x2ddbf8f5",
 676             "omap_digest": "0xf5fba2c6",
 677             "expected_object_size": 0,
 678             "expected_write_size": 0,
 679             "alloc_hint_flags": 0,
 680             "manifest": {
 681               "type": 0
 682             },
 683             "watchers": {}
 684           },
 685           "size": 9,
 686           "errors": [
 687             "size_mismatch_info",
 688             "obj_size_info_mismatch"
 689           ],
 690           "osd": 1,
 691           "primary": true
 692         }
 693       ],
 694       "selected_object_info": {
 695         "oid": {
 696           "oid": "ROBJ1",
 697           "key": "",
 698           "snapid": -2,
 699           "hash": 1454963827,
 700           "max": 0,
 701           "pool": 3,
 702           "namespace": ""
 703         },
 704         "version": "51'58",
 705         "prior_version": "21'3",
 706         "last_reqid": "osd.1.0:57",
 707         "user_version": 3,
 708         "size": 7,
 709         "mtime": "2018-04-05 14:33:19.804040",
 710         "local_mtime": "2018-04-05 14:33:19.804839",
 711         "lost": 0,
 712         "flags": [
 713           "dirty",
 714           "omap",
 715           "data_digest",
 716           "omap_digest"
 717         ],
 718         "truncate_seq": 0,
 719         "truncate_size": 0,
 720         "data_digest": "0x2ddbf8f5",
 721         "omap_digest": "0xf5fba2c6",
 722         "expected_object_size": 0,
 723         "expected_write_size": 0,
 724         "alloc_hint_flags": 0,
 725         "manifest": {
 726           "type": 0
 727         },
 728         "watchers": {}
 729       },
 730       "union_shard_errors": [
 731         "size_mismatch_info",
 732         "obj_size_info_mismatch"
 733       ],
 734       "errors": [
 735         "size_mismatch"
 736       ],
 737       "object": {
 738         "version": 3,
 739         "snap": "head",
 740         "locator": "",
 741         "nspace": "",
 742         "name": "ROBJ1"
 743       }
 744     },
 745     {
 746       "shards": [
 747         {
 748           "errors": [
 749             "stat_error"
 750           ],
 751           "osd": 0,
 752           "primary": false
 753         },
 754         {
 755           "size": 7,
 756           "errors": [],
 757           "osd": 1,
 758           "primary": true
 759         }
 760       ],
 761       "selected_object_info": {
 762         "oid": {
 763           "oid": "ROBJ12",
 764           "key": "",
 765           "snapid": -2,
 766           "hash": 3920199997,
 767           "max": 0,
 768           "pool": 3,
 769           "namespace": ""
 770         },
 771         "version": "51'56",
 772         "prior_version": "43'36",
 773         "last_reqid": "osd.1.0:55",
 774         "user_version": 36,
 775         "size": 7,
 776         "mtime": "",
 777         "local_mtime": "",
 778         "lost": 0,
 779         "flags": [
 780           "dirty",
 781           "omap",
 782           "data_digest",
 783           "omap_digest"
 784         ],
 785         "truncate_seq": 0,
 786         "truncate_size": 0,
 787         "data_digest": "0x2ddbf8f5",
 788         "omap_digest": "0x067f306a",
 789         "expected_object_size": 0,
 790         "expected_write_size": 0,
 791         "alloc_hint_flags": 0,
 792         "manifest": {
 793           "type": 0
 794         },
 795         "watchers": {}
 796       },
 797       "union_shard_errors": [
 798         "stat_error"
 799       ],
 800       "errors": [],
 801       "object": {
 802         "version": 36,
 803         "snap": "head",
 804         "locator": "",
 805         "nspace": "",
 806         "name": "ROBJ12"
 807       }
 808     },
 809     {
 810       "shards": [
 811         {
 812           "errors": [
 813             "stat_error"
 814           ],
 815           "osd": 0,
 816           "primary": false
 817         },
 818         {
 819           "size": 7,
 820           "errors": [],
 821           "osd": 1,
 822           "primary": true
 823         }
 824       ],
 825       "selected_object_info": {
 826         "oid": {
 827           "oid": "ROBJ13",
 828           "key": "",
 829           "snapid": -2,
 830           "hash": 2682806379,
 831           "max": 0,
 832           "pool": 3,
 833           "namespace": ""
 834         },
 835         "version": "51'59",
 836         "prior_version": "45'39",
 837         "last_reqid": "osd.1.0:58",
 838         "user_version": 39,
 839         "size": 7,
 840         "mtime": "",
 841         "local_mtime": "",
 842         "lost": 0,
 843         "flags": [
 844           "dirty",
 845           "omap",
 846           "data_digest",
 847           "omap_digest"
 848         ],
 849         "truncate_seq": 0,
 850         "truncate_size": 0,
 851         "data_digest": "0x2ddbf8f5",
 852         "omap_digest": "0x6441854d",
 853         "expected_object_size": 0,
 854         "expected_write_size": 0,
 855         "alloc_hint_flags": 0,
 856         "manifest": {
 857           "type": 0
 858         },
 859         "watchers": {}
 860       },
 861       "union_shard_errors": [
 862         "stat_error"
 863       ],
 864       "errors": [],
 865       "object": {
 866         "version": 39,
 867         "snap": "head",
 868         "locator": "",
 869         "nspace": "",
 870         "name": "ROBJ13"
 871       }
 872     },
 873     {
 874       "shards": [
 875         {
 876           "object_info": "bad-val",
 877           "size": 7,
 878           "errors": [
 879             "info_corrupted"
 880           ],
 881           "osd": 0,
 882           "primary": false
 883         },
 884         {
 885           "size": 7,
 886           "errors": [
 887             "info_missing"
 888           ],
 889           "osd": 1,
 890           "primary": true
 891         }
 892       ],
 893       "union_shard_errors": [
 894         "info_missing",
 895         "info_corrupted"
 896       ],
 897       "errors": [],
 898       "object": {
 899         "version": 0,
 900         "snap": "head",
 901         "locator": "",
 902         "nspace": "",
 903         "name": "ROBJ14"
 904       }
 905     },
 906     {
 907       "shards": [
 908         {
 909           "object_info": {
 910             "oid": {
 911               "oid": "ROBJ15",
 912               "key": "",
 913               "snapid": -2,
 914               "hash": 504996876,
 915               "max": 0,
 916               "pool": 3,
 917               "namespace": ""
 918             },
 919             "version": "51'49",
 920             "prior_version": "49'45",
 921             "last_reqid": "osd.1.0:48",
 922             "user_version": 45,
 923             "size": 7,
 924             "mtime": "2018-04-05 14:33:29.498969",
 925             "local_mtime": "2018-04-05 14:33:29.499890",
 926             "lost": 0,
 927             "flags": [
 928               "dirty",
 929               "omap",
 930               "data_digest",
 931               "omap_digest"
 932             ],
 933             "truncate_seq": 0,
 934             "truncate_size": 0,
 935             "data_digest": "0x2ddbf8f5",
 936             "omap_digest": "0x2d2a4d6e",
 937             "expected_object_size": 0,
 938             "expected_write_size": 0,
 939             "alloc_hint_flags": 0,
 940             "manifest": {
 941               "type": 0
 942             },
 943             "watchers": {}
 944           },
 945           "size": 7,
 946           "errors": [],
 947           "osd": 0,
 948           "primary": false
 949         },
 950         {
 951           "size": 7,
 952           "errors": [
 953             "info_missing"
 954           ],
 955           "osd": 1,
 956           "primary": true
 957         }
 958       ],
 959       "selected_object_info": {
 960         "oid": {
 961           "oid": "ROBJ15",
 962           "key": "",
 963           "snapid": -2,
 964           "hash": 504996876,
 965           "max": 0,
 966           "pool": 3,
 967           "namespace": ""
 968         },
 969         "version": "51'49",
 970         "prior_version": "49'45",
 971         "last_reqid": "osd.1.0:48",
 972         "user_version": 45,
 973         "size": 7,
 974         "mtime": "",
 975         "local_mtime": "",
 976         "lost": 0,
 977         "flags": [
 978           "dirty",
 979           "omap",
 980           "data_digest",
 981           "omap_digest"
 982         ],
 983         "truncate_seq": 0,
 984         "truncate_size": 0,
 985         "data_digest": "0x2ddbf8f5",
 986         "omap_digest": "0x2d2a4d6e",
 987         "expected_object_size": 0,
 988         "expected_write_size": 0,
 989         "alloc_hint_flags": 0,
 990         "manifest": {
 991           "type": 0
 992         },
 993         "watchers": {}
 994       },
 995       "union_shard_errors": [
 996         "info_missing"
 997       ],
 998       "errors": [],
 999       "object": {
1000         "version": 45,
1001         "snap": "head",
1002         "locator": "",
1003         "nspace": "",
1004         "name": "ROBJ15"
1005       }
1006     },
1007     {
1008       "errors": [],
1009       "object": {
1010       "locator": "",
1011       "name": "ROBJ16",
1012       "nspace": "",
1013       "snap": "head",
1014       "version": 0
1015        },
1016         "shards": [
1017       {
1018         "errors": [
1019           "snapset_missing"
1020         ],
1021         "osd": 0,
1022         "primary": false,
1023         "size": 7
1024       },
1025       {
1026         "errors": [
1027           "snapset_corrupted"
1028         ],
1029         "osd": 1,
1030         "primary": true,
1031         "snapset": "bad-val",
1032         "size": 7
1033       }
1034       ],
1035       "union_shard_errors": [
1036         "snapset_missing",
1037         "snapset_corrupted"
1038       ]
1039     },
1040     {
1041       "shards": [
1042         {
1043           "size": 7,
1044           "errors": [],
1045           "osd": 0,
1046           "primary": false
1047         },
1048         {
1049           "errors": [
1050             "missing"
1051           ],
1052           "osd": 1,
1053           "primary": true
1054         }
1055       ],
1056       "selected_object_info": {
1057         "oid": {
1058           "oid": "ROBJ3",
1059           "key": "",
1060           "snapid": -2,
1061           "hash": 625845583,
1062           "max": 0,
1063           "pool": 3,
1064           "namespace": ""
1065         },
1066         "version": "51'61",
1067         "prior_version": "25'9",
1068         "last_reqid": "osd.1.0:60",
1069         "user_version": 9,
1070         "size": 7,
1071         "mtime": "",
1072         "local_mtime": "",
1073         "lost": 0,
1074         "flags": [
1075           "dirty",
1076           "omap",
1077           "data_digest",
1078           "omap_digest"
1079         ],
1080         "truncate_seq": 0,
1081         "truncate_size": 0,
1082         "data_digest": "0x2ddbf8f5",
1083         "omap_digest": "0x00b35dfd",
1084         "expected_object_size": 0,
1085         "expected_write_size": 0,
1086         "alloc_hint_flags": 0,
1087         "manifest": {
1088           "type": 0
1089         },
1090         "watchers": {}
1091       },
1092       "union_shard_errors": [
1093         "missing"
1094       ],
1095       "errors": [],
1096       "object": {
1097         "version": 9,
1098         "snap": "head",
1099         "locator": "",
1100         "nspace": "",
1101         "name": "ROBJ3"
1102       }
1103     },
1104     {
1105       "shards": [
1106         {
1107           "attrs": [
1108             {
1109               "Base64": false,
1110               "value": "bad-val",
1111               "name": "key1-ROBJ8"
1112             },
1113             {
1114               "Base64": false,
1115               "value": "val2-ROBJ8",
1116               "name": "key2-ROBJ8"
1117             }
1118           ],
1119           "size": 7,
1120           "errors": [],
1121           "osd": 0,
1122           "primary": false
1123         },
1124         {
1125           "attrs": [
1126             {
1127               "Base64": false,
1128               "value": "val1-ROBJ8",
1129               "name": "key1-ROBJ8"
1130             },
1131             {
1132               "Base64": false,
1133               "value": "val3-ROBJ8",
1134               "name": "key3-ROBJ8"
1135             }
1136           ],
1137           "size": 7,
1138           "errors": [],
1139           "osd": 1,
1140           "primary": true
1141         }
1142       ],
1143       "selected_object_info": {
1144         "oid": {
1145           "oid": "ROBJ8",
1146           "key": "",
1147           "snapid": -2,
1148           "hash": 2359695969,
1149           "max": 0,
1150           "pool": 3,
1151           "namespace": ""
1152         },
1153         "version": "79'66",
1154         "prior_version": "79'65",
1155         "last_reqid": "client.4554.0:1",
1156         "user_version": 66,
1157         "size": 7,
1158         "mtime": "",
1159         "local_mtime": "",
1160         "lost": 0,
1161         "flags": [
1162           "dirty",
1163           "omap",
1164           "data_digest",
1165           "omap_digest"
1166         ],
1167         "truncate_seq": 0,
1168         "truncate_size": 0,
1169         "data_digest": "0x2ddbf8f5",
1170         "omap_digest": "0xd6be81dc",
1171         "expected_object_size": 0,
1172         "expected_write_size": 0,
1173         "alloc_hint_flags": 0,
1174         "manifest": {
1175           "type": 0
1176         },
1177         "watchers": {}
1178       },
1179       "union_shard_errors": [],
1180       "errors": [
1181         "attr_value_mismatch",
1182         "attr_name_mismatch"
1183       ],
1184       "object": {
1185         "version": 66,
1186         "snap": "head",
1187         "locator": "",
1188         "nspace": "",
1189         "name": "ROBJ8"
1190       }
1191     },
1192     {
1193       "shards": [
1194         {
1195           "object_info": {
1196             "oid": {
1197               "oid": "ROBJ9",
1198               "key": "",
1199               "snapid": -2,
1200               "hash": 537189375,
1201               "max": 0,
1202               "pool": 3,
1203               "namespace": ""
1204             },
1205             "version": "95'67",
1206             "prior_version": "51'64",
1207             "last_reqid": "client.4649.0:1",
1208             "user_version": 67,
1209             "size": 1,
1210             "mtime": "",
1211             "local_mtime": "",
1212             "lost": 0,
1213             "flags": [
1214               "dirty",
1215               "omap",
1216               "data_digest",
1217               "omap_digest"
1218             ],
1219             "truncate_seq": 0,
1220             "truncate_size": 0,
1221             "data_digest": "0x2b63260d",
1222             "omap_digest": "0x2eecc539",
1223             "expected_object_size": 0,
1224             "expected_write_size": 0,
1225             "alloc_hint_flags": 0,
1226             "manifest": {
1227               "type": 0
1228             },
1229             "watchers": {}
1230           },
1231           "size": 1,
1232           "errors": [],
1233           "osd": 0,
1234           "primary": false
1235         },
1236         {
1237           "object_info": {
1238             "oid": {
1239               "oid": "ROBJ9",
1240               "key": "",
1241               "snapid": -2,
1242               "hash": 537189375,
1243               "max": 0,
1244               "pool": 3,
1245               "namespace": ""
1246             },
1247             "version": "51'64",
1248             "prior_version": "37'27",
1249             "last_reqid": "osd.1.0:63",
1250             "user_version": 27,
1251             "size": 7,
1252             "mtime": "2018-04-05 14:33:25.352485",
1253             "local_mtime": "2018-04-05 14:33:25.353746",
1254             "lost": 0,
1255             "flags": [
1256               "dirty",
1257               "omap",
1258               "data_digest",
1259               "omap_digest"
1260             ],
1261             "truncate_seq": 0,
1262             "truncate_size": 0,
1263             "data_digest": "0x2ddbf8f5",
1264             "omap_digest": "0x2eecc539",
1265             "expected_object_size": 0,
1266             "expected_write_size": 0,
1267             "alloc_hint_flags": 0,
1268             "manifest": {
1269               "type": 0
1270             },
1271             "watchers": {}
1272           },
1273           "size": 1,
1274           "errors": [
1275             "obj_size_info_mismatch"
1276           ],
1277           "osd": 1,
1278           "primary": true
1279         }
1280       ],
1281       "selected_object_info": {
1282         "oid": {
1283           "oid": "ROBJ9",
1284           "key": "",
1285           "snapid": -2,
1286           "hash": 537189375,
1287           "max": 0,
1288           "pool": 3,
1289           "namespace": ""
1290         },
1291         "version": "95'67",
1292         "prior_version": "51'64",
1293         "last_reqid": "client.4649.0:1",
1294         "user_version": 67,
1295         "size": 1,
1296         "mtime": "",
1297         "local_mtime": "",
1298         "lost": 0,
1299         "flags": [
1300           "dirty",
1301           "omap",
1302           "data_digest",
1303           "omap_digest"
1304         ],
1305         "truncate_seq": 0,
1306         "truncate_size": 0,
1307         "data_digest": "0x2b63260d",
1308         "omap_digest": "0x2eecc539",
1309         "expected_object_size": 0,
1310         "expected_write_size": 0,
1311         "alloc_hint_flags": 0,
1312         "manifest": {
1313           "type": 0
1314         },
1315         "watchers": {}
1316       },
1317       "union_shard_errors": [
1318          "obj_size_info_mismatch"
1319       ],
1320       "errors": [
1321         "object_info_inconsistency"
1322       ],
1323       "object": {
1324         "version": 67,
1325         "snap": "head",
1326         "locator": "",
1327         "nspace": "",
1328         "name": "ROBJ9"
1329       }
1330     }
1331   ],
1332   "epoch": 0
1333 }
1334 EOF
1335
1336     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
1337     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
1338     if test $getjson = "yes"
1339     then
1340         jq '.' $dir/json > save1.json
1341     fi
1342
1343     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
1344     then
1345       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
1346     fi
1347
1348     objname=ROBJ9
1349     # Change data and size again because digest was recomputed
1350     echo -n ZZZ > $dir/change
1351     rados --pool $poolname put $objname $dir/change
1352     # Set one to an even older value
1353     objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi
1354     rm $dir/oi $dir/change
1355
1356     objname=ROBJ10
1357     objectstore_tool $dir 1 $objname get-attr _ > $dir/oi
1358     rados --pool $poolname setomapval $objname key2-$objname val2-$objname
1359     objectstore_tool $dir 0 $objname set-attr _ $dir/oi
1360     objectstore_tool $dir 1 $objname set-attr _ $dir/oi
1361     rm $dir/oi
1362
1363     inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1364     inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1365     inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1366     inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1367     pg_deep_scrub $pg
1368
1369     rados list-inconsistent-pg $poolname > $dir/json || return 1
1370     # Check pg count
1371     test $(jq '. | length' $dir/json) = "1" || return 1
1372     # Check pgid
1373     test $(jq -r '.[0]' $dir/json) = $pg || return 1
1374
1375     rados list-inconsistent-obj $pg > $dir/json || return 1
1376     # Get epoch for repair-get requests
1377     epoch=$(jq .epoch $dir/json)
1378
1379     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
1380 {
1381   "inconsistents": [
1382     {
1383       "shards": [
1384         {
1385           "data_digest": "0x2ddbf8f5",
1386           "omap_digest": "0xf5fba2c6",
1387           "size": 7,
1388           "errors": [],
1389           "osd": 0,
1390           "primary": false
1391         },
1392         {
1393           "object_info": {
1394             "oid": {
1395               "oid": "ROBJ1",
1396               "key": "",
1397               "snapid": -2,
1398               "hash": 1454963827,
1399               "max": 0,
1400               "pool": 3,
1401               "namespace": ""
1402             },
1403             "version": "51'58",
1404             "prior_version": "21'3",
1405             "last_reqid": "osd.1.0:57",
1406             "user_version": 3,
1407             "size": 7,
1408             "mtime": "2018-04-05 14:33:19.804040",
1409             "local_mtime": "2018-04-05 14:33:19.804839",
1410             "lost": 0,
1411             "flags": [
1412               "dirty",
1413               "omap",
1414               "data_digest",
1415               "omap_digest"
1416             ],
1417             "truncate_seq": 0,
1418             "truncate_size": 0,
1419             "data_digest": "0x2ddbf8f5",
1420             "omap_digest": "0xf5fba2c6",
1421             "expected_object_size": 0,
1422             "expected_write_size": 0,
1423             "alloc_hint_flags": 0,
1424             "manifest": {
1425               "type": 0
1426             },
1427             "watchers": {}
1428           },
1429           "data_digest": "0x2d4a11c2",
1430           "omap_digest": "0xf5fba2c6",
1431           "size": 9,
1432           "errors": [
1433             "data_digest_mismatch_info",
1434             "size_mismatch_info",
1435             "obj_size_info_mismatch"
1436           ],
1437           "osd": 1,
1438           "primary": true
1439         }
1440       ],
1441       "selected_object_info": {
1442         "oid": {
1443           "oid": "ROBJ1",
1444           "key": "",
1445           "snapid": -2,
1446           "hash": 1454963827,
1447           "max": 0,
1448           "pool": 3,
1449           "namespace": ""
1450         },
1451         "version": "51'58",
1452         "prior_version": "21'3",
1453         "last_reqid": "osd.1.0:57",
1454         "user_version": 3,
1455         "size": 7,
1456         "mtime": "2018-04-05 14:33:19.804040",
1457         "local_mtime": "2018-04-05 14:33:19.804839",
1458         "lost": 0,
1459         "flags": [
1460           "dirty",
1461           "omap",
1462           "data_digest",
1463           "omap_digest"
1464         ],
1465         "truncate_seq": 0,
1466         "truncate_size": 0,
1467         "data_digest": "0x2ddbf8f5",
1468         "omap_digest": "0xf5fba2c6",
1469         "expected_object_size": 0,
1470         "expected_write_size": 0,
1471         "alloc_hint_flags": 0,
1472         "manifest": {
1473           "type": 0
1474         },
1475         "watchers": {}
1476       },
1477       "union_shard_errors": [
1478         "data_digest_mismatch_info",
1479         "size_mismatch_info",
1480         "obj_size_info_mismatch"
1481       ],
1482       "errors": [
1483         "data_digest_mismatch",
1484         "size_mismatch"
1485       ],
1486       "object": {
1487         "version": 3,
1488         "snap": "head",
1489         "locator": "",
1490         "nspace": "",
1491         "name": "ROBJ1"
1492       }
1493     },
1494     {
1495       "shards": [
1496         {
1497           "data_digest": "0x2ddbf8f5",
1498           "omap_digest": "0xa8dd5adc",
1499           "size": 7,
1500           "errors": [
1501             "omap_digest_mismatch_info"
1502           ],
1503           "osd": 0,
1504           "primary": false
1505         },
1506         {
1507           "data_digest": "0x2ddbf8f5",
1508           "omap_digest": "0xa8dd5adc",
1509           "size": 7,
1510           "errors": [
1511             "omap_digest_mismatch_info"
1512           ],
1513           "osd": 1,
1514           "primary": true
1515         }
1516       ],
1517      "selected_object_info": {
1518        "alloc_hint_flags": 0,
1519        "data_digest": "0x2ddbf8f5",
1520        "expected_object_size": 0,
1521        "expected_write_size": 0,
1522        "flags": [
1523          "dirty",
1524          "omap",
1525          "data_digest",
1526          "omap_digest"
1527        ],
1528        "last_reqid": "osd.0.0:50",
1529        "local_mtime": "2018-04-05 14:33:26.762368",
1530        "lost": 0,
1531       "manifest": {
1532          "type": 0
1533        },
1534        "mtime": "2018-04-05 14:33:26.762368",
1535        "oid": {
1536          "hash": 3174666125,
1537          "key": "",
1538          "max": 0,
1539          "namespace": "",
1540          "oid": "ROBJ10",
1541          "pool": 3,
1542          "snapid": -2
1543        },
1544        "omap_digest": "0xc2025a24",
1545        "prior_version": "41'33",
1546        "size": 7,
1547        "truncate_seq": 0,
1548        "truncate_size": 0,
1549        "user_version": 30,
1550        "version": "47'51",
1551        "watchers": {}
1552       },
1553       "union_shard_errors": [
1554         "omap_digest_mismatch_info"
1555       ],
1556       "errors": [],
1557       "object": {
1558         "version": 30,
1559         "snap": "head",
1560         "locator": "",
1561         "nspace": "",
1562         "name": "ROBJ10"
1563       }
1564     },
1565     {
1566       "shards": [
1567         {
1568           "data_digest": "0x2ddbf8f5",
1569           "omap_digest": "0xa03cef03",
1570           "size": 7,
1571           "errors": [],
1572           "osd": 0,
1573           "primary": false
1574         },
1575         {
1576           "size": 7,
1577           "errors": [
1578             "read_error"
1579           ],
1580           "osd": 1,
1581           "primary": true
1582         }
1583       ],
1584       "selected_object_info": {
1585         "oid": {
1586           "oid": "ROBJ11",
1587           "key": "",
1588           "snapid": -2,
1589           "hash": 1828574689,
1590           "max": 0,
1591           "pool": 3,
1592           "namespace": ""
1593         },
1594         "version": "51'52",
1595         "prior_version": "41'33",
1596         "last_reqid": "osd.1.0:51",
1597         "user_version": 33,
1598         "size": 7,
1599         "mtime": "2018-04-05 14:33:26.761286",
1600         "local_mtime": "2018-04-05 14:33:26.762368",
1601         "lost": 0,
1602         "flags": [
1603           "dirty",
1604           "omap",
1605           "data_digest",
1606           "omap_digest"
1607         ],
1608         "truncate_seq": 0,
1609         "truncate_size": 0,
1610         "data_digest": "0x2ddbf8f5",
1611         "omap_digest": "0xa03cef03",
1612         "expected_object_size": 0,
1613         "expected_write_size": 0,
1614         "alloc_hint_flags": 0,
1615         "manifest": {
1616           "type": 0
1617         },
1618         "watchers": {}
1619       },
1620       "union_shard_errors": [
1621         "read_error"
1622       ],
1623       "errors": [],
1624       "object": {
1625         "version": 33,
1626         "snap": "head",
1627         "locator": "",
1628         "nspace": "",
1629         "name": "ROBJ11"
1630       }
1631     },
1632     {
1633       "shards": [
1634         {
1635           "errors": [
1636             "stat_error"
1637           ],
1638           "osd": 0,
1639           "primary": false
1640         },
1641         {
1642           "data_digest": "0x2ddbf8f5",
1643           "omap_digest": "0x067f306a",
1644           "size": 7,
1645           "errors": [],
1646           "osd": 1,
1647           "primary": true
1648         }
1649       ],
1650       "selected_object_info": {
1651         "oid": {
1652           "oid": "ROBJ12",
1653           "key": "",
1654           "snapid": -2,
1655           "hash": 3920199997,
1656           "max": 0,
1657           "pool": 3,
1658           "namespace": ""
1659         },
1660         "version": "51'56",
1661         "prior_version": "43'36",
1662         "last_reqid": "osd.1.0:55",
1663         "user_version": 36,
1664         "size": 7,
1665         "mtime": "2018-04-05 14:33:27.460958",
1666         "local_mtime": "2018-04-05 14:33:27.462109",
1667         "lost": 0,
1668         "flags": [
1669           "dirty",
1670           "omap",
1671           "data_digest",
1672           "omap_digest"
1673         ],
1674         "truncate_seq": 0,
1675         "truncate_size": 0,
1676         "data_digest": "0x2ddbf8f5",
1677         "omap_digest": "0x067f306a",
1678         "expected_object_size": 0,
1679         "expected_write_size": 0,
1680         "alloc_hint_flags": 0,
1681         "manifest": {
1682           "type": 0
1683         },
1684         "watchers": {}
1685       },
1686       "union_shard_errors": [
1687         "stat_error"
1688       ],
1689       "errors": [],
1690       "object": {
1691         "version": 36,
1692         "snap": "head",
1693         "locator": "",
1694         "nspace": "",
1695         "name": "ROBJ12"
1696       }
1697     },
1698     {
1699       "shards": [
1700         {
1701           "errors": [
1702             "stat_error"
1703           ],
1704           "osd": 0,
1705           "primary": false
1706         },
1707         {
1708           "size": 7,
1709           "errors": [
1710             "read_error"
1711           ],
1712           "osd": 1,
1713           "primary": true
1714         }
1715       ],
1716       "union_shard_errors": [
1717         "stat_error",
1718         "read_error"
1719       ],
1720       "errors": [],
1721       "object": {
1722         "version": 0,
1723         "snap": "head",
1724         "locator": "",
1725         "nspace": "",
1726         "name": "ROBJ13"
1727       }
1728     },
1729     {
1730       "shards": [
1731         {
1732           "object_info": "bad-val",
1733           "data_digest": "0x2ddbf8f5",
1734           "omap_digest": "0x4f14f849",
1735           "size": 7,
1736           "errors": [
1737             "info_corrupted"
1738           ],
1739           "osd": 0,
1740           "primary": false
1741         },
1742         {
1743           "data_digest": "0x2ddbf8f5",
1744           "omap_digest": "0x4f14f849",
1745           "size": 7,
1746           "errors": [
1747             "info_missing"
1748           ],
1749           "osd": 1,
1750           "primary": true
1751         }
1752       ],
1753       "union_shard_errors": [
1754         "info_missing",
1755         "info_corrupted"
1756       ],
1757       "errors": [],
1758       "object": {
1759         "version": 0,
1760         "snap": "head",
1761         "locator": "",
1762         "nspace": "",
1763         "name": "ROBJ14"
1764       }
1765     },
1766     {
1767       "shards": [
1768         {
1769           "object_info": {
1770             "oid": {
1771               "oid": "ROBJ15",
1772               "key": "",
1773               "snapid": -2,
1774               "hash": 504996876,
1775               "max": 0,
1776               "pool": 3,
1777               "namespace": ""
1778             },
1779             "version": "51'49",
1780             "prior_version": "49'45",
1781             "last_reqid": "osd.1.0:48",
1782             "user_version": 45,
1783             "size": 7,
1784             "mtime": "2018-04-05 14:33:29.498969",
1785             "local_mtime": "2018-04-05 14:33:29.499890",
1786             "lost": 0,
1787             "flags": [
1788               "dirty",
1789               "omap",
1790               "data_digest",
1791               "omap_digest"
1792             ],
1793             "truncate_seq": 0,
1794             "truncate_size": 0,
1795             "data_digest": "0x2ddbf8f5",
1796             "omap_digest": "0x2d2a4d6e",
1797             "expected_object_size": 0,
1798             "expected_write_size": 0,
1799             "alloc_hint_flags": 0,
1800             "manifest": {
1801               "type": 0
1802             },
1803             "watchers": {}
1804           },
1805           "data_digest": "0x2ddbf8f5",
1806           "omap_digest": "0x2d2a4d6e",
1807           "size": 7,
1808           "errors": [],
1809           "osd": 0,
1810           "primary": false
1811         },
1812         {
1813           "data_digest": "0x2ddbf8f5",
1814           "omap_digest": "0x2d2a4d6e",
1815           "size": 7,
1816           "errors": [
1817             "info_missing"
1818           ],
1819           "osd": 1,
1820           "primary": true
1821         }
1822       ],
1823       "selected_object_info": {
1824         "oid": {
1825           "oid": "ROBJ15",
1826           "key": "",
1827           "snapid": -2,
1828           "hash": 504996876,
1829           "max": 0,
1830           "pool": 3,
1831           "namespace": ""
1832         },
1833         "version": "51'49",
1834         "prior_version": "49'45",
1835         "last_reqid": "osd.1.0:48",
1836         "user_version": 45,
1837         "size": 7,
1838         "mtime": "2018-04-05 14:33:29.498969",
1839         "local_mtime": "2018-04-05 14:33:29.499890",
1840         "lost": 0,
1841         "flags": [
1842           "dirty",
1843           "omap",
1844           "data_digest",
1845           "omap_digest"
1846         ],
1847         "truncate_seq": 0,
1848         "truncate_size": 0,
1849         "data_digest": "0x2ddbf8f5",
1850         "omap_digest": "0x2d2a4d6e",
1851         "expected_object_size": 0,
1852         "expected_write_size": 0,
1853         "alloc_hint_flags": 0,
1854         "manifest": {
1855           "type": 0
1856         },
1857         "watchers": {}
1858       },
1859       "union_shard_errors": [
1860         "info_missing"
1861       ],
1862       "errors": [],
1863       "object": {
1864         "version": 45,
1865         "snap": "head",
1866         "locator": "",
1867         "nspace": "",
1868         "name": "ROBJ15"
1869       }
1870     },
1871     {
1872       "errors": [],
1873       "object": {
1874       "locator": "",
1875       "name": "ROBJ16",
1876       "nspace": "",
1877       "snap": "head",
1878       "version": 0
1879        },
1880         "shards": [
1881       {
1882         "data_digest": "0x2ddbf8f5",
1883         "errors": [
1884           "snapset_missing"
1885         ],
1886         "omap_digest": "0x8b699207",
1887         "osd": 0,
1888         "primary": false,
1889         "size": 7
1890       },
1891       {
1892         "snapset": "bad-val",
1893         "data_digest": "0x2ddbf8f5",
1894         "errors": [
1895           "snapset_corrupted"
1896         ],
1897         "omap_digest": "0x8b699207",
1898         "osd": 1,
1899         "primary": true,
1900         "size": 7
1901       }
1902       ],
1903       "union_shard_errors": [
1904         "snapset_missing",
1905         "snapset_corrupted"
1906       ]
1907     },
1908     {
1909       "shards": [
1910         {
1911           "data_digest": "0x578a4830",
1912           "omap_digest": "0xf8e11918",
1913           "size": 7,
1914           "errors": [
1915             "data_digest_mismatch_info"
1916           ],
1917           "osd": 0,
1918           "primary": false
1919         },
1920         {
1921           "data_digest": "0x2ddbf8f5",
1922           "omap_digest": "0xf8e11918",
1923           "size": 7,
1924           "errors": [],
1925           "osd": 1,
1926           "primary": true
1927         }
1928       ],
1929       "selected_object_info": {
1930         "oid": {
1931           "oid": "ROBJ2",
1932           "key": "",
1933           "snapid": -2,
1934           "hash": 2026323607,
1935           "max": 0,
1936           "pool": 3,
1937           "namespace": ""
1938         },
1939         "version": "51'60",
1940         "prior_version": "23'6",
1941         "last_reqid": "osd.1.0:59",
1942         "user_version": 6,
1943         "size": 7,
1944         "mtime": "2018-04-05 14:33:20.498756",
1945         "local_mtime": "2018-04-05 14:33:20.499704",
1946         "lost": 0,
1947         "flags": [
1948           "dirty",
1949           "omap",
1950           "data_digest",
1951           "omap_digest"
1952         ],
1953         "truncate_seq": 0,
1954         "truncate_size": 0,
1955         "data_digest": "0x2ddbf8f5",
1956         "omap_digest": "0xf8e11918",
1957         "expected_object_size": 0,
1958         "expected_write_size": 0,
1959         "alloc_hint_flags": 0,
1960         "manifest": {
1961           "type": 0
1962         },
1963         "watchers": {}
1964       },
1965       "union_shard_errors": [
1966         "data_digest_mismatch_info"
1967       ],
1968       "errors": [
1969         "data_digest_mismatch"
1970       ],
1971       "object": {
1972         "version": 6,
1973         "snap": "head",
1974         "locator": "",
1975         "nspace": "",
1976         "name": "ROBJ2"
1977       }
1978     },
1979     {
1980       "shards": [
1981         {
1982           "data_digest": "0x2ddbf8f5",
1983           "omap_digest": "0x00b35dfd",
1984           "size": 7,
1985           "errors": [],
1986           "osd": 0,
1987           "primary": false
1988         },
1989         {
1990           "errors": [
1991             "missing"
1992           ],
1993           "osd": 1,
1994           "primary": true
1995         }
1996       ],
1997       "selected_object_info": {
1998         "oid": {
1999           "oid": "ROBJ3",
2000           "key": "",
2001           "snapid": -2,
2002           "hash": 625845583,
2003           "max": 0,
2004           "pool": 3,
2005           "namespace": ""
2006         },
2007         "version": "51'61",
2008         "prior_version": "25'9",
2009         "last_reqid": "osd.1.0:60",
2010         "user_version": 9,
2011         "size": 7,
2012         "mtime": "2018-04-05 14:33:21.189382",
2013         "local_mtime": "2018-04-05 14:33:21.190446",
2014         "lost": 0,
2015         "flags": [
2016           "dirty",
2017           "omap",
2018           "data_digest",
2019           "omap_digest"
2020         ],
2021         "truncate_seq": 0,
2022         "truncate_size": 0,
2023         "data_digest": "0x2ddbf8f5",
2024         "omap_digest": "0x00b35dfd",
2025         "expected_object_size": 0,
2026         "expected_write_size": 0,
2027         "alloc_hint_flags": 0,
2028         "manifest": {
2029           "type": 0
2030         },
2031         "watchers": {}
2032       },
2033       "union_shard_errors": [
2034         "missing"
2035       ],
2036       "errors": [],
2037       "object": {
2038         "version": 9,
2039         "snap": "head",
2040         "locator": "",
2041         "nspace": "",
2042         "name": "ROBJ3"
2043       }
2044     },
2045     {
2046       "shards": [
2047         {
2048           "data_digest": "0x2ddbf8f5",
2049           "omap_digest": "0xd7178dfe",
2050           "size": 7,
2051           "errors": [
2052             "omap_digest_mismatch_info"
2053           ],
2054           "osd": 0,
2055           "primary": false
2056         },
2057         {
2058           "data_digest": "0x2ddbf8f5",
2059           "omap_digest": "0xe2d46ea4",
2060           "size": 7,
2061           "errors": [],
2062           "osd": 1,
2063           "primary": true
2064         }
2065       ],
2066       "selected_object_info": {
2067         "oid": {
2068           "oid": "ROBJ4",
2069           "key": "",
2070           "snapid": -2,
2071           "hash": 2360875311,
2072           "max": 0,
2073           "pool": 3,
2074           "namespace": ""
2075         },
2076         "version": "51'62",
2077         "prior_version": "27'12",
2078         "last_reqid": "osd.1.0:61",
2079         "user_version": 12,
2080         "size": 7,
2081         "mtime": "2018-04-05 14:33:21.862313",
2082         "local_mtime": "2018-04-05 14:33:21.863261",
2083         "lost": 0,
2084         "flags": [
2085           "dirty",
2086           "omap",
2087           "data_digest",
2088           "omap_digest"
2089         ],
2090         "truncate_seq": 0,
2091         "truncate_size": 0,
2092         "data_digest": "0x2ddbf8f5",
2093         "omap_digest": "0xe2d46ea4",
2094         "expected_object_size": 0,
2095         "expected_write_size": 0,
2096         "alloc_hint_flags": 0,
2097         "manifest": {
2098           "type": 0
2099         },
2100         "watchers": {}
2101       },
2102       "union_shard_errors": [
2103         "omap_digest_mismatch_info"
2104       ],
2105       "errors": [
2106         "omap_digest_mismatch"
2107       ],
2108       "object": {
2109         "version": 12,
2110         "snap": "head",
2111         "locator": "",
2112         "nspace": "",
2113         "name": "ROBJ4"
2114       }
2115     },
2116     {
2117       "shards": [
2118         {
2119           "data_digest": "0x2ddbf8f5",
2120           "omap_digest": "0x1a862a41",
2121           "size": 7,
2122           "errors": [],
2123           "osd": 0,
2124           "primary": false
2125         },
2126         {
2127           "data_digest": "0x2ddbf8f5",
2128           "omap_digest": "0x06cac8f6",
2129           "size": 7,
2130           "errors": [
2131             "omap_digest_mismatch_info"
2132           ],
2133           "osd": 1,
2134           "primary": true
2135         }
2136       ],
2137       "selected_object_info": {
2138         "oid": {
2139           "oid": "ROBJ5",
2140           "key": "",
2141           "snapid": -2,
2142           "hash": 2334915887,
2143           "max": 0,
2144           "pool": 3,
2145           "namespace": ""
2146         },
2147         "version": "51'63",
2148         "prior_version": "29'15",
2149         "last_reqid": "osd.1.0:62",
2150         "user_version": 15,
2151         "size": 7,
2152         "mtime": "2018-04-05 14:33:22.589300",
2153         "local_mtime": "2018-04-05 14:33:22.590376",
2154         "lost": 0,
2155         "flags": [
2156           "dirty",
2157           "omap",
2158           "data_digest",
2159           "omap_digest"
2160         ],
2161         "truncate_seq": 0,
2162         "truncate_size": 0,
2163         "data_digest": "0x2ddbf8f5",
2164         "omap_digest": "0x1a862a41",
2165         "expected_object_size": 0,
2166         "expected_write_size": 0,
2167         "alloc_hint_flags": 0,
2168         "manifest": {
2169           "type": 0
2170         },
2171         "watchers": {}
2172       },
2173       "union_shard_errors": [
2174         "omap_digest_mismatch_info"
2175       ],
2176       "errors": [
2177         "omap_digest_mismatch"
2178       ],
2179       "object": {
2180         "version": 15,
2181         "snap": "head",
2182         "locator": "",
2183         "nspace": "",
2184         "name": "ROBJ5"
2185       }
2186     },
2187     {
2188       "shards": [
2189         {
2190           "data_digest": "0x2ddbf8f5",
2191           "omap_digest": "0x689ee887",
2192           "size": 7,
2193           "errors": [
2194             "omap_digest_mismatch_info"
2195           ],
2196           "osd": 0,
2197           "primary": false
2198         },
2199         {
2200           "data_digest": "0x2ddbf8f5",
2201           "omap_digest": "0x179c919f",
2202           "size": 7,
2203           "errors": [],
2204           "osd": 1,
2205           "primary": true
2206         }
2207       ],
2208       "selected_object_info": {
2209         "oid": {
2210           "oid": "ROBJ6",
2211           "key": "",
2212           "snapid": -2,
2213           "hash": 390610085,
2214           "max": 0,
2215           "pool": 3,
2216           "namespace": ""
2217         },
2218         "version": "51'54",
2219         "prior_version": "31'18",
2220         "last_reqid": "osd.1.0:53",
2221         "user_version": 18,
2222         "size": 7,
2223         "mtime": "2018-04-05 14:33:23.289188",
2224         "local_mtime": "2018-04-05 14:33:23.290130",
2225         "lost": 0,
2226         "flags": [
2227           "dirty",
2228           "omap",
2229           "data_digest",
2230           "omap_digest"
2231         ],
2232         "truncate_seq": 0,
2233         "truncate_size": 0,
2234         "data_digest": "0x2ddbf8f5",
2235         "omap_digest": "0x179c919f",
2236         "expected_object_size": 0,
2237         "expected_write_size": 0,
2238         "alloc_hint_flags": 0,
2239         "manifest": {
2240           "type": 0
2241         },
2242         "watchers": {}
2243       },
2244       "union_shard_errors": [
2245         "omap_digest_mismatch_info"
2246       ],
2247       "errors": [
2248         "omap_digest_mismatch"
2249       ],
2250       "object": {
2251         "version": 18,
2252         "snap": "head",
2253         "locator": "",
2254         "nspace": "",
2255         "name": "ROBJ6"
2256       }
2257     },
2258     {
2259       "shards": [
2260         {
2261           "data_digest": "0x2ddbf8f5",
2262           "omap_digest": "0xefced57a",
2263           "size": 7,
2264           "errors": [],
2265           "osd": 0,
2266           "primary": false
2267         },
2268         {
2269           "data_digest": "0x2ddbf8f5",
2270           "omap_digest": "0x6a73cc07",
2271           "size": 7,
2272           "errors": [
2273             "omap_digest_mismatch_info"
2274           ],
2275           "osd": 1,
2276           "primary": true
2277         }
2278       ],
2279       "selected_object_info": {
2280         "oid": {
2281           "oid": "ROBJ7",
2282           "key": "",
2283           "snapid": -2,
2284           "hash": 3529485009,
2285           "max": 0,
2286           "pool": 3,
2287           "namespace": ""
2288         },
2289         "version": "51'53",
2290         "prior_version": "33'21",
2291         "last_reqid": "osd.1.0:52",
2292         "user_version": 21,
2293         "size": 7,
2294         "mtime": "2018-04-05 14:33:23.979658",
2295         "local_mtime": "2018-04-05 14:33:23.980731",
2296         "lost": 0,
2297         "flags": [
2298           "dirty",
2299           "omap",
2300           "data_digest",
2301           "omap_digest"
2302         ],
2303         "truncate_seq": 0,
2304         "truncate_size": 0,
2305         "data_digest": "0x2ddbf8f5",
2306         "omap_digest": "0xefced57a",
2307         "expected_object_size": 0,
2308         "expected_write_size": 0,
2309         "alloc_hint_flags": 0,
2310         "manifest": {
2311           "type": 0
2312         },
2313         "watchers": {}
2314       },
2315       "union_shard_errors": [
2316         "omap_digest_mismatch_info"
2317       ],
2318       "errors": [
2319         "omap_digest_mismatch"
2320       ],
2321       "object": {
2322         "version": 21,
2323         "snap": "head",
2324         "locator": "",
2325         "nspace": "",
2326         "name": "ROBJ7"
2327       }
2328     },
2329     {
2330       "shards": [
2331         {
2332           "attrs": [
2333             {
2334               "Base64": false,
2335               "value": "bad-val",
2336               "name": "key1-ROBJ8"
2337             },
2338             {
2339               "Base64": false,
2340               "value": "val2-ROBJ8",
2341               "name": "key2-ROBJ8"
2342             }
2343           ],
2344           "data_digest": "0x2ddbf8f5",
2345           "omap_digest": "0xd6be81dc",
2346           "size": 7,
2347           "errors": [],
2348           "osd": 0,
2349           "primary": false
2350         },
2351         {
2352           "attrs": [
2353             {
2354               "Base64": false,
2355               "value": "val1-ROBJ8",
2356               "name": "key1-ROBJ8"
2357             },
2358             {
2359               "Base64": false,
2360               "value": "val3-ROBJ8",
2361               "name": "key3-ROBJ8"
2362             }
2363           ],
2364           "data_digest": "0x2ddbf8f5",
2365           "omap_digest": "0xd6be81dc",
2366           "size": 7,
2367           "errors": [],
2368           "osd": 1,
2369           "primary": true
2370         }
2371       ],
2372       "selected_object_info": {
2373         "oid": {
2374           "oid": "ROBJ8",
2375           "key": "",
2376           "snapid": -2,
2377           "hash": 2359695969,
2378           "max": 0,
2379           "pool": 3,
2380           "namespace": ""
2381         },
2382         "version": "79'66",
2383         "prior_version": "79'65",
2384         "last_reqid": "client.4554.0:1",
2385         "user_version": 66,
2386         "size": 7,
2387         "mtime": "2018-04-05 14:34:05.598688",
2388         "local_mtime": "2018-04-05 14:34:05.599698",
2389         "lost": 0,
2390         "flags": [
2391           "dirty",
2392           "omap",
2393           "data_digest",
2394           "omap_digest"
2395         ],
2396         "truncate_seq": 0,
2397         "truncate_size": 0,
2398         "data_digest": "0x2ddbf8f5",
2399         "omap_digest": "0xd6be81dc",
2400         "expected_object_size": 0,
2401         "expected_write_size": 0,
2402         "alloc_hint_flags": 0,
2403         "manifest": {
2404           "type": 0
2405         },
2406         "watchers": {}
2407       },
2408       "union_shard_errors": [],
2409       "errors": [
2410         "attr_value_mismatch",
2411         "attr_name_mismatch"
2412       ],
2413       "object": {
2414         "version": 66,
2415         "snap": "head",
2416         "locator": "",
2417         "nspace": "",
2418         "name": "ROBJ8"
2419       }
2420     },
2421     {
2422       "shards": [
2423         {
2424           "object_info": {
2425             "oid": {
2426               "oid": "ROBJ9",
2427               "key": "",
2428               "snapid": -2,
2429               "hash": 537189375,
2430               "max": 0,
2431               "pool": 3,
2432               "namespace": ""
2433             },
2434             "version": "51'64",
2435             "prior_version": "37'27",
2436             "last_reqid": "osd.1.0:63",
2437             "user_version": 27,
2438             "size": 7,
2439             "mtime": "2018-04-05 14:33:25.352485",
2440             "local_mtime": "2018-04-05 14:33:25.353746",
2441             "lost": 0,
2442             "flags": [
2443               "dirty",
2444               "omap",
2445               "data_digest",
2446               "omap_digest"
2447             ],
2448             "truncate_seq": 0,
2449             "truncate_size": 0,
2450             "data_digest": "0x2ddbf8f5",
2451            "omap_digest": "0x2eecc539",
2452             "expected_object_size": 0,
2453             "expected_write_size": 0,
2454             "alloc_hint_flags": 0,
2455             "manifest": {
2456               "type": 0
2457             },
2458             "watchers": {}
2459           },
2460           "data_digest": "0x1f26fb26",
2461           "omap_digest": "0x2eecc539",
2462           "size": 3,
2463           "errors": [
2464             "obj_size_info_mismatch"
2465           ],
2466           "osd": 0,
2467           "primary": false
2468         },
2469         {
2470           "object_info": {
2471             "oid": {
2472               "oid": "ROBJ9",
2473               "key": "",
2474               "snapid": -2,
2475               "hash": 537189375,
2476               "max": 0,
2477               "pool": 3,
2478               "namespace": ""
2479             },
2480             "version": "119'68",
2481             "prior_version": "51'64",
2482             "last_reqid": "client.4834.0:1",
2483             "user_version": 68,
2484             "size": 3,
2485             "mtime": "2018-04-05 14:35:01.500659",
2486             "local_mtime": "2018-04-05 14:35:01.502117",
2487             "lost": 0,
2488             "flags": [
2489               "dirty",
2490               "omap",
2491               "data_digest",
2492               "omap_digest"
2493             ],
2494             "truncate_seq": 0,
2495             "truncate_size": 0,
2496             "data_digest": "0x1f26fb26",
2497             "omap_digest": "0x2eecc539",
2498             "expected_object_size": 0,
2499             "expected_write_size": 0,
2500             "alloc_hint_flags": 0,
2501             "manifest": {
2502               "type": 0
2503             },
2504             "watchers": {}
2505           },
2506           "data_digest": "0x1f26fb26",
2507           "omap_digest": "0x2eecc539",
2508           "size": 3,
2509           "errors": [],
2510           "osd": 1,
2511           "primary": true
2512         }
2513       ],
2514       "selected_object_info": {
2515         "oid": {
2516           "oid": "ROBJ9",
2517           "key": "",
2518           "snapid": -2,
2519           "hash": 537189375,
2520           "max": 0,
2521           "pool": 3,
2522           "namespace": ""
2523         },
2524         "version": "119'68",
2525         "prior_version": "51'64",
2526         "last_reqid": "client.4834.0:1",
2527         "user_version": 68,
2528         "size": 3,
2529         "mtime": "2018-04-05 14:35:01.500659",
2530         "local_mtime": "2018-04-05 14:35:01.502117",
2531         "lost": 0,
2532         "flags": [
2533           "dirty",
2534           "omap",
2535           "data_digest",
2536           "omap_digest"
2537         ],
2538         "truncate_seq": 0,
2539         "truncate_size": 0,
2540         "data_digest": "0x1f26fb26",
2541         "omap_digest": "0x2eecc539",
2542         "expected_object_size": 0,
2543         "expected_write_size": 0,
2544         "alloc_hint_flags": 0,
2545         "manifest": {
2546           "type": 0
2547         },
2548         "watchers": {}
2549       },
2550       "union_shard_errors": [
2551         "obj_size_info_mismatch"
2552       ],
2553       "errors": [
2554         "object_info_inconsistency"
2555       ],
2556       "object": {
2557         "version": 68,
2558         "snap": "head",
2559         "locator": "",
2560         "nspace": "",
2561         "name": "ROBJ9"
2562       }
2563     }
2564   ],
2565   "epoch": 0
2566 }
2567 EOF
2568
2569     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
2570     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
2571     if test $getjson = "yes"
2572     then
2573         jq '.' $dir/json > save2.json
2574     fi
2575
2576     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
2577     then
2578       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
2579     fi
2580
2581     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
2582     teardown $dir || return 1
2583 }
2584
2585
2586 #
2587 # Test scrub errors for an erasure coded pool
2588 #
2589 function corrupt_scrub_erasure() {
2590     local dir=$1
2591     local allow_overwrites=$2
2592     local poolname=ecpool
2593     local total_objs=7
2594
2595     setup $dir || return 1
2596     run_mon $dir a || return 1
2597     run_mgr $dir x || return 1
2598     for id in $(seq 0 2) ; do
2599         if [ "$allow_overwrites" = "true" ]; then
2600             run_osd_bluestore $dir $id || return 1
2601         else
2602             run_osd $dir $id || return 1
2603         fi
2604     done
2605     create_rbd_pool || return 1
2606     create_pool foo 1
2607
2608     create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1
2609     wait_for_clean || return 1
2610
2611     for i in $(seq 1 $total_objs) ; do
2612         objname=EOBJ${i}
2613         add_something $dir $poolname $objname || return 1
2614
2615         local osd=$(expr $i % 2)
2616
2617         case $i in
2618         1)
2619             # Size (deep scrub data_digest too)
2620             local payload=UVWXYZZZ
2621             echo $payload > $dir/CORRUPT
2622             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2623             ;;
2624
2625         2)
2626             # Corrupt EC shard
2627             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1
2628             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2629             ;;
2630
2631         3)
2632              # missing
2633              objectstore_tool $dir $osd $objname remove || return 1
2634              ;;
2635
2636         4)
2637             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
2638             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
2639
2640             # Break xattrs
2641             echo -n bad-val > $dir/bad-val
2642             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
2643             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
2644             echo -n val3-$objname > $dir/newval
2645             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
2646             rm $dir/bad-val $dir/newval
2647             ;;
2648
2649         5)
2650             # Corrupt EC shard
2651             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2
2652             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2653             ;;
2654
2655         6)
2656             objectstore_tool $dir 0 $objname rm-attr hinfo_key || return 1
2657             echo -n bad-val > $dir/bad-val
2658             objectstore_tool $dir 1 $objname set-attr hinfo_key $dir/bad-val || return 1
2659             ;;
2660
2661         7)
2662             local payload=MAKETHISDIFFERENTFROMOTHEROBJECTS
2663             echo $payload > $dir/DIFFERENT
2664             rados --pool $poolname put $objname $dir/DIFFERENT || return 1
2665
2666             # Get hinfo_key from EOBJ1
2667             objectstore_tool $dir 0 EOBJ1 get-attr hinfo_key > $dir/hinfo
2668             objectstore_tool $dir 0 $objname set-attr hinfo_key $dir/hinfo || return 1
2669             rm -f $dir/hinfo
2670             ;;
2671
2672         esac
2673     done
2674
2675     local pg=$(get_pg $poolname EOBJ0)
2676
2677     pg_scrub $pg
2678
2679     rados list-inconsistent-pg $poolname > $dir/json || return 1
2680     # Check pg count
2681     test $(jq '. | length' $dir/json) = "1" || return 1
2682     # Check pgid
2683     test $(jq -r '.[0]' $dir/json) = $pg || return 1
2684
2685     rados list-inconsistent-obj $pg > $dir/json || return 1
2686     # Get epoch for repair-get requests
2687     epoch=$(jq .epoch $dir/json)
2688
2689     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
2690 {
2691   "inconsistents": [
2692     {
2693       "shards": [
2694         {
2695           "size": 2048,
2696           "errors": [],
2697           "shard": 2,
2698           "osd": 0,
2699           "primary": false
2700         },
2701         {
2702           "object_info": {
2703             "oid": {
2704               "oid": "EOBJ1",
2705               "key": "",
2706               "snapid": -2,
2707               "hash": 560836233,
2708               "max": 0,
2709               "pool": 3,
2710               "namespace": ""
2711             },
2712             "version": "27'1",
2713             "prior_version": "0'0",
2714             "last_reqid": "client.4184.0:1",
2715             "user_version": 1,
2716             "size": 7,
2717             "mtime": "",
2718             "local_mtime": "",
2719             "lost": 0,
2720             "flags": [
2721               "dirty",
2722               "data_digest",
2723               "omap_digest"
2724             ],
2725             "truncate_seq": 0,
2726             "truncate_size": 0,
2727             "data_digest": "0x2ddbf8f5",
2728             "omap_digest": "0xffffffff",
2729             "expected_object_size": 0,
2730             "expected_write_size": 0,
2731             "alloc_hint_flags": 0,
2732             "manifest": {
2733               "type": 0
2734             },
2735             "watchers": {}
2736           },
2737           "size": 9,
2738           "shard": 0,
2739           "errors": [
2740             "size_mismatch_info",
2741             "obj_size_info_mismatch"
2742           ],
2743           "osd": 1,
2744           "primary": true
2745         },
2746         {
2747           "size": 2048,
2748           "shard": 1,
2749           "errors": [],
2750           "osd": 2,
2751           "primary": false
2752         }
2753       ],
2754       "selected_object_info": {
2755         "oid": {
2756           "oid": "EOBJ1",
2757           "key": "",
2758           "snapid": -2,
2759           "hash": 560836233,
2760           "max": 0,
2761           "pool": 3,
2762           "namespace": ""
2763         },
2764         "version": "27'1",
2765         "prior_version": "0'0",
2766         "last_reqid": "client.4184.0:1",
2767         "user_version": 1,
2768         "size": 7,
2769         "mtime": "",
2770         "local_mtime": "",
2771         "lost": 0,
2772         "flags": [
2773           "dirty",
2774           "data_digest",
2775           "omap_digest"
2776         ],
2777         "truncate_seq": 0,
2778         "truncate_size": 0,
2779         "data_digest": "0x2ddbf8f5",
2780         "omap_digest": "0xffffffff",
2781         "expected_object_size": 0,
2782         "expected_write_size": 0,
2783         "alloc_hint_flags": 0,
2784         "manifest": {
2785           "type": 0
2786         },
2787         "watchers": {}
2788       },
2789       "union_shard_errors": [
2790         "size_mismatch_info",
2791         "obj_size_info_mismatch"
2792       ],
2793       "errors": [
2794         "size_mismatch"
2795       ],
2796       "object": {
2797         "version": 1,
2798         "snap": "head",
2799         "locator": "",
2800         "nspace": "",
2801         "name": "EOBJ1"
2802       }
2803     },
2804     {
2805       "shards": [
2806         {
2807           "size": 2048,
2808           "errors": [],
2809           "shard": 2,
2810           "osd": 0,
2811           "primary": false
2812         },
2813         {
2814           "shard": 0,
2815           "errors": [
2816             "missing"
2817           ],
2818           "osd": 1,
2819           "primary": true
2820         },
2821         {
2822           "size": 2048,
2823           "shard": 1,
2824           "errors": [],
2825           "osd": 2,
2826           "primary": false
2827         }
2828       ],
2829       "selected_object_info": {
2830         "oid": {
2831           "oid": "EOBJ3",
2832           "key": "",
2833           "snapid": -2,
2834           "hash": 3125668237,
2835           "max": 0,
2836           "pool": 3,
2837           "namespace": ""
2838         },
2839         "version": "39'3",
2840         "prior_version": "0'0",
2841         "last_reqid": "client.4252.0:1",
2842         "user_version": 3,
2843         "size": 7,
2844         "mtime": "",
2845         "local_mtime": "",
2846         "lost": 0,
2847         "flags": [
2848           "dirty",
2849           "data_digest",
2850           "omap_digest"
2851         ],
2852         "truncate_seq": 0,
2853         "truncate_size": 0,
2854         "data_digest": "0x2ddbf8f5",
2855         "omap_digest": "0xffffffff",
2856         "expected_object_size": 0,
2857         "expected_write_size": 0,
2858         "alloc_hint_flags": 0,
2859         "manifest": {
2860           "type": 0
2861         },
2862         "watchers": {}
2863       },
2864       "union_shard_errors": [
2865         "missing"
2866       ],
2867       "errors": [],
2868       "object": {
2869         "version": 3,
2870         "snap": "head",
2871         "locator": "",
2872         "nspace": "",
2873         "name": "EOBJ3"
2874       }
2875     },
2876     {
2877       "shards": [
2878         {
2879           "attrs": [
2880             {
2881               "Base64": false,
2882               "value": "bad-val",
2883               "name": "key1-EOBJ4"
2884             },
2885             {
2886               "Base64": false,
2887               "value": "val2-EOBJ4",
2888               "name": "key2-EOBJ4"
2889             }
2890           ],
2891           "size": 2048,
2892           "errors": [],
2893           "shard": 2,
2894           "osd": 0,
2895           "primary": false
2896         },
2897         {
2898           "osd": 1,
2899           "primary": true,
2900           "shard": 0,
2901           "errors": [],
2902           "size": 2048,
2903           "attrs": [
2904             {
2905               "Base64": false,
2906               "value": "val1-EOBJ4",
2907               "name": "key1-EOBJ4"
2908             },
2909             {
2910               "Base64": false,
2911               "value": "val2-EOBJ4",
2912               "name": "key2-EOBJ4"
2913             }
2914           ]
2915         },
2916         {
2917           "osd": 2,
2918           "primary": false,
2919           "shard": 1,
2920           "errors": [],
2921           "size": 2048,
2922           "attrs": [
2923             {
2924               "Base64": false,
2925               "value": "val1-EOBJ4",
2926               "name": "key1-EOBJ4"
2927             },
2928             {
2929               "Base64": false,
2930               "value": "val3-EOBJ4",
2931               "name": "key3-EOBJ4"
2932             }
2933           ]
2934         }
2935       ],
2936       "selected_object_info": {
2937         "oid": {
2938           "oid": "EOBJ4",
2939           "key": "",
2940           "snapid": -2,
2941           "hash": 1618759290,
2942           "max": 0,
2943           "pool": 3,
2944           "namespace": ""
2945         },
2946         "version": "45'6",
2947         "prior_version": "45'5",
2948         "last_reqid": "client.4294.0:1",
2949         "user_version": 6,
2950         "size": 7,
2951         "mtime": "",
2952         "local_mtime": "",
2953         "lost": 0,
2954         "flags": [
2955           "dirty",
2956           "data_digest",
2957           "omap_digest"
2958         ],
2959         "truncate_seq": 0,
2960         "truncate_size": 0,
2961         "data_digest": "0x2ddbf8f5",
2962         "omap_digest": "0xffffffff",
2963         "expected_object_size": 0,
2964         "expected_write_size": 0,
2965         "alloc_hint_flags": 0,
2966         "manifest": {
2967           "type": 0
2968         },
2969         "watchers": {}
2970       },
2971       "union_shard_errors": [],
2972       "errors": [
2973         "attr_value_mismatch",
2974         "attr_name_mismatch"
2975       ],
2976       "object": {
2977         "version": 6,
2978         "snap": "head",
2979         "locator": "",
2980         "nspace": "",
2981         "name": "EOBJ4"
2982       }
2983     },
2984     {
2985       "shards": [
2986         {
2987           "size": 2048,
2988           "errors": [],
2989           "shard": 2,
2990           "osd": 0,
2991           "primary": false
2992         },
2993         {
2994           "object_info": {
2995             "oid": {
2996               "oid": "EOBJ5",
2997               "key": "",
2998               "snapid": -2,
2999               "hash": 2918945441,
3000               "max": 0,
3001               "pool": 3,
3002               "namespace": ""
3003             },
3004             "version": "59'7",
3005             "prior_version": "0'0",
3006             "last_reqid": "client.4382.0:1",
3007             "user_version": 7,
3008             "size": 7,
3009             "mtime": "",
3010             "local_mtime": "",
3011             "lost": 0,
3012             "flags": [
3013               "dirty",
3014               "data_digest",
3015               "omap_digest"
3016             ],
3017             "truncate_seq": 0,
3018             "truncate_size": 0,
3019             "data_digest": "0x2ddbf8f5",
3020             "omap_digest": "0xffffffff",
3021             "expected_object_size": 0,
3022             "expected_write_size": 0,
3023             "alloc_hint_flags": 0,
3024             "manifest": {
3025               "type": 0
3026             },
3027             "watchers": {}
3028           },
3029           "size": 4096,
3030           "shard": 0,
3031           "errors": [
3032             "size_mismatch_info",
3033             "obj_size_info_mismatch"
3034           ],
3035           "osd": 1,
3036           "primary": true
3037         },
3038         {
3039           "size": 2048,
3040           "shard": 1,
3041           "errors": [],
3042           "osd": 2,
3043           "primary": false
3044         }
3045       ],
3046       "selected_object_info": {
3047         "oid": {
3048           "oid": "EOBJ5",
3049           "key": "",
3050           "snapid": -2,
3051           "hash": 2918945441,
3052           "max": 0,
3053           "pool": 3,
3054           "namespace": ""
3055         },
3056         "version": "59'7",
3057         "prior_version": "0'0",
3058         "last_reqid": "client.4382.0:1",
3059         "user_version": 7,
3060         "size": 7,
3061         "mtime": "",
3062         "local_mtime": "",
3063         "lost": 0,
3064         "flags": [
3065           "dirty",
3066           "data_digest",
3067           "omap_digest"
3068         ],
3069         "truncate_seq": 0,
3070         "truncate_size": 0,
3071         "data_digest": "0x2ddbf8f5",
3072         "omap_digest": "0xffffffff",
3073         "expected_object_size": 0,
3074         "expected_write_size": 0,
3075         "alloc_hint_flags": 0,
3076         "manifest": {
3077           "type": 0
3078         },
3079         "watchers": {}
3080       },
3081       "union_shard_errors": [
3082         "size_mismatch_info",
3083         "obj_size_info_mismatch"
3084       ],
3085       "errors": [
3086         "size_mismatch"
3087       ],
3088       "object": {
3089         "version": 7,
3090         "snap": "head",
3091         "locator": "",
3092         "nspace": "",
3093         "name": "EOBJ5"
3094       }
3095    },
3096    {
3097      "errors": [],
3098      "object": {
3099        "locator": "",
3100        "name": "EOBJ6",
3101        "nspace": "",
3102        "snap": "head",
3103        "version": 8
3104      },
3105      "selected_object_info": {
3106         "oid": {
3107           "oid": "EOBJ6",
3108           "key": "",
3109           "snapid": -2,
3110           "hash": 3050890866,
3111           "max": 0,
3112           "pool": 3,
3113           "namespace": ""
3114         },
3115         "version": "65'8",
3116         "prior_version": "0'0",
3117         "last_reqid": "client.4418.0:1",
3118         "user_version": 8,
3119         "size": 7,
3120         "mtime": "",
3121         "local_mtime": "",
3122         "lost": 0,
3123         "flags": [
3124           "dirty",
3125           "data_digest",
3126           "omap_digest"
3127         ],
3128         "truncate_seq": 0,
3129         "truncate_size": 0,
3130         "data_digest": "0x2ddbf8f5",
3131         "omap_digest": "0xffffffff",
3132         "expected_object_size": 0,
3133         "expected_write_size": 0,
3134         "alloc_hint_flags": 0,
3135         "manifest": {
3136           "type": 0
3137         },
3138         "watchers": {}
3139      },
3140      "shards": [
3141        {
3142          "errors": [
3143            "hinfo_missing"
3144          ],
3145          "osd": 0,
3146          "primary": false,
3147          "shard": 2,
3148          "size": 2048
3149        },
3150        {
3151          "errors": [
3152            "hinfo_corrupted"
3153          ],
3154          "osd": 1,
3155          "primary": true,
3156          "shard": 0,
3157          "hashinfo": "bad-val",
3158          "size": 2048
3159        },
3160        {
3161          "errors": [],
3162          "osd": 2,
3163          "primary": false,
3164          "shard": 1,
3165          "size": 2048,
3166          "hashinfo": {
3167            "cumulative_shard_hashes": [
3168             {
3169               "hash": 80717615,
3170               "shard": 0
3171             },
3172             {
3173               "hash": 1534491824,
3174               "shard": 1
3175             },
3176             {
3177               "hash": 80717615,
3178               "shard": 2
3179             }
3180            ],
3181            "total_chunk_size": 2048
3182          }
3183        }
3184      ],
3185      "union_shard_errors": [
3186        "hinfo_missing",
3187        "hinfo_corrupted"
3188      ]
3189    },
3190    {
3191      "errors": [
3192        "hinfo_inconsistency"
3193      ],
3194      "object": {
3195        "locator": "",
3196        "name": "EOBJ7",
3197        "nspace": "",
3198        "snap": "head",
3199        "version": 10
3200      },
3201      "selected_object_info": {
3202         "oid": {
3203           "oid": "EOBJ7",
3204           "key": "",
3205           "snapid": -2,
3206           "hash": 3258066308,
3207           "max": 0,
3208           "pool": 3,
3209           "namespace": ""
3210         },
3211         "version": "75'10",
3212         "prior_version": "75'9",
3213         "last_reqid": "client.4482.0:1",
3214         "user_version": 10,
3215         "size": 34,
3216         "mtime": "",
3217         "local_mtime": "",
3218         "lost": 0,
3219         "flags": [
3220           "dirty",
3221           "data_digest",
3222           "omap_digest"
3223         ],
3224         "truncate_seq": 0,
3225         "truncate_size": 0,
3226         "data_digest": "0x136e4e27",
3227         "omap_digest": "0xffffffff",
3228         "expected_object_size": 0,
3229         "expected_write_size": 0,
3230         "alloc_hint_flags": 0,
3231         "manifest": {
3232           "type": 0
3233         },
3234         "watchers": {}
3235      },
3236      "shards": [
3237        {
3238          "hashinfo": {
3239            "cumulative_shard_hashes": [
3240             {
3241               "hash": 80717615,
3242               "shard": 0
3243             },
3244             {
3245               "hash": 1534491824,
3246               "shard": 1
3247             },
3248             {
3249               "hash": 80717615,
3250               "shard": 2
3251             }
3252            ],
3253            "total_chunk_size": 2048
3254          },
3255          "errors": [],
3256          "osd": 0,
3257          "primary": false,
3258          "shard": 2,
3259          "size": 2048
3260        },
3261        {
3262          "hashinfo": {
3263            "cumulative_shard_hashes": [
3264             {
3265               "hash": 1534350760,
3266               "shard": 0
3267             },
3268             {
3269               "hash": 1534491824,
3270               "shard": 1
3271             },
3272             {
3273               "hash": 1534350760,
3274               "shard": 2
3275             }
3276            ],
3277            "total_chunk_size": 2048
3278          },
3279          "errors": [],
3280          "osd": 1,
3281          "primary": true,
3282          "shard": 0,
3283          "size": 2048
3284        },
3285        {
3286          "hashinfo": {
3287            "cumulative_shard_hashes": [
3288             {
3289               "hash": 1534350760,
3290               "shard": 0
3291             },
3292             {
3293               "hash": 1534491824,
3294               "shard": 1
3295             },
3296             {
3297               "hash": 1534350760,
3298               "shard": 2
3299             }
3300            ],
3301            "total_chunk_size": 2048
3302          },
3303          "errors": [],
3304          "osd": 2,
3305          "primary": false,
3306          "shard": 1,
3307          "size": 2048
3308        }
3309      ],
3310      "union_shard_errors": []
3311     }
3312   ],
3313   "epoch": 0
3314 }
3315 EOF
3316
3317     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
3318     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
3319     if test $getjson = "yes"
3320     then
3321         jq '.' $dir/json > save3.json
3322     fi
3323
3324     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
3325     then
3326       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
3327     fi
3328
3329     pg_deep_scrub $pg
3330
3331     rados list-inconsistent-pg $poolname > $dir/json || return 1
3332     # Check pg count
3333     test $(jq '. | length' $dir/json) = "1" || return 1
3334     # Check pgid
3335     test $(jq -r '.[0]' $dir/json) = $pg || return 1
3336
3337     rados list-inconsistent-obj $pg > $dir/json || return 1
3338     # Get epoch for repair-get requests
3339     epoch=$(jq .epoch $dir/json)
3340
3341     if [ "$allow_overwrites" = "true" ]
3342     then
3343       jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
3344 {
3345   "inconsistents": [
3346     {
3347       "shards": [
3348         {
3349           "data_digest": "0x00000000",
3350           "omap_digest": "0xffffffff",
3351           "size": 2048,
3352           "errors": [],
3353           "shard": 2,
3354           "osd": 0,
3355           "primary": false
3356         },
3357         {
3358           "object_info": {
3359             "oid": {
3360               "oid": "EOBJ1",
3361               "key": "",
3362               "snapid": -2,
3363               "hash": 560836233,
3364               "max": 0,
3365               "pool": 3,
3366               "namespace": ""
3367             },
3368             "version": "27'1",
3369             "prior_version": "0'0",
3370             "last_reqid": "client.4184.0:1",
3371             "user_version": 1,
3372             "size": 7,
3373             "mtime": "2018-04-05 14:31:33.837147",
3374             "local_mtime": "2018-04-05 14:31:33.840763",
3375             "lost": 0,
3376             "flags": [
3377               "dirty",
3378               "data_digest",
3379               "omap_digest"
3380             ],
3381             "truncate_seq": 0,
3382             "truncate_size": 0,
3383             "data_digest": "0x2ddbf8f5",
3384             "omap_digest": "0xffffffff",
3385             "expected_object_size": 0,
3386             "expected_write_size": 0,
3387             "alloc_hint_flags": 0,
3388             "manifest": {
3389               "type": 0
3390             },
3391             "watchers": {}
3392           },
3393           "size": 9,
3394           "shard": 0,
3395           "errors": [
3396             "read_error",
3397             "size_mismatch_info",
3398             "obj_size_info_mismatch"
3399           ],
3400           "osd": 1,
3401           "primary": true
3402         },
3403         {
3404           "data_digest": "0x00000000",
3405           "omap_digest": "0xffffffff",
3406           "size": 2048,
3407           "shard": 1,
3408           "errors": [],
3409           "osd": 2,
3410           "primary": false
3411         }
3412       ],
3413       "selected_object_info": {
3414         "oid": {
3415           "oid": "EOBJ1",
3416           "key": "",
3417           "snapid": -2,
3418           "hash": 560836233,
3419           "max": 0,
3420           "pool": 3,
3421           "namespace": ""
3422         },
3423         "version": "27'1",
3424         "prior_version": "0'0",
3425         "last_reqid": "client.4184.0:1",
3426         "user_version": 1,
3427         "size": 7,
3428         "mtime": "2018-04-05 14:31:33.837147",
3429         "local_mtime": "2018-04-05 14:31:33.840763",
3430         "lost": 0,
3431         "flags": [
3432           "dirty",
3433           "data_digest",
3434           "omap_digest"
3435         ],
3436         "truncate_seq": 0,
3437         "truncate_size": 0,
3438         "data_digest": "0x2ddbf8f5",
3439         "omap_digest": "0xffffffff",
3440         "expected_object_size": 0,
3441         "expected_write_size": 0,
3442         "alloc_hint_flags": 0,
3443         "manifest": {
3444           "type": 0
3445         },
3446         "watchers": {}
3447       },
3448       "union_shard_errors": [
3449         "read_error",
3450         "size_mismatch_info",
3451         "obj_size_info_mismatch"
3452       ],
3453       "errors": [
3454         "size_mismatch"
3455       ],
3456       "object": {
3457         "version": 1,
3458         "snap": "head",
3459         "locator": "",
3460         "nspace": "",
3461         "name": "EOBJ1"
3462       }
3463     },
3464     {
3465       "shards": [
3466         {
3467           "data_digest": "0x00000000",
3468           "omap_digest": "0xffffffff",
3469           "size": 2048,
3470           "errors": [],
3471           "shard": 2,
3472           "osd": 0,
3473           "primary": false
3474         },
3475         {
3476           "shard": 0,
3477           "errors": [
3478             "missing"
3479           ],
3480           "osd": 1,
3481           "primary": true
3482         },
3483         {
3484           "data_digest": "0x00000000",
3485           "omap_digest": "0xffffffff",
3486           "size": 2048,
3487           "shard": 1,
3488           "errors": [],
3489           "osd": 2,
3490           "primary": false
3491         }
3492       ],
3493       "selected_object_info": {
3494         "oid": {
3495           "oid": "EOBJ3",
3496           "key": "",
3497           "snapid": -2,
3498           "hash": 3125668237,
3499           "max": 0,
3500           "pool": 3,
3501           "namespace": ""
3502         },
3503         "version": "39'3",
3504         "prior_version": "0'0",
3505         "last_reqid": "client.4252.0:1",
3506         "user_version": 3,
3507         "size": 7,
3508         "mtime": "2018-04-05 14:31:46.841145",
3509         "local_mtime": "2018-04-05 14:31:46.844996",
3510         "lost": 0,
3511         "flags": [
3512           "dirty",
3513           "data_digest",
3514           "omap_digest"
3515         ],
3516         "truncate_seq": 0,
3517         "truncate_size": 0,
3518         "data_digest": "0x2ddbf8f5",
3519         "omap_digest": "0xffffffff",
3520         "expected_object_size": 0,
3521         "expected_write_size": 0,
3522         "alloc_hint_flags": 0,
3523         "manifest": {
3524           "type": 0
3525         },
3526         "watchers": {}
3527       },
3528       "union_shard_errors": [
3529         "missing"
3530       ],
3531       "errors": [],
3532       "object": {
3533         "version": 3,
3534         "snap": "head",
3535         "locator": "",
3536         "nspace": "",
3537         "name": "EOBJ3"
3538       }
3539     },
3540     {
3541       "shards": [
3542         {
3543           "attrs": [
3544             {
3545               "Base64": false,
3546               "value": "bad-val",
3547               "name": "key1-EOBJ4"
3548             },
3549             {
3550               "Base64": false,
3551               "value": "val2-EOBJ4",
3552               "name": "key2-EOBJ4"
3553             }
3554           ],
3555           "data_digest": "0x00000000",
3556           "omap_digest": "0xffffffff",
3557           "size": 2048,
3558           "errors": [],
3559           "shard": 2,
3560           "osd": 0,
3561           "primary": false
3562         },
3563         {
3564           "attrs": [
3565             {
3566               "Base64": false,
3567               "value": "val1-EOBJ4",
3568               "name": "key1-EOBJ4"
3569             },
3570             {
3571               "Base64": false,
3572               "value": "val2-EOBJ4",
3573               "name": "key2-EOBJ4"
3574             }
3575           ],
3576           "data_digest": "0x00000000",
3577           "omap_digest": "0xffffffff",
3578           "size": 2048,
3579           "errors": [],
3580           "shard": 0,
3581           "osd": 1,
3582           "primary": true
3583         },
3584         {
3585           "attrs": [
3586             {
3587               "Base64": false,
3588               "value": "val1-EOBJ4",
3589               "name": "key1-EOBJ4"
3590             },
3591             {
3592               "Base64": false,
3593               "value": "val3-EOBJ4",
3594               "name": "key3-EOBJ4"
3595             }
3596           ],
3597           "data_digest": "0x00000000",
3598           "omap_digest": "0xffffffff",
3599           "size": 2048,
3600           "errors": [],
3601           "shard": 1,
3602           "osd": 2,
3603           "primary": false
3604         }
3605       ],
3606       "selected_object_info": {
3607         "oid": {
3608           "oid": "EOBJ4",
3609           "key": "",
3610           "snapid": -2,
3611           "hash": 1618759290,
3612           "max": 0,
3613           "pool": 3,
3614           "namespace": ""
3615         },
3616         "version": "45'6",
3617         "prior_version": "45'5",
3618         "last_reqid": "client.4294.0:1",
3619         "user_version": 6,
3620         "size": 7,
3621         "mtime": "2018-04-05 14:31:54.663622",
3622         "local_mtime": "2018-04-05 14:31:54.664527",
3623         "lost": 0,
3624         "flags": [
3625           "dirty",
3626           "data_digest",
3627           "omap_digest"
3628         ],
3629         "truncate_seq": 0,
3630         "truncate_size": 0,
3631         "data_digest": "0x2ddbf8f5",
3632         "omap_digest": "0xffffffff",
3633         "expected_object_size": 0,
3634         "expected_write_size": 0,
3635         "alloc_hint_flags": 0,
3636         "manifest": {
3637           "type": 0
3638         },
3639         "watchers": {}
3640       },
3641       "union_shard_errors": [],
3642       "errors": [
3643         "attr_value_mismatch",
3644         "attr_name_mismatch"
3645       ],
3646       "object": {
3647         "version": 6,
3648         "snap": "head",
3649         "locator": "",
3650         "nspace": "",
3651         "name": "EOBJ4"
3652       }
3653     },
3654     {
3655       "shards": [
3656         {
3657           "data_digest": "0x00000000",
3658           "omap_digest": "0xffffffff",
3659           "size": 2048,
3660           "errors": [],
3661           "shard": 2,
3662           "osd": 0,
3663           "primary": false
3664         },
3665         {
3666           "data_digest": "0x00000000",
3667           "omap_digest": "0xffffffff",
3668           "object_info": {
3669             "oid": {
3670               "oid": "EOBJ5",
3671               "key": "",
3672               "snapid": -2,
3673               "hash": 2918945441,
3674               "max": 0,
3675               "pool": 3,
3676               "namespace": ""
3677             },
3678             "version": "59'7",
3679             "prior_version": "0'0",
3680             "last_reqid": "client.4382.0:1",
3681             "user_version": 7,
3682             "size": 7,
3683             "mtime": "2018-04-05 14:32:12.929161",
3684             "local_mtime": "2018-04-05 14:32:12.934707",
3685             "lost": 0,
3686             "flags": [
3687               "dirty",
3688               "data_digest",
3689               "omap_digest"
3690             ],
3691             "truncate_seq": 0,
3692             "truncate_size": 0,
3693             "data_digest": "0x2ddbf8f5",
3694             "omap_digest": "0xffffffff",
3695             "expected_object_size": 0,
3696             "expected_write_size": 0,
3697             "alloc_hint_flags": 0,
3698             "manifest": {
3699               "type": 0
3700             },
3701             "watchers": {}
3702           },
3703           "size": 4096,
3704           "errors": [
3705             "size_mismatch_info",
3706             "obj_size_info_mismatch"
3707           ],
3708           "shard": 0,
3709           "osd": 1,
3710           "primary": true
3711         },
3712         {
3713           "data_digest": "0x00000000",
3714           "omap_digest": "0xffffffff",
3715           "size": 2048,
3716           "errors": [],
3717           "shard": 1,
3718           "osd": 2,
3719           "primary": false
3720         }
3721       ],
3722       "selected_object_info": {
3723         "oid": {
3724           "oid": "EOBJ5",
3725           "key": "",
3726           "snapid": -2,
3727           "hash": 2918945441,
3728           "max": 0,
3729           "pool": 3,
3730           "namespace": ""
3731         },
3732         "version": "59'7",
3733         "prior_version": "0'0",
3734         "last_reqid": "client.4382.0:1",
3735         "user_version": 7,
3736         "size": 7,
3737         "mtime": "2018-04-05 14:32:12.929161",
3738         "local_mtime": "2018-04-05 14:32:12.934707",
3739         "lost": 0,
3740         "flags": [
3741           "dirty",
3742           "data_digest",
3743           "omap_digest"
3744         ],
3745         "truncate_seq": 0,
3746         "truncate_size": 0,
3747         "data_digest": "0x2ddbf8f5",
3748         "omap_digest": "0xffffffff",
3749         "expected_object_size": 0,
3750         "expected_write_size": 0,
3751         "alloc_hint_flags": 0,
3752         "manifest": {
3753           "type": 0
3754         },
3755         "watchers": {}
3756       },
3757       "union_shard_errors": [
3758         "size_mismatch_info",
3759         "obj_size_info_mismatch"
3760       ],
3761       "errors": [
3762         "size_mismatch"
3763       ],
3764       "object": {
3765         "version": 7,
3766         "snap": "head",
3767         "locator": "",
3768         "nspace": "",
3769         "name": "EOBJ5"
3770       }
3771     },
3772     {
3773       "object": {
3774         "name": "EOBJ6",
3775         "nspace": "",
3776         "locator": "",
3777         "snap": "head",
3778         "version": 8
3779       },
3780       "errors": [],
3781       "union_shard_errors": [
3782         "read_error",
3783         "hinfo_missing",
3784         "hinfo_corrupted"
3785       ],
3786       "selected_object_info": {
3787         "oid": {
3788           "oid": "EOBJ6",
3789           "key": "",
3790           "snapid": -2,
3791           "hash": 3050890866,
3792           "max": 0,
3793           "pool": 3,
3794           "namespace": ""
3795         },
3796         "version": "65'8",
3797         "prior_version": "0'0",
3798         "last_reqid": "client.4418.0:1",
3799         "user_version": 8,
3800         "size": 7,
3801         "mtime": "2018-04-05 14:32:20.634116",
3802         "local_mtime": "2018-04-05 14:32:20.637999",
3803         "lost": 0,
3804         "flags": [
3805           "dirty",
3806           "data_digest",
3807           "omap_digest"
3808         ],
3809         "truncate_seq": 0,
3810         "truncate_size": 0,
3811         "data_digest": "0x2ddbf8f5",
3812         "omap_digest": "0xffffffff",
3813         "expected_object_size": 0,
3814         "expected_write_size": 0,
3815         "alloc_hint_flags": 0,
3816         "manifest": {
3817           "type": 0
3818         },
3819         "watchers": {}
3820       },
3821       "shards": [
3822         {
3823           "osd": 0,
3824           "primary": false,
3825           "shard": 2,
3826           "errors": [
3827             "read_error",
3828             "hinfo_missing"
3829           ],
3830           "size": 2048
3831         },
3832         {
3833           "osd": 1,
3834           "primary": true,
3835           "shard": 0,
3836           "errors": [
3837             "read_error",
3838             "hinfo_corrupted"
3839           ],
3840           "size": 2048,
3841           "hashinfo": "bad-val"
3842         },
3843         {
3844           "osd": 2,
3845           "primary": false,
3846           "shard": 1,
3847           "errors": [],
3848           "size": 2048,
3849           "omap_digest": "0xffffffff",
3850           "data_digest": "0x00000000",
3851           "hashinfo": {
3852             "cumulative_shard_hashes": [
3853             {
3854               "hash": 80717615,
3855               "shard": 0
3856             },
3857             {
3858               "hash": 1534491824,
3859               "shard": 1
3860             },
3861             {
3862               "hash": 80717615,
3863               "shard": 2
3864             }
3865            ],
3866            "total_chunk_size": 2048
3867          }
3868         }
3869       ]
3870     },
3871     {
3872       "object": {
3873         "name": "EOBJ7",
3874         "nspace": "",
3875         "locator": "",
3876         "snap": "head",
3877         "version": 10
3878       },
3879       "errors": [
3880         "hinfo_inconsistency"
3881       ],
3882       "union_shard_errors": [],
3883       "selected_object_info": {
3884         "oid": {
3885           "oid": "EOBJ7",
3886           "key": "",
3887           "snapid": -2,
3888           "hash": 3258066308,
3889           "max": 0,
3890           "pool": 3,
3891           "namespace": ""
3892         },
3893         "version": "75'10",
3894         "prior_version": "75'9",
3895         "last_reqid": "client.4482.0:1",
3896         "user_version": 10,
3897         "size": 34,
3898         "mtime": "2018-04-05 14:32:33.058782",
3899         "local_mtime": "2018-04-05 14:32:33.059679",
3900         "lost": 0,
3901         "flags": [
3902           "dirty",
3903           "data_digest",
3904           "omap_digest"
3905         ],
3906         "truncate_seq": 0,
3907         "truncate_size": 0,
3908         "data_digest": "0x136e4e27",
3909         "omap_digest": "0xffffffff",
3910         "expected_object_size": 0,
3911         "expected_write_size": 0,
3912         "alloc_hint_flags": 0,
3913         "manifest": {
3914           "type": 0
3915         },
3916         "watchers": {}
3917       },
3918       "shards": [
3919         {
3920           "osd": 0,
3921           "primary": false,
3922           "shard": 2,
3923           "errors": [],
3924           "size": 2048,
3925           "omap_digest": "0xffffffff",
3926           "data_digest": "0x00000000",
3927           "hashinfo": {
3928            "cumulative_shard_hashes": [
3929             {
3930               "hash": 80717615,
3931               "shard": 0
3932             },
3933             {
3934               "hash": 1534491824,
3935               "shard": 1
3936             },
3937             {
3938               "hash": 80717615,
3939               "shard": 2
3940             }
3941            ],
3942            "total_chunk_size": 2048
3943           }
3944         },
3945         {
3946           "osd": 1,
3947           "primary": true,
3948           "shard": 0,
3949           "errors": [],
3950           "size": 2048,
3951           "omap_digest": "0xffffffff",
3952           "data_digest": "0x00000000",
3953           "hashinfo": {
3954            "cumulative_shard_hashes": [
3955             {
3956               "hash": 1534350760,
3957               "shard": 0
3958             },
3959             {
3960               "hash": 1534491824,
3961               "shard": 1
3962             },
3963             {
3964               "hash": 1534350760,
3965               "shard": 2
3966             }
3967            ],
3968            "total_chunk_size": 2048
3969           }
3970         },
3971         {
3972           "osd": 2,
3973           "primary": false,
3974           "shard": 1,
3975           "errors": [],
3976           "size": 2048,
3977           "omap_digest": "0xffffffff",
3978           "data_digest": "0x00000000",
3979           "hashinfo": {
3980            "cumulative_shard_hashes": [
3981             {
3982               "hash": 1534350760,
3983               "shard": 0
3984             },
3985             {
3986               "hash": 1534491824,
3987               "shard": 1
3988             },
3989             {
3990               "hash": 1534350760,
3991               "shard": 2
3992             }
3993            ],
3994            "total_chunk_size": 2048
3995           }
3996         }
3997       ]
3998     }
3999   ],
4000   "epoch": 0
4001 }
4002 EOF
4003
4004     else
4005
4006       jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
4007 {
4008   "inconsistents": [
4009     {
4010       "shards": [
4011         {
4012           "data_digest": "0x04cfa72f",
4013           "omap_digest": "0xffffffff",
4014           "size": 2048,
4015           "errors": [],
4016           "shard": 2,
4017           "osd": 0,
4018           "primary": false
4019         },
4020         {
4021           "object_info": {
4022             "oid": {
4023               "oid": "EOBJ1",
4024               "key": "",
4025               "snapid": -2,
4026               "hash": 560836233,
4027               "max": 0,
4028               "pool": 3,
4029               "namespace": ""
4030             },
4031             "version": "27'1",
4032             "prior_version": "0'0",
4033             "last_reqid": "client.4192.0:1",
4034             "user_version": 1,
4035             "size": 7,
4036             "mtime": "2018-04-05 14:30:10.688009",
4037             "local_mtime": "2018-04-05 14:30:10.691774",
4038             "lost": 0,
4039             "flags": [
4040               "dirty",
4041               "data_digest",
4042               "omap_digest"
4043             ],
4044             "truncate_seq": 0,
4045             "truncate_size": 0,
4046             "data_digest": "0x2ddbf8f5",
4047             "omap_digest": "0xffffffff",
4048             "expected_object_size": 0,
4049             "expected_write_size": 0,
4050             "alloc_hint_flags": 0,
4051             "manifest": {
4052               "type": 0
4053             },
4054             "watchers": {}
4055           },
4056           "size": 9,
4057           "shard": 0,
4058           "errors": [
4059             "read_error",
4060             "size_mismatch_info",
4061             "obj_size_info_mismatch"
4062           ],
4063           "osd": 1,
4064           "primary": true
4065         },
4066         {
4067           "data_digest": "0x04cfa72f",
4068           "omap_digest": "0xffffffff",
4069           "size": 2048,
4070           "shard": 1,
4071           "errors": [],
4072           "osd": 2,
4073           "primary": false
4074         }
4075       ],
4076       "selected_object_info": {
4077         "oid": {
4078           "oid": "EOBJ1",
4079           "key": "",
4080           "snapid": -2,
4081           "hash": 560836233,
4082           "max": 0,
4083           "pool": 3,
4084           "namespace": ""
4085         },
4086         "version": "27'1",
4087         "prior_version": "0'0",
4088         "last_reqid": "client.4192.0:1",
4089         "user_version": 1,
4090         "size": 7,
4091         "mtime": "2018-04-05 14:30:10.688009",
4092         "local_mtime": "2018-04-05 14:30:10.691774",
4093         "lost": 0,
4094         "flags": [
4095           "dirty",
4096           "data_digest",
4097           "omap_digest"
4098         ],
4099         "truncate_seq": 0,
4100         "truncate_size": 0,
4101         "data_digest": "0x2ddbf8f5",
4102         "omap_digest": "0xffffffff",
4103         "expected_object_size": 0,
4104         "expected_write_size": 0,
4105         "alloc_hint_flags": 0,
4106         "manifest": {
4107           "type": 0
4108         },
4109         "watchers": {}
4110       },
4111       "union_shard_errors": [
4112         "read_error",
4113         "size_mismatch_info",
4114         "obj_size_info_mismatch"
4115       ],
4116       "errors": [
4117         "size_mismatch"
4118       ],
4119       "object": {
4120         "version": 1,
4121         "snap": "head",
4122         "locator": "",
4123         "nspace": "",
4124         "name": "EOBJ1"
4125       }
4126     },
4127     {
4128       "shards": [
4129         {
4130           "size": 2048,
4131           "errors": [
4132             "ec_hash_error"
4133           ],
4134           "shard": 2,
4135           "osd": 0,
4136           "primary": false
4137         },
4138         {
4139           "data_digest": "0x04cfa72f",
4140           "omap_digest": "0xffffffff",
4141           "size": 2048,
4142           "errors": [],
4143           "shard": 0,
4144           "osd": 1,
4145           "primary": true
4146         },
4147         {
4148           "data_digest": "0x04cfa72f",
4149           "omap_digest": "0xffffffff",
4150           "size": 2048,
4151           "errors": [],
4152           "shard": 1,
4153           "osd": 2,
4154           "primary": false
4155         }
4156       ],
4157       "selected_object_info": {
4158         "oid": {
4159           "oid": "EOBJ2",
4160           "key": "",
4161           "snapid": -2,
4162           "hash": 562812377,
4163           "max": 0,
4164           "pool": 3,
4165           "namespace": ""
4166         },
4167         "version": "33'2",
4168         "prior_version": "0'0",
4169         "last_reqid": "client.4224.0:1",
4170         "user_version": 2,
4171         "size": 7,
4172         "mtime": "2018-04-05 14:30:14.152945",
4173         "local_mtime": "2018-04-05 14:30:14.154014",
4174         "lost": 0,
4175         "flags": [
4176           "dirty",
4177           "data_digest",
4178           "omap_digest"
4179         ],
4180         "truncate_seq": 0,
4181         "truncate_size": 0,
4182         "data_digest": "0x2ddbf8f5",
4183         "omap_digest": "0xffffffff",
4184         "expected_object_size": 0,
4185         "expected_write_size": 0,
4186         "alloc_hint_flags": 0,
4187         "manifest": {
4188           "type": 0
4189         },
4190         "watchers": {}
4191       },
4192       "union_shard_errors": [
4193         "ec_hash_error"
4194       ],
4195       "errors": [],
4196       "object": {
4197         "version": 2,
4198         "snap": "head",
4199         "locator": "",
4200         "nspace": "",
4201         "name": "EOBJ2"
4202       }
4203     },
4204     {
4205       "shards": [
4206         {
4207           "data_digest": "0x04cfa72f",
4208           "omap_digest": "0xffffffff",
4209           "size": 2048,
4210           "errors": [],
4211           "shard": 2,
4212           "osd": 0,
4213           "primary": false
4214         },
4215         {
4216           "osd": 1,
4217           "primary": true,
4218           "shard": 0,
4219           "errors": [
4220             "missing"
4221           ]
4222         },
4223         {
4224           "data_digest": "0x04cfa72f",
4225           "omap_digest": "0xffffffff",
4226           "size": 2048,
4227           "shard": 1,
4228           "errors": [],
4229           "osd": 2,
4230           "primary": false
4231         }
4232       ],
4233       "selected_object_info": {
4234         "oid": {
4235           "oid": "EOBJ3",
4236           "key": "",
4237           "snapid": -2,
4238           "hash": 3125668237,
4239           "max": 0,
4240           "pool": 3,
4241           "namespace": ""
4242         },
4243         "version": "39'3",
4244         "prior_version": "0'0",
4245         "last_reqid": "client.4258.0:1",
4246         "user_version": 3,
4247         "size": 7,
4248         "mtime": "2018-04-05 14:30:18.875544",
4249         "local_mtime": "2018-04-05 14:30:18.880153",
4250         "lost": 0,
4251         "flags": [
4252           "dirty",
4253           "data_digest",
4254           "omap_digest"
4255         ],
4256         "truncate_seq": 0,
4257         "truncate_size": 0,
4258         "data_digest": "0x2ddbf8f5",
4259         "omap_digest": "0xffffffff",
4260         "expected_object_size": 0,
4261         "expected_write_size": 0,
4262         "alloc_hint_flags": 0,
4263         "manifest": {
4264           "type": 0
4265         },
4266         "watchers": {}
4267       },
4268       "union_shard_errors": [
4269         "missing"
4270       ],
4271       "errors": [],
4272       "object": {
4273         "version": 3,
4274         "snap": "head",
4275         "locator": "",
4276         "nspace": "",
4277         "name": "EOBJ3"
4278       }
4279     },
4280     {
4281       "shards": [
4282         {
4283           "attrs": [
4284             {
4285               "Base64": false,
4286               "value": "bad-val",
4287               "name": "key1-EOBJ4"
4288             },
4289             {
4290               "Base64": false,
4291               "value": "val2-EOBJ4",
4292               "name": "key2-EOBJ4"
4293             }
4294           ],
4295           "data_digest": "0x04cfa72f",
4296           "omap_digest": "0xffffffff",
4297           "size": 2048,
4298           "errors": [],
4299           "shard": 2,
4300           "osd": 0,
4301           "primary": false
4302         },
4303         {
4304           "osd": 1,
4305           "primary": true,
4306           "shard": 0,
4307           "errors": [],
4308           "size": 2048,
4309           "omap_digest": "0xffffffff",
4310           "data_digest": "0x04cfa72f",
4311           "attrs": [
4312             {
4313               "Base64": false,
4314               "value": "val1-EOBJ4",
4315               "name": "key1-EOBJ4"
4316             },
4317             {
4318               "Base64": false,
4319               "value": "val2-EOBJ4",
4320               "name": "key2-EOBJ4"
4321             }
4322           ]
4323         },
4324         {
4325           "osd": 2,
4326           "primary": false,
4327           "shard": 1,
4328           "errors": [],
4329           "size": 2048,
4330           "omap_digest": "0xffffffff",
4331           "data_digest": "0x04cfa72f",
4332           "attrs": [
4333             {
4334               "Base64": false,
4335               "value": "val1-EOBJ4",
4336               "name": "key1-EOBJ4"
4337             },
4338             {
4339               "Base64": false,
4340               "value": "val3-EOBJ4",
4341               "name": "key3-EOBJ4"
4342             }
4343           ]
4344         }
4345       ],
4346       "selected_object_info": {
4347         "oid": {
4348           "oid": "EOBJ4",
4349           "key": "",
4350           "snapid": -2,
4351           "hash": 1618759290,
4352           "max": 0,
4353           "pool": 3,
4354           "namespace": ""
4355         },
4356         "version": "45'6",
4357         "prior_version": "45'5",
4358         "last_reqid": "client.4296.0:1",
4359         "user_version": 6,
4360         "size": 7,
4361         "mtime": "2018-04-05 14:30:22.271983",
4362         "local_mtime": "2018-04-05 14:30:22.272840",
4363         "lost": 0,
4364         "flags": [
4365           "dirty",
4366           "data_digest",
4367           "omap_digest"
4368         ],
4369         "truncate_seq": 0,
4370         "truncate_size": 0,
4371         "data_digest": "0x2ddbf8f5",
4372         "omap_digest": "0xffffffff",
4373         "expected_object_size": 0,
4374         "expected_write_size": 0,
4375         "alloc_hint_flags": 0,
4376         "manifest": {
4377           "type": 0
4378         },
4379         "watchers": {}
4380       },
4381       "union_shard_errors": [],
4382       "errors": [
4383         "attr_value_mismatch",
4384         "attr_name_mismatch"
4385       ],
4386       "object": {
4387         "version": 6,
4388         "snap": "head",
4389         "locator": "",
4390         "nspace": "",
4391         "name": "EOBJ4"
4392       }
4393     },
4394     {
4395       "shards": [
4396         {
4397           "data_digest": "0x04cfa72f",
4398           "omap_digest": "0xffffffff",
4399           "size": 2048,
4400           "errors": [],
4401           "shard": 2,
4402           "osd": 0,
4403           "primary": false
4404         },
4405         {
4406           "object_info": {
4407             "oid": {
4408               "oid": "EOBJ5",
4409               "key": "",
4410               "snapid": -2,
4411               "hash": 2918945441,
4412               "max": 0,
4413               "pool": 3,
4414               "namespace": ""
4415             },
4416             "version": "59'7",
4417             "prior_version": "0'0",
4418             "last_reqid": "client.4384.0:1",
4419             "user_version": 7,
4420             "size": 7,
4421             "mtime": "2018-04-05 14:30:35.162395",
4422             "local_mtime": "2018-04-05 14:30:35.166390",
4423             "lost": 0,
4424             "flags": [
4425               "dirty",
4426               "data_digest",
4427               "omap_digest"
4428             ],
4429             "truncate_seq": 0,
4430             "truncate_size": 0,
4431             "data_digest": "0x2ddbf8f5",
4432             "omap_digest": "0xffffffff",
4433             "expected_object_size": 0,
4434             "expected_write_size": 0,
4435             "alloc_hint_flags": 0,
4436             "manifest": {
4437               "type": 0
4438             },
4439             "watchers": {}
4440           },
4441           "size": 4096,
4442           "shard": 0,
4443           "errors": [
4444             "size_mismatch_info",
4445             "ec_size_error",
4446             "obj_size_info_mismatch"
4447           ],
4448           "osd": 1,
4449           "primary": true
4450         },
4451         {
4452           "data_digest": "0x04cfa72f",
4453           "omap_digest": "0xffffffff",
4454           "size": 2048,
4455           "shard": 1,
4456           "errors": [],
4457           "osd": 2,
4458           "primary": false
4459         }
4460       ],
4461       "selected_object_info": {
4462         "oid": {
4463           "oid": "EOBJ5",
4464           "key": "",
4465           "snapid": -2,
4466           "hash": 2918945441,
4467           "max": 0,
4468           "pool": 3,
4469           "namespace": ""
4470         },
4471         "version": "59'7",
4472         "prior_version": "0'0",
4473         "last_reqid": "client.4384.0:1",
4474         "user_version": 7,
4475         "size": 7,
4476         "mtime": "2018-04-05 14:30:35.162395",
4477         "local_mtime": "2018-04-05 14:30:35.166390",
4478         "lost": 0,
4479         "flags": [
4480           "dirty",
4481           "data_digest",
4482           "omap_digest"
4483         ],
4484         "truncate_seq": 0,
4485         "truncate_size": 0,
4486         "data_digest": "0x2ddbf8f5",
4487         "omap_digest": "0xffffffff",
4488         "expected_object_size": 0,
4489         "expected_write_size": 0,
4490         "alloc_hint_flags": 0,
4491         "manifest": {
4492           "type": 0
4493         },
4494         "watchers": {}
4495       },
4496       "union_shard_errors": [
4497         "size_mismatch_info",
4498         "ec_size_error",
4499         "obj_size_info_mismatch"
4500       ],
4501       "errors": [
4502         "size_mismatch"
4503       ],
4504       "object": {
4505         "version": 7,
4506         "snap": "head",
4507         "locator": "",
4508         "nspace": "",
4509         "name": "EOBJ5"
4510       }
4511     },
4512     {
4513       "object": {
4514         "name": "EOBJ6",
4515         "nspace": "",
4516         "locator": "",
4517         "snap": "head",
4518         "version": 8
4519       },
4520       "errors": [],
4521       "union_shard_errors": [
4522         "read_error",
4523         "hinfo_missing",
4524         "hinfo_corrupted"
4525       ],
4526       "selected_object_info": {
4527         "oid": {
4528           "oid": "EOBJ6",
4529           "key": "",
4530           "snapid": -2,
4531           "hash": 3050890866,
4532           "max": 0,
4533           "pool": 3,
4534           "namespace": ""
4535         },
4536         "version": "65'8",
4537         "prior_version": "0'0",
4538         "last_reqid": "client.4420.0:1",
4539         "user_version": 8,
4540         "size": 7,
4541         "mtime": "2018-04-05 14:30:40.914673",
4542         "local_mtime": "2018-04-05 14:30:40.917705",
4543         "lost": 0,
4544         "flags": [
4545           "dirty",
4546           "data_digest",
4547           "omap_digest"
4548         ],
4549         "truncate_seq": 0,
4550         "truncate_size": 0,
4551         "data_digest": "0x2ddbf8f5",
4552         "omap_digest": "0xffffffff",
4553         "expected_object_size": 0,
4554         "expected_write_size": 0,
4555         "alloc_hint_flags": 0,
4556         "manifest": {
4557           "type": 0
4558         },
4559         "watchers": {}
4560       },
4561       "shards": [
4562         {
4563           "osd": 0,
4564           "primary": false,
4565           "shard": 2,
4566           "errors": [
4567             "read_error",
4568             "hinfo_missing"
4569           ],
4570           "size": 2048
4571         },
4572         {
4573           "osd": 1,
4574           "primary": true,
4575           "shard": 0,
4576           "errors": [
4577             "read_error",
4578             "hinfo_corrupted"
4579           ],
4580           "size": 2048,
4581           "hashinfo": "bad-val"
4582         },
4583         {
4584           "osd": 2,
4585           "primary": false,
4586           "shard": 1,
4587           "errors": [],
4588           "size": 2048,
4589           "omap_digest": "0xffffffff",
4590           "data_digest": "0x04cfa72f",
4591           "hashinfo": {
4592            "cumulative_shard_hashes": [
4593             {
4594               "hash": 80717615,
4595               "shard": 0
4596             },
4597             {
4598               "hash": 1534491824,
4599               "shard": 1
4600             },
4601             {
4602               "hash": 80717615,
4603               "shard": 2
4604             }
4605            ],
4606            "total_chunk_size": 2048
4607           }
4608         }
4609       ]
4610     },
4611     {
4612       "object": {
4613         "name": "EOBJ7",
4614         "nspace": "",
4615         "locator": "",
4616         "snap": "head",
4617         "version": 10
4618       },
4619       "errors": [
4620         "hinfo_inconsistency"
4621       ],
4622       "union_shard_errors": [
4623         "ec_hash_error"
4624       ],
4625       "selected_object_info": {
4626         "oid": {
4627           "oid": "EOBJ7",
4628           "key": "",
4629           "snapid": -2,
4630           "hash": 3258066308,
4631           "max": 0,
4632           "pool": 3,
4633           "namespace": ""
4634         },
4635         "version": "75'10",
4636         "prior_version": "75'9",
4637         "last_reqid": "client.4486.0:1",
4638         "user_version": 10,
4639         "size": 34,
4640         "mtime": "2018-04-05 14:30:50.995009",
4641         "local_mtime": "2018-04-05 14:30:50.996112",
4642         "lost": 0,
4643         "flags": [
4644           "dirty",
4645           "data_digest",
4646           "omap_digest"
4647         ],
4648         "truncate_seq": 0,
4649         "truncate_size": 0,
4650         "data_digest": "0x136e4e27",
4651         "omap_digest": "0xffffffff",
4652         "expected_object_size": 0,
4653         "expected_write_size": 0,
4654         "alloc_hint_flags": 0,
4655         "manifest": {
4656           "type": 0
4657         },
4658         "watchers": {}
4659       },
4660       "shards": [
4661         {
4662           "osd": 0,
4663           "primary": false,
4664           "shard": 2,
4665           "errors": [
4666             "ec_hash_error"
4667           ],
4668           "size": 2048,
4669           "hashinfo": {
4670            "cumulative_shard_hashes": [
4671             {
4672               "hash": 80717615,
4673               "shard": 0
4674             },
4675             {
4676               "hash": 1534491824,
4677               "shard": 1
4678             },
4679             {
4680               "hash": 80717615,
4681               "shard": 2
4682             }
4683            ],
4684            "total_chunk_size": 2048
4685          }
4686         },
4687         {
4688           "osd": 1,
4689           "primary": true,
4690           "shard": 0,
4691           "errors": [],
4692           "size": 2048,
4693           "omap_digest": "0xffffffff",
4694           "data_digest": "0x5b7455a8",
4695           "hashinfo": {
4696            "cumulative_shard_hashes": [
4697             {
4698               "hash": 1534350760,
4699               "shard": 0
4700             },
4701             {
4702               "hash": 1534491824,
4703               "shard": 1
4704             },
4705             {
4706               "hash": 1534350760,
4707               "shard": 2
4708             }
4709            ],
4710            "total_chunk_size": 2048
4711           }
4712         },
4713         {
4714           "osd": 2,
4715           "primary": false,
4716           "shard": 1,
4717           "errors": [],
4718           "size": 2048,
4719           "omap_digest": "0xffffffff",
4720           "data_digest": "0x5b7455a8",
4721           "hashinfo": {
4722            "cumulative_shard_hashes": [
4723             {
4724               "hash": 1534350760,
4725               "shard": 0
4726             },
4727             {
4728               "hash": 1534491824,
4729               "shard": 1
4730             },
4731             {
4732               "hash": 1534350760,
4733               "shard": 2
4734             }
4735            ],
4736            "total_chunk_size": 2048
4737           }
4738         }
4739       ]
4740     }
4741   ],
4742   "epoch": 0
4743 }
4744 EOF
4745
4746     fi
4747
4748     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
4749     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
4750     if test $getjson = "yes"
4751     then
4752       if [ "$allow_overwrites" = "true" ]
4753       then
4754         num=4
4755       else
4756         num=5
4757       fi
4758       jq '.' $dir/json > save${num}.json
4759     fi
4760
4761     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
4762     then
4763       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
4764     fi
4765
4766     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
4767     teardown $dir || return 1
4768 }
4769
4770 function TEST_corrupt_scrub_erasure_appends() {
4771     corrupt_scrub_erasure $1 false
4772 }
4773
4774 function TEST_corrupt_scrub_erasure_overwrites() {
4775     if [ "$use_ec_overwrite" = "true" ]; then
4776         corrupt_scrub_erasure $1 true
4777     fi
4778 }
4779
4780 #
4781 # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost
4782 #
4783 function TEST_periodic_scrub_replicated() {
4784     local dir=$1
4785     local poolname=psr_pool
4786     local objname=POBJ
4787
4788     setup $dir || return 1
4789     run_mon $dir a --osd_pool_default_size=2 || return 1
4790     run_mgr $dir x || return 1
4791     local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
4792     ceph_osd_args+="--osd_scrub_backoff_ratio=0"
4793     run_osd $dir 0 $ceph_osd_args || return 1
4794     run_osd $dir 1 $ceph_osd_args || return 1
4795     create_rbd_pool || return 1
4796     wait_for_clean || return 1
4797
4798     create_pool $poolname 1 1 || return 1
4799     wait_for_clean || return 1
4800
4801     local osd=0
4802     add_something $dir $poolname $objname scrub || return 1
4803     local primary=$(get_primary $poolname $objname)
4804     local pg=$(get_pg $poolname $objname)
4805
4806     # Add deep-scrub only error
4807     local payload=UVWXYZ
4808     echo $payload > $dir/CORRUPT
4809     # Uses $ceph_osd_args for osd restart
4810     objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
4811
4812     # No scrub information available, so expect failure
4813     set -o pipefail
4814     !  rados list-inconsistent-obj $pg | jq '.' || return 1
4815     set +o pipefail
4816
4817     pg_deep_scrub $pg || return 1
4818
4819     # Make sure bad object found
4820     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
4821
4822     flush_pg_stats
4823     local last_scrub=$(get_last_scrub_stamp $pg)
4824     # Fake a schedule scrub
4825     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
4826              trigger_scrub $pg || return 1
4827     # Wait for schedule regular scrub
4828     wait_for_scrub $pg "$last_scrub"
4829
4830     # It needed to be upgraded
4831     grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1
4832
4833     # Bad object still known
4834     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
4835
4836     # Can't upgrade with this set
4837     ceph osd set nodeep-scrub
4838     # Let map change propagate to OSDs
4839     flush pg_stats
4840     sleep 5
4841
4842     # Fake a schedule scrub
4843     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
4844              trigger_scrub $pg || return 1
4845     # Wait for schedule regular scrub
4846     # to notice scrub and skip it
4847     local found=false
4848     for i in $(seq 14 -1 0)
4849     do
4850       sleep 1
4851       ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; }
4852       echo Time left: $i seconds
4853     done
4854     test $found = "true" || return 1
4855
4856     # Bad object still known
4857     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
4858
4859     flush_pg_stats
4860     # Request a regular scrub and it will be done
4861     pg_scrub $pg
4862     grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1
4863
4864     # deep-scrub error is no longer present
4865     rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1
4866 }
4867
4868 #
4869 # Corrupt snapset in replicated pool
4870 #
4871 function TEST_corrupt_snapset_scrub_rep() {
4872     local dir=$1
4873     local poolname=csr_pool
4874     local total_objs=2
4875
4876     setup $dir || return 1
4877     run_mon $dir a --osd_pool_default_size=2 || return 1
4878     run_mgr $dir x || return 1
4879     run_osd $dir 0 || return 1
4880     run_osd $dir 1 || return 1
4881     create_rbd_pool || return 1
4882     wait_for_clean || return 1
4883
4884     create_pool foo 1 || return 1
4885     create_pool $poolname 1 1 || return 1
4886     wait_for_clean || return 1
4887
4888     for i in $(seq 1 $total_objs) ; do
4889         objname=ROBJ${i}
4890         add_something $dir $poolname $objname || return 1
4891
4892         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
4893         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
4894     done
4895
4896     local pg=$(get_pg $poolname ROBJ0)
4897
4898     for i in $(seq 1 $total_objs) ; do
4899         objname=ROBJ${i}
4900
4901         # Alternate corruption between osd.0 and osd.1
4902         local osd=$(expr $i % 2)
4903
4904         rados -p $poolname mksnap snap1
4905         echo -n head_of_snapshot_data > $dir/change
4906
4907         case $i in
4908         1)
4909           rados --pool $poolname put $objname $dir/change
4910           objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
4911           ;;
4912
4913         2)
4914           rados --pool $poolname put $objname $dir/change
4915           objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
4916           ;;
4917
4918         esac
4919     done
4920     rm $dir/change
4921
4922     pg_scrub $pg
4923
4924     rados list-inconsistent-pg $poolname > $dir/json || return 1
4925     # Check pg count
4926     test $(jq '. | length' $dir/json) = "1" || return 1
4927     # Check pgid
4928     test $(jq -r '.[0]' $dir/json) = $pg || return 1
4929
4930     rados list-inconsistent-obj $pg > $dir/json || return 1
4931
4932     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
4933 {
4934   "epoch": 34,
4935   "inconsistents": [
4936     {
4937       "object": {
4938         "name": "ROBJ1",
4939         "nspace": "",
4940         "locator": "",
4941         "snap": "head",
4942         "version": 8
4943       },
4944       "errors": [
4945         "snapset_inconsistency"
4946       ],
4947       "union_shard_errors": [],
4948       "selected_object_info": {
4949         "oid": {
4950           "oid": "ROBJ1",
4951           "key": "",
4952           "snapid": -2,
4953           "hash": 1454963827,
4954           "max": 0,
4955           "pool": 3,
4956           "namespace": ""
4957         },
4958         "version": "24'8",
4959         "prior_version": "21'3",
4960         "last_reqid": "client.4195.0:1",
4961         "user_version": 8,
4962         "size": 21,
4963         "mtime": "2018-04-05 14:35:43.286117",
4964         "local_mtime": "2018-04-05 14:35:43.288990",
4965         "lost": 0,
4966         "flags": [
4967           "dirty",
4968           "omap",
4969           "data_digest"
4970         ],
4971         "truncate_seq": 0,
4972         "truncate_size": 0,
4973         "data_digest": "0x53acb008",
4974         "omap_digest": "0xffffffff",
4975         "expected_object_size": 0,
4976         "expected_write_size": 0,
4977         "alloc_hint_flags": 0,
4978         "manifest": {
4979           "type": 0
4980         },
4981         "watchers": {}
4982       },
4983       "shards": [
4984         {
4985           "osd": 0,
4986           "primary": false,
4987           "errors": [],
4988           "size": 21,
4989           "snapset": {
4990             "clones": [
4991               {
4992                 "overlap": "[]",
4993                 "size": 7,
4994                 "snap": 1,
4995                 "snaps": [
4996                   1
4997                 ]
4998               }
4999             ],
5000             "head_exists": 1,
5001             "snap_context": {
5002               "seq": 1,
5003               "snaps": [
5004                 1
5005               ]
5006             }
5007           }
5008         },
5009         {
5010           "osd": 1,
5011           "primary": true,
5012           "errors": [],
5013           "size": 21,
5014           "snapset": {
5015             "clones": [],
5016             "head_exists": 0,
5017             "snap_context": {
5018               "seq": 0,
5019               "snaps": []
5020             }
5021           }
5022         }
5023       ]
5024     },
5025     {
5026       "object": {
5027         "name": "ROBJ2",
5028         "nspace": "",
5029         "locator": "",
5030         "snap": "head",
5031         "version": 10
5032       },
5033       "errors": [
5034         "snapset_inconsistency"
5035       ],
5036       "union_shard_errors": [],
5037       "selected_object_info": {
5038         "oid": {
5039           "oid": "ROBJ2",
5040           "key": "",
5041           "snapid": -2,
5042           "hash": 2026323607,
5043           "max": 0,
5044           "pool": 3,
5045           "namespace": ""
5046         },
5047         "version": "28'10",
5048         "prior_version": "23'6",
5049         "last_reqid": "client.4223.0:1",
5050         "user_version": 10,
5051         "size": 21,
5052         "mtime": "2018-04-05 14:35:48.326856",
5053         "local_mtime": "2018-04-05 14:35:48.328097",
5054         "lost": 0,
5055         "flags": [
5056           "dirty",
5057           "omap",
5058           "data_digest"
5059         ],
5060         "truncate_seq": 0,
5061         "truncate_size": 0,
5062         "data_digest": "0x53acb008",
5063         "omap_digest": "0xffffffff",
5064         "expected_object_size": 0,
5065         "expected_write_size": 0,
5066         "alloc_hint_flags": 0,
5067         "manifest": {
5068           "type": 0
5069         },
5070         "watchers": {}
5071       },
5072       "shards": [
5073         {
5074           "osd": 0,
5075           "primary": false,
5076           "errors": [],
5077           "size": 21,
5078           "snapset": {
5079             "clones": [],
5080             "head_exists": 0,
5081             "snap_context": {
5082               "seq": 0,
5083               "snaps": []
5084             }
5085           }
5086         },
5087         {
5088           "osd": 1,
5089           "primary": true,
5090           "errors": [],
5091           "size": 21,
5092           "snapset": {
5093             "clones": [
5094               {
5095                 "overlap": "[]",
5096                 "size": 7,
5097                 "snap": 1,
5098                 "snaps": [
5099                   1
5100                 ]
5101               }
5102             ],
5103             "head_exists": 1,
5104             "snap_context": {
5105               "seq": 1,
5106               "snaps": [
5107                 1
5108               ]
5109             }
5110           }
5111         }
5112       ]
5113     }
5114   ]
5115 }
5116 EOF
5117
5118     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
5119     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
5120     if test $getjson = "yes"
5121     then
5122         jq '.' $dir/json > save6.json
5123     fi
5124
5125     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
5126     then
5127       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
5128     fi
5129
5130     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
5131     teardown $dir || return 1
5132 }
5133
5134 main osd-scrub-repair "$@"
5135
5136 # Local Variables:
5137 # compile-command: "cd ../.. ; make -j4 && \
5138 #    test/osd/osd-scrub-repair.sh # TEST_corrupt_and_repair_replicated"
5139 # End: