ceph/qa/standalone/scrub/osd-scrub-repair.sh

   1 #!/bin/bash -x
   2 #
   3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
   4 #
   5 # Author: Loic Dachary <loic@dachary.org>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  18
  19 if [ `uname` = FreeBSD ]; then
  20     # erasure coding overwrites are only tested on Bluestore
  21     # erasure coding on filestore is unsafe
  22     # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites
  23     use_ec_overwrite=false
  24 else
  25     use_ec_overwrite=true
  26 fi
  27
  28 # Test development and debugging
  29 # Set to "yes" in order to ignore diff errors and save results to update test
  30 getjson="no"
  31
  32 # Filter out mtime and local_mtime dates, version, prior_version and last_reqid (client) from any object_info.
  33 jqfilter='def walk(f):
  34   . as $in
  35   | if type == "object" then
  36       reduce keys[] as $key
  37         ( {}; . + { ($key):  ($in[$key] | walk(f)) } ) | f
  38     elif type == "array" then map( walk(f) ) | f
  39     else f
  40     end;
  41 walk(if type == "object" then del(.mtime) else . end)
  42 | walk(if type == "object" then del(.local_mtime) else . end)
  43 | walk(if type == "object" then del(.last_reqid) else . end)
  44 | walk(if type == "object" then del(.version) else . end)
  45 | walk(if type == "object" then del(.prior_version) else . end)
  46 | walk(if type == "object" then del(.redirect_target) else . end)
  47 | walk(if type == "object" then del(.legacy_snaps) else . end)'
  48
  49 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
  50
  51 function run() {
  52     local dir=$1
  53     shift
  54
  55     export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
  56     export CEPH_ARGS
  57     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  58     CEPH_ARGS+="--mon-host=$CEPH_MON "
  59     CEPH_ARGS+="--osd-skip-data-digest=false "
  60
  61     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  62     for func in $funcs ; do
  63         $func $dir || return 1
  64     done
  65 }
  66
  67 function add_something() {
  68     local dir=$1
  69     local poolname=$2
  70     local obj=${3:-SOMETHING}
  71     local scrub=${4:-noscrub}
  72
  73     if [ "$scrub" = "noscrub" ];
  74     then
  75         ceph osd set noscrub || return 1
  76         ceph osd set nodeep-scrub || return 1
  77     else
  78         ceph osd unset noscrub || return 1
  79         ceph osd unset nodeep-scrub || return 1
  80     fi
  81
  82     local payload=ABCDEF
  83     echo $payload > $dir/ORIGINAL
  84     rados --pool $poolname put $obj $dir/ORIGINAL || return 1
  85 }
  86
  87 #
  88 # Corrupt one copy of a replicated pool
  89 #
  90 function TEST_corrupt_and_repair_replicated() {
  91     local dir=$1
  92     local poolname=rbd
  93
  94     setup $dir || return 1
  95     run_mon $dir a --osd_pool_default_size=2 || return 1
  96     run_mgr $dir x || return 1
  97     run_osd $dir 0 || return 1
  98     run_osd $dir 1 || return 1
  99     create_rbd_pool || return 1
 100     wait_for_clean || return 1
 101
 102     add_something $dir $poolname || return 1
 103     corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
 104     # Reproduces http://tracker.ceph.com/issues/8914
 105     corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1
 106
 107     teardown $dir || return 1
 108 }
 109
 110 function corrupt_and_repair_two() {
 111     local dir=$1
 112     local poolname=$2
 113     local first=$3
 114     local second=$4
 115
 116     #
 117     # 1) remove the corresponding file from the OSDs
 118     #
 119     pids=""
 120     run_in_background pids objectstore_tool $dir $first SOMETHING remove
 121     run_in_background pids objectstore_tool $dir $second SOMETHING remove
 122     wait_background pids
 123     return_code=$?
 124     if [ $return_code -ne 0 ]; then return $return_code; fi
 125
 126     #
 127     # 2) repair the PG
 128     #
 129     local pg=$(get_pg $poolname SOMETHING)
 130     repair $pg
 131     #
 132     # 3) The files must be back
 133     #
 134     pids=""
 135     run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs
 136     run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs
 137     wait_background pids
 138     return_code=$?
 139     if [ $return_code -ne 0 ]; then return $return_code; fi
 140
 141     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 142     diff $dir/ORIGINAL $dir/COPY || return 1
 143 }
 144
 145 #
 146 # 1) add an object
 147 # 2) remove the corresponding file from a designated OSD
 148 # 3) repair the PG
 149 # 4) check that the file has been restored in the designated OSD
 150 #
 151 function corrupt_and_repair_one() {
 152     local dir=$1
 153     local poolname=$2
 154     local osd=$3
 155
 156     #
 157     # 1) remove the corresponding file from the OSD
 158     #
 159     objectstore_tool $dir $osd SOMETHING remove || return 1
 160     #
 161     # 2) repair the PG
 162     #
 163     local pg=$(get_pg $poolname SOMETHING)
 164     repair $pg
 165     #
 166     # 3) The file must be back
 167     #
 168     objectstore_tool $dir $osd SOMETHING list-attrs || return 1
 169     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 170     diff $dir/ORIGINAL $dir/COPY || return 1
 171 }
 172
 173 function corrupt_and_repair_erasure_coded() {
 174     local dir=$1
 175     local poolname=$2
 176
 177     add_something $dir $poolname || return 1
 178
 179     local primary=$(get_primary $poolname SOMETHING)
 180     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 181     local not_primary_first=${osds[0]}
 182     local not_primary_second=${osds[1]}
 183
 184     # Reproduces http://tracker.ceph.com/issues/10017
 185     corrupt_and_repair_one $dir $poolname $primary  || return 1
 186     # Reproduces http://tracker.ceph.com/issues/10409
 187     corrupt_and_repair_one $dir $poolname $not_primary_first || return 1
 188     corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1
 189     corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1
 190
 191 }
 192
 193 function create_ec_pool() {
 194     local pool_name=$1
 195     local allow_overwrites=$2
 196
 197     ceph osd erasure-code-profile set myprofile crush-failure-domain=osd $3 $4 $5 $6 $7 || return 1
 198
 199     create_pool "$poolname" 1 1 erasure myprofile || return 1
 200
 201     if [ "$allow_overwrites" = "true" ]; then
 202         ceph osd pool set "$poolname" allow_ec_overwrites true || return 1
 203     fi
 204
 205     wait_for_clean || return 1
 206     return 0
 207 }
 208
 209 function auto_repair_erasure_coded() {
 210     local dir=$1
 211     local allow_overwrites=$2
 212     local poolname=ecpool
 213
 214     # Launch a cluster with 5 seconds scrub interval
 215     setup $dir || return 1
 216     run_mon $dir a || return 1
 217     run_mgr $dir x || return 1
 218     local ceph_osd_args="--osd-scrub-auto-repair=true \
 219             --osd-deep-scrub-interval=5 \
 220             --osd-scrub-max-interval=5 \
 221             --osd-scrub-min-interval=5 \
 222             --osd-scrub-interval-randomize-ratio=0"
 223     for id in $(seq 0 2) ; do
 224         if [ "$allow_overwrites" = "true" ]; then
 225             run_osd_bluestore $dir $id $ceph_osd_args || return 1
 226         else
 227             run_osd $dir $id $ceph_osd_args || return 1
 228         fi
 229     done
 230     create_rbd_pool || return 1
 231     wait_for_clean || return 1
 232
 233     # Create an EC pool
 234     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 235
 236     # Put an object
 237     local payload=ABCDEF
 238     echo $payload > $dir/ORIGINAL
 239     rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
 240
 241     # Remove the object from one shard physically
 242     # Restarted osd get $ceph_osd_args passed
 243     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
 244     # Wait for auto repair
 245     local pgid=$(get_pg $poolname SOMETHING)
 246     wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)"
 247     wait_for_clean || return 1
 248     # Verify - the file should be back
 249     # Restarted osd get $ceph_osd_args passed
 250     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
 251     rados --pool $poolname get SOMETHING $dir/COPY || return 1
 252     diff $dir/ORIGINAL $dir/COPY || return 1
 253
 254     # Tear down
 255     teardown $dir || return 1
 256 }
 257
 258 function TEST_auto_repair_erasure_coded_appends() {
 259     auto_repair_erasure_coded $1 false
 260 }
 261
 262 function TEST_auto_repair_erasure_coded_overwrites() {
 263     if [ "$use_ec_overwrite" = "true" ]; then
 264         auto_repair_erasure_coded $1 true
 265     fi
 266 }
 267
 268 function corrupt_and_repair_jerasure() {
 269     local dir=$1
 270     local allow_overwrites=$2
 271     local poolname=ecpool
 272
 273     setup $dir || return 1
 274     run_mon $dir a || return 1
 275     run_mgr $dir x || return 1
 276     for id in $(seq 0 3) ; do
 277         if [ "$allow_overwrites" = "true" ]; then
 278             run_osd_bluestore $dir $id || return 1
 279         else
 280             run_osd $dir $id || return 1
 281         fi
 282     done
 283     create_rbd_pool || return 1
 284     wait_for_clean || return 1
 285
 286     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 287     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 288
 289     teardown $dir || return 1
 290 }
 291
 292 function TEST_corrupt_and_repair_jerasure_appends() {
 293     corrupt_and_repair_jerasure $1
 294 }
 295
 296 function TEST_corrupt_and_repair_jerasure_overwrites() {
 297     if [ "$use_ec_overwrite" = "true" ]; then
 298         corrupt_and_repair_jerasure $1 true
 299     fi
 300 }
 301
 302 function corrupt_and_repair_lrc() {
 303     local dir=$1
 304     local allow_overwrites=$2
 305     local poolname=ecpool
 306
 307     setup $dir || return 1
 308     run_mon $dir a || return 1
 309     run_mgr $dir x || return 1
 310     for id in $(seq 0 9) ; do
 311         if [ "$allow_overwrites" = "true" ]; then
 312             run_osd_bluestore $dir $id || return 1
 313         else
 314             run_osd $dir $id || return 1
 315         fi
 316     done
 317     create_rbd_pool || return 1
 318     wait_for_clean || return 1
 319
 320     create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1
 321     corrupt_and_repair_erasure_coded $dir $poolname || return 1
 322
 323     teardown $dir || return 1
 324 }
 325
 326 function TEST_corrupt_and_repair_lrc_appends() {
 327     corrupt_and_repair_jerasure $1
 328 }
 329
 330 function TEST_corrupt_and_repair_lrc_overwrites() {
 331     if [ "$use_ec_overwrite" = "true" ]; then
 332         corrupt_and_repair_jerasure $1 true
 333     fi
 334 }
 335
 336 function unfound_erasure_coded() {
 337     local dir=$1
 338     local allow_overwrites=$2
 339     local poolname=ecpool
 340     local payload=ABCDEF
 341
 342     setup $dir || return 1
 343     run_mon $dir a || return 1
 344     run_mgr $dir x || return 1
 345     for id in $(seq 0 3) ; do
 346         if [ "$allow_overwrites" = "true" ]; then
 347             run_osd_bluestore $dir $id || return 1
 348         else
 349             run_osd $dir $id || return 1
 350         fi
 351     done
 352     create_rbd_pool || return 1
 353     wait_for_clean || return 1
 354
 355     create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
 356
 357     add_something $dir $poolname || return 1
 358
 359     local primary=$(get_primary $poolname SOMETHING)
 360     local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
 361     local not_primary_first=${osds[0]}
 362     local not_primary_second=${osds[1]}
 363     local not_primary_third=${osds[2]}
 364
 365     #
 366     # 1) remove the corresponding file from the OSDs
 367     #
 368     pids=""
 369     run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove
 370     run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove
 371     run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove
 372     wait_background pids
 373     return_code=$?
 374     if [ $return_code -ne 0 ]; then return $return_code; fi
 375
 376     #
 377     # 2) repair the PG
 378     #
 379     local pg=$(get_pg $poolname SOMETHING)
 380     repair $pg
 381     #
 382     # 3) check pg state
 383     #
 384     # it may take a bit to appear due to mon/mgr asynchrony
 385     for f in `seq 1 60`; do
 386         ceph -s | grep "1/1 objects unfound" && break
 387         sleep 1
 388     done
 389     ceph -s|grep "4 osds: 4 up, 4 in" || return 1
 390     ceph -s|grep "1/1 objects unfound" || return 1
 391
 392     teardown $dir || return 1
 393 }
 394
 395 function TEST_unfound_erasure_coded_appends() {
 396     unfound_erasure_coded $1
 397 }
 398
 399 function TEST_unfound_erasure_coded_overwrites() {
 400     if [ "$use_ec_overwrite" = "true" ]; then
 401         unfound_erasure_coded $1 true
 402     fi
 403 }
 404
 405 #
 406 # list_missing for EC pool
 407 #
 408 function list_missing_erasure_coded() {
 409     local dir=$1
 410     local allow_overwrites=$2
 411     local poolname=ecpool
 412
 413     setup $dir || return 1
 414     run_mon $dir a || return 1
 415     run_mgr $dir x || return 1
 416     for id in $(seq 0 2) ; do
 417         if [ "$allow_overwrites" = "true" ]; then
 418             run_osd_bluestore $dir $id || return 1
 419         else
 420             run_osd $dir $id || return 1
 421         fi
 422     done
 423     create_rbd_pool || return 1
 424     wait_for_clean || return 1
 425
 426     create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
 427
 428     # Put an object and remove the two shards (including primary)
 429     add_something $dir $poolname MOBJ0 || return 1
 430     local -a osds0=($(get_osds $poolname MOBJ0))
 431
 432     # Put another object and remove two shards (excluding primary)
 433     add_something $dir $poolname MOBJ1 || return 1
 434     local -a osds1=($(get_osds $poolname MOBJ1))
 435
 436     # Stop all osd daemons
 437     for id in $(seq 0 2) ; do
 438         kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1
 439     done
 440
 441     id=${osds0[0]}
 442     ceph-objectstore-tool --data-path $dir/$id \
 443         MOBJ0 remove || return 1
 444     id=${osds0[1]}
 445     ceph-objectstore-tool --data-path $dir/$id \
 446         MOBJ0 remove || return 1
 447
 448     id=${osds1[1]}
 449     ceph-objectstore-tool --data-path $dir/$id \
 450         MOBJ1 remove || return 1
 451     id=${osds1[2]}
 452     ceph-objectstore-tool --data-path $dir/$id \
 453         MOBJ1 remove || return 1
 454
 455     for id in $(seq 0 2) ; do
 456         activate_osd $dir $id >&2 || return 1
 457     done
 458     create_rbd_pool || return 1
 459     wait_for_clean || return 1
 460
 461     # Get get - both objects should in the same PG
 462     local pg=$(get_pg $poolname MOBJ0)
 463
 464     # Repair the PG, which triggers the recovering,
 465     # and should mark the object as unfound
 466     repair $pg
 467
 468     for i in $(seq 0 120) ; do
 469         [ $i -lt 60 ] || return 1
 470         matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l)
 471         [ $matches -eq 2 ] && break
 472     done
 473
 474     teardown $dir || return 1
 475 }
 476
 477 function TEST_list_missing_erasure_coded_appends() {
 478     list_missing_erasure_coded $1 false
 479 }
 480
 481 function TEST_list_missing_erasure_coded_overwrites() {
 482     if [ "$use_ec_overwrite" = "true" ]; then
 483         list_missing_erasure_coded $1 true
 484     fi
 485 }
 486
 487 #
 488 # Corrupt one copy of a replicated pool
 489 #
 490 function TEST_corrupt_scrub_replicated() {
 491     local dir=$1
 492     local poolname=csr_pool
 493     local total_objs=18
 494
 495     setup $dir || return 1
 496     run_mon $dir a --osd_pool_default_size=2 || return 1
 497     run_mgr $dir x || return 1
 498     run_osd $dir 0 || return 1
 499     run_osd $dir 1 || return 1
 500     create_rbd_pool || return 1
 501     wait_for_clean || return 1
 502
 503     create_pool foo 1 || return 1
 504     create_pool $poolname 1 1 || return 1
 505     wait_for_clean || return 1
 506
 507     for i in $(seq 1 $total_objs) ; do
 508         objname=ROBJ${i}
 509         add_something $dir $poolname $objname || return 1
 510
 511         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
 512         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
 513     done
 514
 515     local pg=$(get_pg $poolname ROBJ0)
 516
 517     # Compute an old omap digest and save oi
 518     CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \
 519         config set osd_deep_scrub_update_digest_min_age 0
 520     CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \
 521         config set osd_deep_scrub_update_digest_min_age 0
 522     pg_deep_scrub $pg
 523
 524     for i in $(seq 1 $total_objs) ; do
 525         objname=ROBJ${i}
 526
 527         # Alternate corruption between osd.0 and osd.1
 528         local osd=$(expr $i % 2)
 529
 530         case $i in
 531         1)
 532             # Size (deep scrub data_digest too)
 533             local payload=UVWXYZZZ
 534             echo $payload > $dir/CORRUPT
 535             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 536             ;;
 537
 538         2)
 539             # digest (deep scrub only)
 540             local payload=UVWXYZ
 541             echo $payload > $dir/CORRUPT
 542             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
 543             ;;
 544
 545         3)
 546              # missing
 547              objectstore_tool $dir $osd $objname remove || return 1
 548              ;;
 549
 550          4)
 551              # Modify omap value (deep scrub only)
 552              objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1
 553              ;;
 554
 555          5)
 556             # Delete omap key (deep scrub only)
 557             objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1
 558             ;;
 559
 560          6)
 561             # Add extra omap key (deep scrub only)
 562             echo extra > $dir/extra-val
 563             objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1
 564             rm $dir/extra-val
 565             ;;
 566
 567          7)
 568             # Modify omap header (deep scrub only)
 569             echo -n newheader > $dir/hdr
 570             objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1
 571             rm $dir/hdr
 572             ;;
 573
 574          8)
 575             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
 576             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
 577
 578             # Break xattrs
 579             echo -n bad-val > $dir/bad-val
 580             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
 581             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
 582             echo -n val3-$objname > $dir/newval
 583             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
 584             rm $dir/bad-val $dir/newval
 585             ;;
 586
 587         9)
 588             objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi
 589             echo -n D > $dir/change
 590             rados --pool $poolname put $objname $dir/change
 591             objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi
 592             rm $dir/oi $dir/change
 593             ;;
 594
 595           # ROBJ10 must be handled after digests are re-computed by a deep scrub below
 596           # ROBJ11 must be handled with config change before deep scrub
 597           # ROBJ12 must be handled with config change before scrubs
 598           # ROBJ13 must be handled before scrubs
 599
 600         14)
 601             echo -n bad-val > $dir/bad-val
 602             objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1
 603             objectstore_tool $dir 1 $objname rm-attr _ || return 1
 604             rm $dir/bad-val
 605             ;;
 606
 607         15)
 608             objectstore_tool $dir $osd $objname rm-attr _ || return 1
 609             ;;
 610
 611         16)
 612             objectstore_tool $dir 0 $objname rm-attr snapset || return 1
 613             echo -n bad-val > $dir/bad-val
 614             objectstore_tool $dir 1 $objname set-attr snapset $dir/bad-val || return 1
 615             ;;
 616
 617         17)
 618             # Deep-scrub only (all replicas are diffent than the object info
 619            local payload=ROBJ17
 620            echo $payload > $dir/new.ROBJ17
 621            objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ17 || return 1
 622            objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ17 || return 1
 623            ;;
 624
 625         18)
 626             # Deep-scrub only (all replicas are diffent than the object info
 627            local payload=ROBJ18
 628            echo $payload > $dir/new.ROBJ18
 629            objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ18 || return 1
 630            objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ18 || return 1
 631            # Make one replica have a different object info, so a full repair must happen too
 632            objectstore_tool $dir $osd $objname corrupt-info || return 1
 633
 634         esac
 635     done
 636
 637     local pg=$(get_pg $poolname ROBJ0)
 638
 639     inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
 640     inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
 641     inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
 642     inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
 643
 644     pg_scrub $pg
 645
 646     rados list-inconsistent-pg $poolname > $dir/json || return 1
 647     # Check pg count
 648     test $(jq '. | length' $dir/json) = "1" || return 1
 649     # Check pgid
 650     test $(jq -r '.[0]' $dir/json) = $pg || return 1
 651
 652     rados list-inconsistent-obj $pg > $dir/json || return 1
 653     # Get epoch for repair-get requests
 654     epoch=$(jq .epoch $dir/json)
 655
 656     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
 657 {
 658   "inconsistents": [
 659     {
 660       "shards": [
 661         {
 662           "size": 7,
 663           "errors": [],
 664           "osd": 0,
 665           "primary": false
 666         },
 667         {
 668           "object_info": {
 669             "oid": {
 670               "oid": "ROBJ1",
 671               "key": "",
 672               "snapid": -2,
 673               "hash": 1454963827,
 674               "max": 0,
 675               "pool": 3,
 676               "namespace": ""
 677             },
 678             "version": "51'58",
 679             "prior_version": "21'3",
 680             "last_reqid": "osd.1.0:57",
 681             "user_version": 3,
 682             "size": 7,
 683             "mtime": "",
 684             "local_mtime": "",
 685             "lost": 0,
 686             "flags": [
 687               "dirty",
 688               "omap",
 689               "data_digest",
 690               "omap_digest"
 691             ],
 692             "truncate_seq": 0,
 693             "truncate_size": 0,
 694             "data_digest": "0x2ddbf8f5",
 695             "omap_digest": "0xf5fba2c6",
 696             "expected_object_size": 0,
 697             "expected_write_size": 0,
 698             "alloc_hint_flags": 0,
 699             "manifest": {
 700               "type": 0
 701             },
 702             "watchers": {}
 703           },
 704           "size": 9,
 705           "errors": [
 706             "size_mismatch_info",
 707             "obj_size_info_mismatch"
 708           ],
 709           "osd": 1,
 710           "primary": true
 711         }
 712       ],
 713       "selected_object_info": {
 714         "oid": {
 715           "oid": "ROBJ1",
 716           "key": "",
 717           "snapid": -2,
 718           "hash": 1454963827,
 719           "max": 0,
 720           "pool": 3,
 721           "namespace": ""
 722         },
 723         "version": "51'58",
 724         "prior_version": "21'3",
 725         "last_reqid": "osd.1.0:57",
 726         "user_version": 3,
 727         "size": 7,
 728         "mtime": "2018-04-05 14:33:19.804040",
 729         "local_mtime": "2018-04-05 14:33:19.804839",
 730         "lost": 0,
 731         "flags": [
 732           "dirty",
 733           "omap",
 734           "data_digest",
 735           "omap_digest"
 736         ],
 737         "truncate_seq": 0,
 738         "truncate_size": 0,
 739         "data_digest": "0x2ddbf8f5",
 740         "omap_digest": "0xf5fba2c6",
 741         "expected_object_size": 0,
 742         "expected_write_size": 0,
 743         "alloc_hint_flags": 0,
 744         "manifest": {
 745           "type": 0
 746         },
 747         "watchers": {}
 748       },
 749       "union_shard_errors": [
 750         "size_mismatch_info",
 751         "obj_size_info_mismatch"
 752       ],
 753       "errors": [
 754         "size_mismatch"
 755       ],
 756       "object": {
 757         "version": 3,
 758         "snap": "head",
 759         "locator": "",
 760         "nspace": "",
 761         "name": "ROBJ1"
 762       }
 763     },
 764     {
 765       "shards": [
 766         {
 767           "errors": [
 768             "stat_error"
 769           ],
 770           "osd": 0,
 771           "primary": false
 772         },
 773         {
 774           "size": 7,
 775           "errors": [],
 776           "osd": 1,
 777           "primary": true
 778         }
 779       ],
 780       "selected_object_info": {
 781         "oid": {
 782           "oid": "ROBJ12",
 783           "key": "",
 784           "snapid": -2,
 785           "hash": 3920199997,
 786           "max": 0,
 787           "pool": 3,
 788           "namespace": ""
 789         },
 790         "version": "51'56",
 791         "prior_version": "43'36",
 792         "last_reqid": "osd.1.0:55",
 793         "user_version": 36,
 794         "size": 7,
 795         "mtime": "",
 796         "local_mtime": "",
 797         "lost": 0,
 798         "flags": [
 799           "dirty",
 800           "omap",
 801           "data_digest",
 802           "omap_digest"
 803         ],
 804         "truncate_seq": 0,
 805         "truncate_size": 0,
 806         "data_digest": "0x2ddbf8f5",
 807         "omap_digest": "0x067f306a",
 808         "expected_object_size": 0,
 809         "expected_write_size": 0,
 810         "alloc_hint_flags": 0,
 811         "manifest": {
 812           "type": 0
 813         },
 814         "watchers": {}
 815       },
 816       "union_shard_errors": [
 817         "stat_error"
 818       ],
 819       "errors": [],
 820       "object": {
 821         "version": 36,
 822         "snap": "head",
 823         "locator": "",
 824         "nspace": "",
 825         "name": "ROBJ12"
 826       }
 827     },
 828     {
 829       "shards": [
 830         {
 831           "errors": [
 832             "stat_error"
 833           ],
 834           "osd": 0,
 835           "primary": false
 836         },
 837         {
 838           "size": 7,
 839           "errors": [],
 840           "osd": 1,
 841           "primary": true
 842         }
 843       ],
 844       "selected_object_info": {
 845         "oid": {
 846           "oid": "ROBJ13",
 847           "key": "",
 848           "snapid": -2,
 849           "hash": 2682806379,
 850           "max": 0,
 851           "pool": 3,
 852           "namespace": ""
 853         },
 854         "version": "51'59",
 855         "prior_version": "45'39",
 856         "last_reqid": "osd.1.0:58",
 857         "user_version": 39,
 858         "size": 7,
 859         "mtime": "",
 860         "local_mtime": "",
 861         "lost": 0,
 862         "flags": [
 863           "dirty",
 864           "omap",
 865           "data_digest",
 866           "omap_digest"
 867         ],
 868         "truncate_seq": 0,
 869         "truncate_size": 0,
 870         "data_digest": "0x2ddbf8f5",
 871         "omap_digest": "0x6441854d",
 872         "expected_object_size": 0,
 873         "expected_write_size": 0,
 874         "alloc_hint_flags": 0,
 875         "manifest": {
 876           "type": 0
 877         },
 878         "watchers": {}
 879       },
 880       "union_shard_errors": [
 881         "stat_error"
 882       ],
 883       "errors": [],
 884       "object": {
 885         "version": 39,
 886         "snap": "head",
 887         "locator": "",
 888         "nspace": "",
 889         "name": "ROBJ13"
 890       }
 891     },
 892     {
 893       "shards": [
 894         {
 895           "object_info": "bad-val",
 896           "size": 7,
 897           "errors": [
 898             "info_corrupted"
 899           ],
 900           "osd": 0,
 901           "primary": false
 902         },
 903         {
 904           "size": 7,
 905           "errors": [
 906             "info_missing"
 907           ],
 908           "osd": 1,
 909           "primary": true
 910         }
 911       ],
 912       "union_shard_errors": [
 913         "info_missing",
 914         "info_corrupted"
 915       ],
 916       "errors": [],
 917       "object": {
 918         "version": 0,
 919         "snap": "head",
 920         "locator": "",
 921         "nspace": "",
 922         "name": "ROBJ14"
 923       }
 924     },
 925     {
 926       "shards": [
 927         {
 928           "object_info": {
 929             "oid": {
 930               "oid": "ROBJ15",
 931               "key": "",
 932               "snapid": -2,
 933               "hash": 504996876,
 934               "max": 0,
 935               "pool": 3,
 936               "namespace": ""
 937             },
 938             "version": "51'49",
 939             "prior_version": "49'45",
 940             "last_reqid": "osd.1.0:48",
 941             "user_version": 45,
 942             "size": 7,
 943             "mtime": "2018-04-05 14:33:29.498969",
 944             "local_mtime": "2018-04-05 14:33:29.499890",
 945             "lost": 0,
 946             "flags": [
 947               "dirty",
 948               "omap",
 949               "data_digest",
 950               "omap_digest"
 951             ],
 952             "truncate_seq": 0,
 953             "truncate_size": 0,
 954             "data_digest": "0x2ddbf8f5",
 955             "omap_digest": "0x2d2a4d6e",
 956             "expected_object_size": 0,
 957             "expected_write_size": 0,
 958             "alloc_hint_flags": 0,
 959             "manifest": {
 960               "type": 0
 961             },
 962             "watchers": {}
 963           },
 964           "size": 7,
 965           "errors": [],
 966           "osd": 0,
 967           "primary": false
 968         },
 969         {
 970           "size": 7,
 971           "errors": [
 972             "info_missing"
 973           ],
 974           "osd": 1,
 975           "primary": true
 976         }
 977       ],
 978       "selected_object_info": {
 979         "oid": {
 980           "oid": "ROBJ15",
 981           "key": "",
 982           "snapid": -2,
 983           "hash": 504996876,
 984           "max": 0,
 985           "pool": 3,
 986           "namespace": ""
 987         },
 988         "version": "51'49",
 989         "prior_version": "49'45",
 990         "last_reqid": "osd.1.0:48",
 991         "user_version": 45,
 992         "size": 7,
 993         "mtime": "",
 994         "local_mtime": "",
 995         "lost": 0,
 996         "flags": [
 997           "dirty",
 998           "omap",
 999           "data_digest",
1000           "omap_digest"
1001         ],
1002         "truncate_seq": 0,
1003         "truncate_size": 0,
1004         "data_digest": "0x2ddbf8f5",
1005         "omap_digest": "0x2d2a4d6e",
1006         "expected_object_size": 0,
1007         "expected_write_size": 0,
1008         "alloc_hint_flags": 0,
1009         "manifest": {
1010           "type": 0
1011         },
1012         "watchers": {}
1013       },
1014       "union_shard_errors": [
1015         "info_missing"
1016       ],
1017       "errors": [],
1018       "object": {
1019         "version": 45,
1020         "snap": "head",
1021         "locator": "",
1022         "nspace": "",
1023         "name": "ROBJ15"
1024       }
1025     },
1026     {
1027       "errors": [],
1028       "object": {
1029       "locator": "",
1030       "name": "ROBJ16",
1031       "nspace": "",
1032       "snap": "head",
1033       "version": 0
1034        },
1035         "shards": [
1036       {
1037         "errors": [
1038           "snapset_missing"
1039         ],
1040         "osd": 0,
1041         "primary": false,
1042         "size": 7
1043       },
1044       {
1045         "errors": [
1046           "snapset_corrupted"
1047         ],
1048         "osd": 1,
1049         "primary": true,
1050         "snapset": "bad-val",
1051         "size": 7
1052       }
1053       ],
1054       "union_shard_errors": [
1055         "snapset_missing",
1056         "snapset_corrupted"
1057       ]
1058     },
1059     {
1060      "errors": [
1061        "object_info_inconsistency"
1062      ],
1063      "object": {
1064        "locator": "",
1065        "name": "ROBJ18",
1066        "nspace": "",
1067        "snap": "head"
1068      },
1069      "selected_object_info": {
1070        "alloc_hint_flags": 255,
1071        "data_digest": "0x2ddbf8f5",
1072        "expected_object_size": 0,
1073        "expected_write_size": 0,
1074        "flags": [
1075          "dirty",
1076          "omap",
1077          "data_digest",
1078          "omap_digest"
1079        ],
1080        "lost": 0,
1081        "manifest": {
1082          "type": 0
1083        },
1084        "oid": {
1085          "hash": 1629828556,
1086          "key": "",
1087          "max": 0,
1088          "namespace": "",
1089          "oid": "ROBJ18",
1090          "pool": 3,
1091          "snapid": -2
1092        },
1093        "omap_digest": "0xddc3680f",
1094        "size": 7,
1095        "truncate_seq": 0,
1096        "truncate_size": 0,
1097        "user_version": 54,
1098        "watchers": {}
1099      },
1100      "shards": [
1101        {
1102          "errors": [],
1103          "object_info": {
1104            "alloc_hint_flags": 0,
1105            "data_digest": "0x2ddbf8f5",
1106            "expected_object_size": 0,
1107            "expected_write_size": 0,
1108            "flags": [
1109              "dirty",
1110              "omap",
1111              "data_digest",
1112              "omap_digest"
1113            ],
1114            "lost": 0,
1115            "manifest": {
1116              "type": 0
1117            },
1118            "oid": {
1119              "hash": 1629828556,
1120              "key": "",
1121              "max": 0,
1122              "namespace": "",
1123              "oid": "ROBJ18",
1124              "pool": 3,
1125              "snapid": -2
1126            },
1127            "omap_digest": "0xddc3680f",
1128            "size": 7,
1129            "truncate_seq": 0,
1130            "truncate_size": 0,
1131            "user_version": 54,
1132            "watchers": {}
1133          },
1134          "osd": 0,
1135          "primary": false,
1136          "size": 7
1137        },
1138        {
1139          "errors": [],
1140          "object_info": {
1141            "alloc_hint_flags": 255,
1142            "data_digest": "0x2ddbf8f5",
1143            "expected_object_size": 0,
1144            "expected_write_size": 0,
1145            "flags": [
1146              "dirty",
1147              "omap",
1148              "data_digest",
1149              "omap_digest"
1150            ],
1151            "lost": 0,
1152            "manifest": {
1153              "type": 0
1154            },
1155            "oid": {
1156              "hash": 1629828556,
1157              "key": "",
1158              "max": 0,
1159              "namespace": "",
1160              "oid": "ROBJ18",
1161              "pool": 3,
1162              "snapid": -2
1163            },
1164            "omap_digest": "0xddc3680f",
1165            "size": 7,
1166            "truncate_seq": 0,
1167            "truncate_size": 0,
1168            "user_version": 54,
1169            "watchers": {}
1170          },
1171          "osd": 1,
1172          "primary": true,
1173          "size": 7
1174        }
1175      ],
1176      "union_shard_errors": []
1177    },
1178    {
1179       "shards": [
1180         {
1181           "size": 7,
1182           "errors": [],
1183           "osd": 0,
1184           "primary": false
1185         },
1186         {
1187           "errors": [
1188             "missing"
1189           ],
1190           "osd": 1,
1191           "primary": true
1192         }
1193       ],
1194       "selected_object_info": {
1195         "oid": {
1196           "oid": "ROBJ3",
1197           "key": "",
1198           "snapid": -2,
1199           "hash": 625845583,
1200           "max": 0,
1201           "pool": 3,
1202           "namespace": ""
1203         },
1204         "version": "51'61",
1205         "prior_version": "25'9",
1206         "last_reqid": "osd.1.0:60",
1207         "user_version": 9,
1208         "size": 7,
1209         "mtime": "",
1210         "local_mtime": "",
1211         "lost": 0,
1212         "flags": [
1213           "dirty",
1214           "omap",
1215           "data_digest",
1216           "omap_digest"
1217         ],
1218         "truncate_seq": 0,
1219         "truncate_size": 0,
1220         "data_digest": "0x2ddbf8f5",
1221         "omap_digest": "0x00b35dfd",
1222         "expected_object_size": 0,
1223         "expected_write_size": 0,
1224         "alloc_hint_flags": 0,
1225         "manifest": {
1226           "type": 0
1227         },
1228         "watchers": {}
1229       },
1230       "union_shard_errors": [
1231         "missing"
1232       ],
1233       "errors": [],
1234       "object": {
1235         "version": 9,
1236         "snap": "head",
1237         "locator": "",
1238         "nspace": "",
1239         "name": "ROBJ3"
1240       }
1241     },
1242     {
1243       "shards": [
1244         {
1245           "attrs": [
1246             {
1247               "Base64": false,
1248               "value": "bad-val",
1249               "name": "key1-ROBJ8"
1250             },
1251             {
1252               "Base64": false,
1253               "value": "val2-ROBJ8",
1254               "name": "key2-ROBJ8"
1255             }
1256           ],
1257           "size": 7,
1258           "errors": [],
1259           "osd": 0,
1260           "primary": false
1261         },
1262         {
1263           "attrs": [
1264             {
1265               "Base64": false,
1266               "value": "val1-ROBJ8",
1267               "name": "key1-ROBJ8"
1268             },
1269             {
1270               "Base64": false,
1271               "value": "val3-ROBJ8",
1272               "name": "key3-ROBJ8"
1273             }
1274           ],
1275           "size": 7,
1276           "errors": [],
1277           "osd": 1,
1278           "primary": true
1279         }
1280       ],
1281       "selected_object_info": {
1282         "oid": {
1283           "oid": "ROBJ8",
1284           "key": "",
1285           "snapid": -2,
1286           "hash": 2359695969,
1287           "max": 0,
1288           "pool": 3,
1289           "namespace": ""
1290         },
1291         "version": "79'66",
1292         "prior_version": "79'65",
1293         "last_reqid": "client.4554.0:1",
1294         "user_version": 74,
1295         "size": 7,
1296         "mtime": "",
1297         "local_mtime": "",
1298         "lost": 0,
1299         "flags": [
1300           "dirty",
1301           "omap",
1302           "data_digest",
1303           "omap_digest"
1304         ],
1305         "truncate_seq": 0,
1306         "truncate_size": 0,
1307         "data_digest": "0x2ddbf8f5",
1308         "omap_digest": "0xd6be81dc",
1309         "expected_object_size": 0,
1310         "expected_write_size": 0,
1311         "alloc_hint_flags": 0,
1312         "manifest": {
1313           "type": 0
1314         },
1315         "watchers": {}
1316       },
1317       "union_shard_errors": [],
1318       "errors": [
1319         "attr_value_mismatch",
1320         "attr_name_mismatch"
1321       ],
1322       "object": {
1323         "version": 66,
1324         "snap": "head",
1325         "locator": "",
1326         "nspace": "",
1327         "name": "ROBJ8"
1328       }
1329     },
1330     {
1331       "shards": [
1332         {
1333           "object_info": {
1334             "oid": {
1335               "oid": "ROBJ9",
1336               "key": "",
1337               "snapid": -2,
1338               "hash": 537189375,
1339               "max": 0,
1340               "pool": 3,
1341               "namespace": ""
1342             },
1343             "version": "95'67",
1344             "prior_version": "51'64",
1345             "last_reqid": "client.4649.0:1",
1346             "user_version": 75,
1347             "size": 1,
1348             "mtime": "",
1349             "local_mtime": "",
1350             "lost": 0,
1351             "flags": [
1352               "dirty",
1353               "omap",
1354               "data_digest",
1355               "omap_digest"
1356             ],
1357             "truncate_seq": 0,
1358             "truncate_size": 0,
1359             "data_digest": "0x2b63260d",
1360             "omap_digest": "0x2eecc539",
1361             "expected_object_size": 0,
1362             "expected_write_size": 0,
1363             "alloc_hint_flags": 0,
1364             "manifest": {
1365               "type": 0
1366             },
1367             "watchers": {}
1368           },
1369           "size": 1,
1370           "errors": [],
1371           "osd": 0,
1372           "primary": false
1373         },
1374         {
1375           "object_info": {
1376             "oid": {
1377               "oid": "ROBJ9",
1378               "key": "",
1379               "snapid": -2,
1380               "hash": 537189375,
1381               "max": 0,
1382               "pool": 3,
1383               "namespace": ""
1384             },
1385             "version": "51'64",
1386             "prior_version": "37'27",
1387             "last_reqid": "osd.1.0:63",
1388             "user_version": 27,
1389             "size": 7,
1390             "mtime": "2018-04-05 14:33:25.352485",
1391             "local_mtime": "2018-04-05 14:33:25.353746",
1392             "lost": 0,
1393             "flags": [
1394               "dirty",
1395               "omap",
1396               "data_digest",
1397               "omap_digest"
1398             ],
1399             "truncate_seq": 0,
1400             "truncate_size": 0,
1401             "data_digest": "0x2ddbf8f5",
1402             "omap_digest": "0x2eecc539",
1403             "expected_object_size": 0,
1404             "expected_write_size": 0,
1405             "alloc_hint_flags": 0,
1406             "manifest": {
1407               "type": 0
1408             },
1409             "watchers": {}
1410           },
1411           "size": 1,
1412           "errors": [
1413             "obj_size_info_mismatch"
1414           ],
1415           "osd": 1,
1416           "primary": true
1417         }
1418       ],
1419       "selected_object_info": {
1420         "oid": {
1421           "oid": "ROBJ9",
1422           "key": "",
1423           "snapid": -2,
1424           "hash": 537189375,
1425           "max": 0,
1426           "pool": 3,
1427           "namespace": ""
1428         },
1429         "version": "95'67",
1430         "prior_version": "51'64",
1431         "last_reqid": "client.4649.0:1",
1432         "user_version": 75,
1433         "size": 1,
1434         "mtime": "",
1435         "local_mtime": "",
1436         "lost": 0,
1437         "flags": [
1438           "dirty",
1439           "omap",
1440           "data_digest",
1441           "omap_digest"
1442         ],
1443         "truncate_seq": 0,
1444         "truncate_size": 0,
1445         "data_digest": "0x2b63260d",
1446         "omap_digest": "0x2eecc539",
1447         "expected_object_size": 0,
1448         "expected_write_size": 0,
1449         "alloc_hint_flags": 0,
1450         "manifest": {
1451           "type": 0
1452         },
1453         "watchers": {}
1454       },
1455       "union_shard_errors": [
1456          "obj_size_info_mismatch"
1457       ],
1458       "errors": [
1459         "object_info_inconsistency"
1460       ],
1461       "object": {
1462         "version": 67,
1463         "snap": "head",
1464         "locator": "",
1465         "nspace": "",
1466         "name": "ROBJ9"
1467       }
1468     }
1469   ],
1470   "epoch": 0
1471 }
1472 EOF
1473
1474     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
1475     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
1476     if test $getjson = "yes"
1477     then
1478         jq '.' $dir/json > save1.json
1479     fi
1480
1481     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
1482     then
1483       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
1484     fi
1485
1486     objname=ROBJ9
1487     # Change data and size again because digest was recomputed
1488     echo -n ZZZ > $dir/change
1489     rados --pool $poolname put $objname $dir/change
1490     # Set one to an even older value
1491     objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi
1492     rm $dir/oi $dir/change
1493
1494     objname=ROBJ10
1495     objectstore_tool $dir 1 $objname get-attr _ > $dir/oi
1496     rados --pool $poolname setomapval $objname key2-$objname val2-$objname
1497     objectstore_tool $dir 0 $objname set-attr _ $dir/oi
1498     objectstore_tool $dir 1 $objname set-attr _ $dir/oi
1499     rm $dir/oi
1500
1501     inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1502     inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1503     inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1504     inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1505     pg_deep_scrub $pg
1506
1507     rados list-inconsistent-pg $poolname > $dir/json || return 1
1508     # Check pg count
1509     test $(jq '. | length' $dir/json) = "1" || return 1
1510     # Check pgid
1511     test $(jq -r '.[0]' $dir/json) = $pg || return 1
1512
1513     rados list-inconsistent-obj $pg > $dir/json || return 1
1514     # Get epoch for repair-get requests
1515     epoch=$(jq .epoch $dir/json)
1516
1517     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
1518 {
1519   "inconsistents": [
1520     {
1521       "shards": [
1522         {
1523           "data_digest": "0x2ddbf8f5",
1524           "omap_digest": "0xf5fba2c6",
1525           "size": 7,
1526           "errors": [],
1527           "osd": 0,
1528           "primary": false
1529         },
1530         {
1531           "object_info": {
1532             "oid": {
1533               "oid": "ROBJ1",
1534               "key": "",
1535               "snapid": -2,
1536               "hash": 1454963827,
1537               "max": 0,
1538               "pool": 3,
1539               "namespace": ""
1540             },
1541             "version": "51'58",
1542             "prior_version": "21'3",
1543             "last_reqid": "osd.1.0:57",
1544             "user_version": 3,
1545             "size": 7,
1546             "mtime": "2018-04-05 14:33:19.804040",
1547             "local_mtime": "2018-04-05 14:33:19.804839",
1548             "lost": 0,
1549             "flags": [
1550               "dirty",
1551               "omap",
1552               "data_digest",
1553               "omap_digest"
1554             ],
1555             "truncate_seq": 0,
1556             "truncate_size": 0,
1557             "data_digest": "0x2ddbf8f5",
1558             "omap_digest": "0xf5fba2c6",
1559             "expected_object_size": 0,
1560             "expected_write_size": 0,
1561             "alloc_hint_flags": 0,
1562             "manifest": {
1563               "type": 0
1564             },
1565             "watchers": {}
1566           },
1567           "data_digest": "0x2d4a11c2",
1568           "omap_digest": "0xf5fba2c6",
1569           "size": 9,
1570           "errors": [
1571             "data_digest_mismatch_info",
1572             "size_mismatch_info",
1573             "obj_size_info_mismatch"
1574           ],
1575           "osd": 1,
1576           "primary": true
1577         }
1578       ],
1579       "selected_object_info": {
1580         "oid": {
1581           "oid": "ROBJ1",
1582           "key": "",
1583           "snapid": -2,
1584           "hash": 1454963827,
1585           "max": 0,
1586           "pool": 3,
1587           "namespace": ""
1588         },
1589         "version": "51'58",
1590         "prior_version": "21'3",
1591         "last_reqid": "osd.1.0:57",
1592         "user_version": 3,
1593         "size": 7,
1594         "mtime": "2018-04-05 14:33:19.804040",
1595         "local_mtime": "2018-04-05 14:33:19.804839",
1596         "lost": 0,
1597         "flags": [
1598           "dirty",
1599           "omap",
1600           "data_digest",
1601           "omap_digest"
1602         ],
1603         "truncate_seq": 0,
1604         "truncate_size": 0,
1605         "data_digest": "0x2ddbf8f5",
1606         "omap_digest": "0xf5fba2c6",
1607         "expected_object_size": 0,
1608         "expected_write_size": 0,
1609         "alloc_hint_flags": 0,
1610         "manifest": {
1611           "type": 0
1612         },
1613         "watchers": {}
1614       },
1615       "union_shard_errors": [
1616         "data_digest_mismatch_info",
1617         "size_mismatch_info",
1618         "obj_size_info_mismatch"
1619       ],
1620       "errors": [
1621         "data_digest_mismatch",
1622         "size_mismatch"
1623       ],
1624       "object": {
1625         "version": 3,
1626         "snap": "head",
1627         "locator": "",
1628         "nspace": "",
1629         "name": "ROBJ1"
1630       }
1631     },
1632     {
1633       "shards": [
1634         {
1635           "data_digest": "0x2ddbf8f5",
1636           "omap_digest": "0xa8dd5adc",
1637           "size": 7,
1638           "errors": [
1639             "omap_digest_mismatch_info"
1640           ],
1641           "osd": 0,
1642           "primary": false
1643         },
1644         {
1645           "data_digest": "0x2ddbf8f5",
1646           "omap_digest": "0xa8dd5adc",
1647           "size": 7,
1648           "errors": [
1649             "omap_digest_mismatch_info"
1650           ],
1651           "osd": 1,
1652           "primary": true
1653         }
1654       ],
1655      "selected_object_info": {
1656        "alloc_hint_flags": 0,
1657        "data_digest": "0x2ddbf8f5",
1658        "expected_object_size": 0,
1659        "expected_write_size": 0,
1660        "flags": [
1661          "dirty",
1662          "omap",
1663          "data_digest",
1664          "omap_digest"
1665        ],
1666        "last_reqid": "osd.0.0:50",
1667        "local_mtime": "2018-04-05 14:33:26.762368",
1668        "lost": 0,
1669       "manifest": {
1670          "type": 0
1671        },
1672        "mtime": "2018-04-05 14:33:26.762368",
1673        "oid": {
1674          "hash": 3174666125,
1675          "key": "",
1676          "max": 0,
1677          "namespace": "",
1678          "oid": "ROBJ10",
1679          "pool": 3,
1680          "snapid": -2
1681        },
1682        "omap_digest": "0xc2025a24",
1683        "prior_version": "41'33",
1684        "size": 7,
1685        "truncate_seq": 0,
1686        "truncate_size": 0,
1687        "user_version": 30,
1688        "version": "47'51",
1689        "watchers": {}
1690       },
1691       "union_shard_errors": [
1692         "omap_digest_mismatch_info"
1693       ],
1694       "errors": [],
1695       "object": {
1696         "version": 30,
1697         "snap": "head",
1698         "locator": "",
1699         "nspace": "",
1700         "name": "ROBJ10"
1701       }
1702     },
1703     {
1704       "shards": [
1705         {
1706           "data_digest": "0x2ddbf8f5",
1707           "omap_digest": "0xa03cef03",
1708           "size": 7,
1709           "errors": [],
1710           "osd": 0,
1711           "primary": false
1712         },
1713         {
1714           "size": 7,
1715           "errors": [
1716             "read_error"
1717           ],
1718           "osd": 1,
1719           "primary": true
1720         }
1721       ],
1722       "selected_object_info": {
1723         "oid": {
1724           "oid": "ROBJ11",
1725           "key": "",
1726           "snapid": -2,
1727           "hash": 1828574689,
1728           "max": 0,
1729           "pool": 3,
1730           "namespace": ""
1731         },
1732         "version": "51'52",
1733         "prior_version": "41'33",
1734         "last_reqid": "osd.1.0:51",
1735         "user_version": 33,
1736         "size": 7,
1737         "mtime": "2018-04-05 14:33:26.761286",
1738         "local_mtime": "2018-04-05 14:33:26.762368",
1739         "lost": 0,
1740         "flags": [
1741           "dirty",
1742           "omap",
1743           "data_digest",
1744           "omap_digest"
1745         ],
1746         "truncate_seq": 0,
1747         "truncate_size": 0,
1748         "data_digest": "0x2ddbf8f5",
1749         "omap_digest": "0xa03cef03",
1750         "expected_object_size": 0,
1751         "expected_write_size": 0,
1752         "alloc_hint_flags": 0,
1753         "manifest": {
1754           "type": 0
1755         },
1756         "watchers": {}
1757       },
1758       "union_shard_errors": [
1759         "read_error"
1760       ],
1761       "errors": [],
1762       "object": {
1763         "version": 33,
1764         "snap": "head",
1765         "locator": "",
1766         "nspace": "",
1767         "name": "ROBJ11"
1768       }
1769     },
1770     {
1771       "shards": [
1772         {
1773           "errors": [
1774             "stat_error"
1775           ],
1776           "osd": 0,
1777           "primary": false
1778         },
1779         {
1780           "data_digest": "0x2ddbf8f5",
1781           "omap_digest": "0x067f306a",
1782           "size": 7,
1783           "errors": [],
1784           "osd": 1,
1785           "primary": true
1786         }
1787       ],
1788       "selected_object_info": {
1789         "oid": {
1790           "oid": "ROBJ12",
1791           "key": "",
1792           "snapid": -2,
1793           "hash": 3920199997,
1794           "max": 0,
1795           "pool": 3,
1796           "namespace": ""
1797         },
1798         "version": "51'56",
1799         "prior_version": "43'36",
1800         "last_reqid": "osd.1.0:55",
1801         "user_version": 36,
1802         "size": 7,
1803         "mtime": "2018-04-05 14:33:27.460958",
1804         "local_mtime": "2018-04-05 14:33:27.462109",
1805         "lost": 0,
1806         "flags": [
1807           "dirty",
1808           "omap",
1809           "data_digest",
1810           "omap_digest"
1811         ],
1812         "truncate_seq": 0,
1813         "truncate_size": 0,
1814         "data_digest": "0x2ddbf8f5",
1815         "omap_digest": "0x067f306a",
1816         "expected_object_size": 0,
1817         "expected_write_size": 0,
1818         "alloc_hint_flags": 0,
1819         "manifest": {
1820           "type": 0
1821         },
1822         "watchers": {}
1823       },
1824       "union_shard_errors": [
1825         "stat_error"
1826       ],
1827       "errors": [],
1828       "object": {
1829         "version": 36,
1830         "snap": "head",
1831         "locator": "",
1832         "nspace": "",
1833         "name": "ROBJ12"
1834       }
1835     },
1836     {
1837       "shards": [
1838         {
1839           "errors": [
1840             "stat_error"
1841           ],
1842           "osd": 0,
1843           "primary": false
1844         },
1845         {
1846           "size": 7,
1847           "errors": [
1848             "read_error"
1849           ],
1850           "osd": 1,
1851           "primary": true
1852         }
1853       ],
1854       "union_shard_errors": [
1855         "stat_error",
1856         "read_error"
1857       ],
1858       "errors": [],
1859       "object": {
1860         "version": 0,
1861         "snap": "head",
1862         "locator": "",
1863         "nspace": "",
1864         "name": "ROBJ13"
1865       }
1866     },
1867     {
1868       "shards": [
1869         {
1870           "object_info": "bad-val",
1871           "data_digest": "0x2ddbf8f5",
1872           "omap_digest": "0x4f14f849",
1873           "size": 7,
1874           "errors": [
1875             "info_corrupted"
1876           ],
1877           "osd": 0,
1878           "primary": false
1879         },
1880         {
1881           "data_digest": "0x2ddbf8f5",
1882           "omap_digest": "0x4f14f849",
1883           "size": 7,
1884           "errors": [
1885             "info_missing"
1886           ],
1887           "osd": 1,
1888           "primary": true
1889         }
1890       ],
1891       "union_shard_errors": [
1892         "info_missing",
1893         "info_corrupted"
1894       ],
1895       "errors": [],
1896       "object": {
1897         "version": 0,
1898         "snap": "head",
1899         "locator": "",
1900         "nspace": "",
1901         "name": "ROBJ14"
1902       }
1903     },
1904     {
1905       "shards": [
1906         {
1907           "object_info": {
1908             "oid": {
1909               "oid": "ROBJ15",
1910               "key": "",
1911               "snapid": -2,
1912               "hash": 504996876,
1913               "max": 0,
1914               "pool": 3,
1915               "namespace": ""
1916             },
1917             "version": "51'49",
1918             "prior_version": "49'45",
1919             "last_reqid": "osd.1.0:48",
1920             "user_version": 45,
1921             "size": 7,
1922             "mtime": "2018-04-05 14:33:29.498969",
1923             "local_mtime": "2018-04-05 14:33:29.499890",
1924             "lost": 0,
1925             "flags": [
1926               "dirty",
1927               "omap",
1928               "data_digest",
1929               "omap_digest"
1930             ],
1931             "truncate_seq": 0,
1932             "truncate_size": 0,
1933             "data_digest": "0x2ddbf8f5",
1934             "omap_digest": "0x2d2a4d6e",
1935             "expected_object_size": 0,
1936             "expected_write_size": 0,
1937             "alloc_hint_flags": 0,
1938             "manifest": {
1939               "type": 0
1940             },
1941             "watchers": {}
1942           },
1943           "data_digest": "0x2ddbf8f5",
1944           "omap_digest": "0x2d2a4d6e",
1945           "size": 7,
1946           "errors": [],
1947           "osd": 0,
1948           "primary": false
1949         },
1950         {
1951           "data_digest": "0x2ddbf8f5",
1952           "omap_digest": "0x2d2a4d6e",
1953           "size": 7,
1954           "errors": [
1955             "info_missing"
1956           ],
1957           "osd": 1,
1958           "primary": true
1959         }
1960       ],
1961       "selected_object_info": {
1962         "oid": {
1963           "oid": "ROBJ15",
1964           "key": "",
1965           "snapid": -2,
1966           "hash": 504996876,
1967           "max": 0,
1968           "pool": 3,
1969           "namespace": ""
1970         },
1971         "version": "51'49",
1972         "prior_version": "49'45",
1973         "last_reqid": "osd.1.0:48",
1974         "user_version": 45,
1975         "size": 7,
1976         "mtime": "2018-04-05 14:33:29.498969",
1977         "local_mtime": "2018-04-05 14:33:29.499890",
1978         "lost": 0,
1979         "flags": [
1980           "dirty",
1981           "omap",
1982           "data_digest",
1983           "omap_digest"
1984         ],
1985         "truncate_seq": 0,
1986         "truncate_size": 0,
1987         "data_digest": "0x2ddbf8f5",
1988         "omap_digest": "0x2d2a4d6e",
1989         "expected_object_size": 0,
1990         "expected_write_size": 0,
1991         "alloc_hint_flags": 0,
1992         "manifest": {
1993           "type": 0
1994         },
1995         "watchers": {}
1996       },
1997       "union_shard_errors": [
1998         "info_missing"
1999       ],
2000       "errors": [],
2001       "object": {
2002         "version": 45,
2003         "snap": "head",
2004         "locator": "",
2005         "nspace": "",
2006         "name": "ROBJ15"
2007       }
2008     },
2009     {
2010       "errors": [],
2011       "object": {
2012       "locator": "",
2013       "name": "ROBJ16",
2014       "nspace": "",
2015       "snap": "head",
2016       "version": 0
2017        },
2018         "shards": [
2019       {
2020         "data_digest": "0x2ddbf8f5",
2021         "errors": [
2022           "snapset_missing"
2023         ],
2024         "omap_digest": "0x8b699207",
2025         "osd": 0,
2026         "primary": false,
2027         "size": 7
2028       },
2029       {
2030         "snapset": "bad-val",
2031         "data_digest": "0x2ddbf8f5",
2032         "errors": [
2033           "snapset_corrupted"
2034         ],
2035         "omap_digest": "0x8b699207",
2036         "osd": 1,
2037         "primary": true,
2038         "size": 7
2039       }
2040       ],
2041       "union_shard_errors": [
2042         "snapset_missing",
2043         "snapset_corrupted"
2044       ]
2045     },
2046     {
2047      "errors": [],
2048      "object": {
2049        "locator": "",
2050        "name": "ROBJ17",
2051        "nspace": "",
2052        "snap": "head"
2053      },
2054      "selected_object_info": {
2055        "alloc_hint_flags": 0,
2056        "data_digest": "0x2ddbf8f5",
2057        "expected_object_size": 0,
2058        "expected_write_size": 0,
2059        "flags": [
2060          "dirty",
2061          "omap",
2062          "data_digest",
2063          "omap_digest"
2064        ],
2065        "lost": 0,
2066        "manifest": {
2067          "type": 0
2068        },
2069        "oid": {
2070          "hash": 1884071249,
2071          "key": "",
2072          "max": 0,
2073          "namespace": "",
2074          "oid": "ROBJ17",
2075          "pool": 3,
2076          "snapid": -2
2077        },
2078        "omap_digest": "0xe9572720",
2079        "size": 7,
2080        "truncate_seq": 0,
2081        "truncate_size": 0,
2082        "user_version": 51,
2083        "watchers": {}
2084      },
2085      "shards": [
2086        {
2087          "data_digest": "0x5af0c3ef",
2088          "errors": [
2089            "data_digest_mismatch_info"
2090          ],
2091          "omap_digest": "0xe9572720",
2092          "osd": 0,
2093          "primary": false,
2094          "size": 7
2095        },
2096        {
2097          "data_digest": "0x5af0c3ef",
2098          "errors": [
2099            "data_digest_mismatch_info"
2100          ],
2101          "omap_digest": "0xe9572720",
2102          "osd": 1,
2103          "primary": true,
2104          "size": 7
2105        }
2106      ],
2107      "union_shard_errors": [
2108        "data_digest_mismatch_info"
2109      ]
2110    },
2111    {
2112      "errors": [
2113        "object_info_inconsistency"
2114      ],
2115      "object": {
2116        "locator": "",
2117        "name": "ROBJ18",
2118        "nspace": "",
2119        "snap": "head"
2120      },
2121      "selected_object_info": {
2122        "alloc_hint_flags": 255,
2123        "data_digest": "0x2ddbf8f5",
2124        "expected_object_size": 0,
2125        "expected_write_size": 0,
2126        "flags": [
2127          "dirty",
2128          "omap",
2129          "data_digest",
2130          "omap_digest"
2131        ],
2132        "lost": 0,
2133        "manifest": {
2134          "type": 0
2135        },
2136        "oid": {
2137          "hash": 1629828556,
2138          "key": "",
2139          "max": 0,
2140          "namespace": "",
2141          "oid": "ROBJ18",
2142          "pool": 3,
2143          "snapid": -2
2144        },
2145        "omap_digest": "0xddc3680f",
2146        "size": 7,
2147        "truncate_seq": 0,
2148        "truncate_size": 0,
2149        "user_version": 54,
2150        "watchers": {}
2151      },
2152      "shards": [
2153        {
2154          "data_digest": "0xbd89c912",
2155          "errors": [
2156            "data_digest_mismatch_info"
2157          ],
2158          "object_info": {
2159            "alloc_hint_flags": 0,
2160            "data_digest": "0x2ddbf8f5",
2161            "expected_object_size": 0,
2162            "expected_write_size": 0,
2163            "flags": [
2164              "dirty",
2165              "omap",
2166              "data_digest",
2167              "omap_digest"
2168            ],
2169            "lost": 0,
2170            "manifest": {
2171              "type": 0
2172            },
2173            "oid": {
2174              "hash": 1629828556,
2175              "key": "",
2176              "max": 0,
2177              "namespace": "",
2178              "oid": "ROBJ18",
2179              "pool": 3,
2180              "snapid": -2
2181            },
2182            "omap_digest": "0xddc3680f",
2183            "size": 7,
2184            "truncate_seq": 0,
2185            "truncate_size": 0,
2186            "user_version": 54,
2187            "watchers": {}
2188          },
2189          "omap_digest": "0xddc3680f",
2190          "osd": 0,
2191          "primary": false,
2192          "size": 7
2193        },
2194        {
2195          "data_digest": "0xbd89c912",
2196          "errors": [
2197            "data_digest_mismatch_info"
2198          ],
2199          "object_info": {
2200            "alloc_hint_flags": 255,
2201            "data_digest": "0x2ddbf8f5",
2202            "expected_object_size": 0,
2203            "expected_write_size": 0,
2204            "flags": [
2205              "dirty",
2206              "omap",
2207              "data_digest",
2208              "omap_digest"
2209            ],
2210            "lost": 0,
2211            "manifest": {
2212              "type": 0
2213            },
2214            "oid": {
2215              "hash": 1629828556,
2216              "key": "",
2217              "max": 0,
2218              "namespace": "",
2219              "oid": "ROBJ18",
2220              "pool": 3,
2221              "snapid": -2
2222            },
2223            "omap_digest": "0xddc3680f",
2224            "size": 7,
2225            "truncate_seq": 0,
2226            "truncate_size": 0,
2227            "user_version": 54,
2228            "watchers": {}
2229          },
2230          "omap_digest": "0xddc3680f",
2231          "osd": 1,
2232          "primary": true,
2233          "size": 7
2234        }
2235      ],
2236      "union_shard_errors": [
2237        "data_digest_mismatch_info"
2238      ]
2239    },
2240    {
2241      "shards": [
2242         {
2243           "data_digest": "0x578a4830",
2244           "omap_digest": "0xf8e11918",
2245           "size": 7,
2246           "errors": [
2247             "data_digest_mismatch_info"
2248           ],
2249           "osd": 0,
2250           "primary": false
2251         },
2252         {
2253           "data_digest": "0x2ddbf8f5",
2254           "omap_digest": "0xf8e11918",
2255           "size": 7,
2256           "errors": [],
2257           "osd": 1,
2258           "primary": true
2259         }
2260       ],
2261       "selected_object_info": {
2262         "oid": {
2263           "oid": "ROBJ2",
2264           "key": "",
2265           "snapid": -2,
2266           "hash": 2026323607,
2267           "max": 0,
2268           "pool": 3,
2269           "namespace": ""
2270         },
2271         "version": "51'60",
2272         "prior_version": "23'6",
2273         "last_reqid": "osd.1.0:59",
2274         "user_version": 6,
2275         "size": 7,
2276         "mtime": "2018-04-05 14:33:20.498756",
2277         "local_mtime": "2018-04-05 14:33:20.499704",
2278         "lost": 0,
2279         "flags": [
2280           "dirty",
2281           "omap",
2282           "data_digest",
2283           "omap_digest"
2284         ],
2285         "truncate_seq": 0,
2286         "truncate_size": 0,
2287         "data_digest": "0x2ddbf8f5",
2288         "omap_digest": "0xf8e11918",
2289         "expected_object_size": 0,
2290         "expected_write_size": 0,
2291         "alloc_hint_flags": 0,
2292         "manifest": {
2293           "type": 0
2294         },
2295         "watchers": {}
2296       },
2297       "union_shard_errors": [
2298         "data_digest_mismatch_info"
2299       ],
2300       "errors": [
2301         "data_digest_mismatch"
2302       ],
2303       "object": {
2304         "version": 6,
2305         "snap": "head",
2306         "locator": "",
2307         "nspace": "",
2308         "name": "ROBJ2"
2309       }
2310     },
2311     {
2312       "shards": [
2313         {
2314           "data_digest": "0x2ddbf8f5",
2315           "omap_digest": "0x00b35dfd",
2316           "size": 7,
2317           "errors": [],
2318           "osd": 0,
2319           "primary": false
2320         },
2321         {
2322           "errors": [
2323             "missing"
2324           ],
2325           "osd": 1,
2326           "primary": true
2327         }
2328       ],
2329       "selected_object_info": {
2330         "oid": {
2331           "oid": "ROBJ3",
2332           "key": "",
2333           "snapid": -2,
2334           "hash": 625845583,
2335           "max": 0,
2336           "pool": 3,
2337           "namespace": ""
2338         },
2339         "version": "51'61",
2340         "prior_version": "25'9",
2341         "last_reqid": "osd.1.0:60",
2342         "user_version": 9,
2343         "size": 7,
2344         "mtime": "2018-04-05 14:33:21.189382",
2345         "local_mtime": "2018-04-05 14:33:21.190446",
2346         "lost": 0,
2347         "flags": [
2348           "dirty",
2349           "omap",
2350           "data_digest",
2351           "omap_digest"
2352         ],
2353         "truncate_seq": 0,
2354         "truncate_size": 0,
2355         "data_digest": "0x2ddbf8f5",
2356         "omap_digest": "0x00b35dfd",
2357         "expected_object_size": 0,
2358         "expected_write_size": 0,
2359         "alloc_hint_flags": 0,
2360         "manifest": {
2361           "type": 0
2362         },
2363         "watchers": {}
2364       },
2365       "union_shard_errors": [
2366         "missing"
2367       ],
2368       "errors": [],
2369       "object": {
2370         "version": 9,
2371         "snap": "head",
2372         "locator": "",
2373         "nspace": "",
2374         "name": "ROBJ3"
2375       }
2376     },
2377     {
2378       "shards": [
2379         {
2380           "data_digest": "0x2ddbf8f5",
2381           "omap_digest": "0xd7178dfe",
2382           "size": 7,
2383           "errors": [
2384             "omap_digest_mismatch_info"
2385           ],
2386           "osd": 0,
2387           "primary": false
2388         },
2389         {
2390           "data_digest": "0x2ddbf8f5",
2391           "omap_digest": "0xe2d46ea4",
2392           "size": 7,
2393           "errors": [],
2394           "osd": 1,
2395           "primary": true
2396         }
2397       ],
2398       "selected_object_info": {
2399         "oid": {
2400           "oid": "ROBJ4",
2401           "key": "",
2402           "snapid": -2,
2403           "hash": 2360875311,
2404           "max": 0,
2405           "pool": 3,
2406           "namespace": ""
2407         },
2408         "version": "51'62",
2409         "prior_version": "27'12",
2410         "last_reqid": "osd.1.0:61",
2411         "user_version": 12,
2412         "size": 7,
2413         "mtime": "2018-04-05 14:33:21.862313",
2414         "local_mtime": "2018-04-05 14:33:21.863261",
2415         "lost": 0,
2416         "flags": [
2417           "dirty",
2418           "omap",
2419           "data_digest",
2420           "omap_digest"
2421         ],
2422         "truncate_seq": 0,
2423         "truncate_size": 0,
2424         "data_digest": "0x2ddbf8f5",
2425         "omap_digest": "0xe2d46ea4",
2426         "expected_object_size": 0,
2427         "expected_write_size": 0,
2428         "alloc_hint_flags": 0,
2429         "manifest": {
2430           "type": 0
2431         },
2432         "watchers": {}
2433       },
2434       "union_shard_errors": [
2435         "omap_digest_mismatch_info"
2436       ],
2437       "errors": [
2438         "omap_digest_mismatch"
2439       ],
2440       "object": {
2441         "version": 12,
2442         "snap": "head",
2443         "locator": "",
2444         "nspace": "",
2445         "name": "ROBJ4"
2446       }
2447     },
2448     {
2449       "shards": [
2450         {
2451           "data_digest": "0x2ddbf8f5",
2452           "omap_digest": "0x1a862a41",
2453           "size": 7,
2454           "errors": [],
2455           "osd": 0,
2456           "primary": false
2457         },
2458         {
2459           "data_digest": "0x2ddbf8f5",
2460           "omap_digest": "0x06cac8f6",
2461           "size": 7,
2462           "errors": [
2463             "omap_digest_mismatch_info"
2464           ],
2465           "osd": 1,
2466           "primary": true
2467         }
2468       ],
2469       "selected_object_info": {
2470         "oid": {
2471           "oid": "ROBJ5",
2472           "key": "",
2473           "snapid": -2,
2474           "hash": 2334915887,
2475           "max": 0,
2476           "pool": 3,
2477           "namespace": ""
2478         },
2479         "version": "51'63",
2480         "prior_version": "29'15",
2481         "last_reqid": "osd.1.0:62",
2482         "user_version": 15,
2483         "size": 7,
2484         "mtime": "2018-04-05 14:33:22.589300",
2485         "local_mtime": "2018-04-05 14:33:22.590376",
2486         "lost": 0,
2487         "flags": [
2488           "dirty",
2489           "omap",
2490           "data_digest",
2491           "omap_digest"
2492         ],
2493         "truncate_seq": 0,
2494         "truncate_size": 0,
2495         "data_digest": "0x2ddbf8f5",
2496         "omap_digest": "0x1a862a41",
2497         "expected_object_size": 0,
2498         "expected_write_size": 0,
2499         "alloc_hint_flags": 0,
2500         "manifest": {
2501           "type": 0
2502         },
2503         "watchers": {}
2504       },
2505       "union_shard_errors": [
2506         "omap_digest_mismatch_info"
2507       ],
2508       "errors": [
2509         "omap_digest_mismatch"
2510       ],
2511       "object": {
2512         "version": 15,
2513         "snap": "head",
2514         "locator": "",
2515         "nspace": "",
2516         "name": "ROBJ5"
2517       }
2518     },
2519     {
2520       "shards": [
2521         {
2522           "data_digest": "0x2ddbf8f5",
2523           "omap_digest": "0x689ee887",
2524           "size": 7,
2525           "errors": [
2526             "omap_digest_mismatch_info"
2527           ],
2528           "osd": 0,
2529           "primary": false
2530         },
2531         {
2532           "data_digest": "0x2ddbf8f5",
2533           "omap_digest": "0x179c919f",
2534           "size": 7,
2535           "errors": [],
2536           "osd": 1,
2537           "primary": true
2538         }
2539       ],
2540       "selected_object_info": {
2541         "oid": {
2542           "oid": "ROBJ6",
2543           "key": "",
2544           "snapid": -2,
2545           "hash": 390610085,
2546           "max": 0,
2547           "pool": 3,
2548           "namespace": ""
2549         },
2550         "version": "51'54",
2551         "prior_version": "31'18",
2552         "last_reqid": "osd.1.0:53",
2553         "user_version": 18,
2554         "size": 7,
2555         "mtime": "2018-04-05 14:33:23.289188",
2556         "local_mtime": "2018-04-05 14:33:23.290130",
2557         "lost": 0,
2558         "flags": [
2559           "dirty",
2560           "omap",
2561           "data_digest",
2562           "omap_digest"
2563         ],
2564         "truncate_seq": 0,
2565         "truncate_size": 0,
2566         "data_digest": "0x2ddbf8f5",
2567         "omap_digest": "0x179c919f",
2568         "expected_object_size": 0,
2569         "expected_write_size": 0,
2570         "alloc_hint_flags": 0,
2571         "manifest": {
2572           "type": 0
2573         },
2574         "watchers": {}
2575       },
2576       "union_shard_errors": [
2577         "omap_digest_mismatch_info"
2578       ],
2579       "errors": [
2580         "omap_digest_mismatch"
2581       ],
2582       "object": {
2583         "version": 18,
2584         "snap": "head",
2585         "locator": "",
2586         "nspace": "",
2587         "name": "ROBJ6"
2588       }
2589     },
2590     {
2591       "shards": [
2592         {
2593           "data_digest": "0x2ddbf8f5",
2594           "omap_digest": "0xefced57a",
2595           "size": 7,
2596           "errors": [],
2597           "osd": 0,
2598           "primary": false
2599         },
2600         {
2601           "data_digest": "0x2ddbf8f5",
2602           "omap_digest": "0x6a73cc07",
2603           "size": 7,
2604           "errors": [
2605             "omap_digest_mismatch_info"
2606           ],
2607           "osd": 1,
2608           "primary": true
2609         }
2610       ],
2611       "selected_object_info": {
2612         "oid": {
2613           "oid": "ROBJ7",
2614           "key": "",
2615           "snapid": -2,
2616           "hash": 3529485009,
2617           "max": 0,
2618           "pool": 3,
2619           "namespace": ""
2620         },
2621         "version": "51'53",
2622         "prior_version": "33'21",
2623         "last_reqid": "osd.1.0:52",
2624         "user_version": 21,
2625         "size": 7,
2626         "mtime": "2018-04-05 14:33:23.979658",
2627         "local_mtime": "2018-04-05 14:33:23.980731",
2628         "lost": 0,
2629         "flags": [
2630           "dirty",
2631           "omap",
2632           "data_digest",
2633           "omap_digest"
2634         ],
2635         "truncate_seq": 0,
2636         "truncate_size": 0,
2637         "data_digest": "0x2ddbf8f5",
2638         "omap_digest": "0xefced57a",
2639         "expected_object_size": 0,
2640         "expected_write_size": 0,
2641         "alloc_hint_flags": 0,
2642         "manifest": {
2643           "type": 0
2644         },
2645         "watchers": {}
2646       },
2647       "union_shard_errors": [
2648         "omap_digest_mismatch_info"
2649       ],
2650       "errors": [
2651         "omap_digest_mismatch"
2652       ],
2653       "object": {
2654         "version": 21,
2655         "snap": "head",
2656         "locator": "",
2657         "nspace": "",
2658         "name": "ROBJ7"
2659       }
2660     },
2661     {
2662       "shards": [
2663         {
2664           "attrs": [
2665             {
2666               "Base64": false,
2667               "value": "bad-val",
2668               "name": "key1-ROBJ8"
2669             },
2670             {
2671               "Base64": false,
2672               "value": "val2-ROBJ8",
2673               "name": "key2-ROBJ8"
2674             }
2675           ],
2676           "data_digest": "0x2ddbf8f5",
2677           "omap_digest": "0xd6be81dc",
2678           "size": 7,
2679           "errors": [],
2680           "osd": 0,
2681           "primary": false
2682         },
2683         {
2684           "attrs": [
2685             {
2686               "Base64": false,
2687               "value": "val1-ROBJ8",
2688               "name": "key1-ROBJ8"
2689             },
2690             {
2691               "Base64": false,
2692               "value": "val3-ROBJ8",
2693               "name": "key3-ROBJ8"
2694             }
2695           ],
2696           "data_digest": "0x2ddbf8f5",
2697           "omap_digest": "0xd6be81dc",
2698           "size": 7,
2699           "errors": [],
2700           "osd": 1,
2701           "primary": true
2702         }
2703       ],
2704       "selected_object_info": {
2705         "oid": {
2706           "oid": "ROBJ8",
2707           "key": "",
2708           "snapid": -2,
2709           "hash": 2359695969,
2710           "max": 0,
2711           "pool": 3,
2712           "namespace": ""
2713         },
2714         "version": "79'66",
2715         "prior_version": "79'65",
2716         "last_reqid": "client.4554.0:1",
2717         "user_version": 74,
2718         "size": 7,
2719         "mtime": "2018-04-05 14:34:05.598688",
2720         "local_mtime": "2018-04-05 14:34:05.599698",
2721         "lost": 0,
2722         "flags": [
2723           "dirty",
2724           "omap",
2725           "data_digest",
2726           "omap_digest"
2727         ],
2728         "truncate_seq": 0,
2729         "truncate_size": 0,
2730         "data_digest": "0x2ddbf8f5",
2731         "omap_digest": "0xd6be81dc",
2732         "expected_object_size": 0,
2733         "expected_write_size": 0,
2734         "alloc_hint_flags": 0,
2735         "manifest": {
2736           "type": 0
2737         },
2738         "watchers": {}
2739       },
2740       "union_shard_errors": [],
2741       "errors": [
2742         "attr_value_mismatch",
2743         "attr_name_mismatch"
2744       ],
2745       "object": {
2746         "version": 66,
2747         "snap": "head",
2748         "locator": "",
2749         "nspace": "",
2750         "name": "ROBJ8"
2751       }
2752     },
2753     {
2754       "shards": [
2755         {
2756           "object_info": {
2757             "oid": {
2758               "oid": "ROBJ9",
2759               "key": "",
2760               "snapid": -2,
2761               "hash": 537189375,
2762               "max": 0,
2763               "pool": 3,
2764               "namespace": ""
2765             },
2766             "version": "51'64",
2767             "prior_version": "37'27",
2768             "last_reqid": "osd.1.0:63",
2769             "user_version": 27,
2770             "size": 7,
2771             "mtime": "2018-04-05 14:33:25.352485",
2772             "local_mtime": "2018-04-05 14:33:25.353746",
2773             "lost": 0,
2774             "flags": [
2775               "dirty",
2776               "omap",
2777               "data_digest",
2778               "omap_digest"
2779             ],
2780             "truncate_seq": 0,
2781             "truncate_size": 0,
2782             "data_digest": "0x2ddbf8f5",
2783            "omap_digest": "0x2eecc539",
2784             "expected_object_size": 0,
2785             "expected_write_size": 0,
2786             "alloc_hint_flags": 0,
2787             "manifest": {
2788               "type": 0
2789             },
2790             "watchers": {}
2791           },
2792           "data_digest": "0x1f26fb26",
2793           "omap_digest": "0x2eecc539",
2794           "size": 3,
2795           "errors": [
2796             "obj_size_info_mismatch"
2797           ],
2798           "osd": 0,
2799           "primary": false
2800         },
2801         {
2802           "object_info": {
2803             "oid": {
2804               "oid": "ROBJ9",
2805               "key": "",
2806               "snapid": -2,
2807               "hash": 537189375,
2808               "max": 0,
2809               "pool": 3,
2810               "namespace": ""
2811             },
2812             "version": "119'68",
2813             "prior_version": "51'64",
2814             "last_reqid": "client.4834.0:1",
2815             "user_version": 76,
2816             "size": 3,
2817             "mtime": "2018-04-05 14:35:01.500659",
2818             "local_mtime": "2018-04-05 14:35:01.502117",
2819             "lost": 0,
2820             "flags": [
2821               "dirty",
2822               "omap",
2823               "data_digest",
2824               "omap_digest"
2825             ],
2826             "truncate_seq": 0,
2827             "truncate_size": 0,
2828             "data_digest": "0x1f26fb26",
2829             "omap_digest": "0x2eecc539",
2830             "expected_object_size": 0,
2831             "expected_write_size": 0,
2832             "alloc_hint_flags": 0,
2833             "manifest": {
2834               "type": 0
2835             },
2836             "watchers": {}
2837           },
2838           "data_digest": "0x1f26fb26",
2839           "omap_digest": "0x2eecc539",
2840           "size": 3,
2841           "errors": [],
2842           "osd": 1,
2843           "primary": true
2844         }
2845       ],
2846       "selected_object_info": {
2847         "oid": {
2848           "oid": "ROBJ9",
2849           "key": "",
2850           "snapid": -2,
2851           "hash": 537189375,
2852           "max": 0,
2853           "pool": 3,
2854           "namespace": ""
2855         },
2856         "version": "119'68",
2857         "prior_version": "51'64",
2858         "last_reqid": "client.4834.0:1",
2859         "user_version": 76,
2860         "size": 3,
2861         "mtime": "2018-04-05 14:35:01.500659",
2862         "local_mtime": "2018-04-05 14:35:01.502117",
2863         "lost": 0,
2864         "flags": [
2865           "dirty",
2866           "omap",
2867           "data_digest",
2868           "omap_digest"
2869         ],
2870         "truncate_seq": 0,
2871         "truncate_size": 0,
2872         "data_digest": "0x1f26fb26",
2873         "omap_digest": "0x2eecc539",
2874         "expected_object_size": 0,
2875         "expected_write_size": 0,
2876         "alloc_hint_flags": 0,
2877         "manifest": {
2878           "type": 0
2879         },
2880         "watchers": {}
2881       },
2882       "union_shard_errors": [
2883         "obj_size_info_mismatch"
2884       ],
2885       "errors": [
2886         "object_info_inconsistency"
2887       ],
2888       "object": {
2889         "version": 68,
2890         "snap": "head",
2891         "locator": "",
2892         "nspace": "",
2893         "name": "ROBJ9"
2894       }
2895     }
2896   ],
2897   "epoch": 0
2898 }
2899 EOF
2900
2901     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
2902     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
2903     if test $getjson = "yes"
2904     then
2905         jq '.' $dir/json > save2.json
2906     fi
2907
2908     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
2909     then
2910       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
2911     fi
2912
2913     repair $pg
2914     wait_for_clean
2915
2916     # This hangs if the repair doesn't work
2917     timeout 30 rados -p $poolname get ROBJ17 $dir/robj17.out || return 1
2918     timeout 30 rados -p $poolname get ROBJ18 $dir/robj18.out || return 1
2919     # Even though we couldn't repair all of the introduced errors, we can fix ROBJ17
2920     diff -q $dir/new.ROBJ17 $dir/robj17.out || return 1
2921     rm -f $dir/new.ROBJ17 $dir/robj17.out || return 1
2922     diff -q $dir/new.ROBJ18 $dir/robj18.out || return 1
2923     rm -f $dir/new.ROBJ18 $dir/robj18.out || return 1
2924
2925     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
2926     teardown $dir || return 1
2927 }
2928
2929
2930 #
2931 # Test scrub errors for an erasure coded pool
2932 #
2933 function corrupt_scrub_erasure() {
2934     local dir=$1
2935     local allow_overwrites=$2
2936     local poolname=ecpool
2937     local total_objs=7
2938
2939     setup $dir || return 1
2940     run_mon $dir a || return 1
2941     run_mgr $dir x || return 1
2942     for id in $(seq 0 2) ; do
2943         if [ "$allow_overwrites" = "true" ]; then
2944             run_osd_bluestore $dir $id || return 1
2945         else
2946             run_osd $dir $id || return 1
2947         fi
2948     done
2949     create_rbd_pool || return 1
2950     create_pool foo 1
2951
2952     create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1
2953     wait_for_clean || return 1
2954
2955     for i in $(seq 1 $total_objs) ; do
2956         objname=EOBJ${i}
2957         add_something $dir $poolname $objname || return 1
2958
2959         local osd=$(expr $i % 2)
2960
2961         case $i in
2962         1)
2963             # Size (deep scrub data_digest too)
2964             local payload=UVWXYZZZ
2965             echo $payload > $dir/CORRUPT
2966             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2967             ;;
2968
2969         2)
2970             # Corrupt EC shard
2971             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1
2972             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2973             ;;
2974
2975         3)
2976              # missing
2977              objectstore_tool $dir $osd $objname remove || return 1
2978              ;;
2979
2980         4)
2981             rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
2982             rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
2983
2984             # Break xattrs
2985             echo -n bad-val > $dir/bad-val
2986             objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
2987             objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
2988             echo -n val3-$objname > $dir/newval
2989             objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
2990             rm $dir/bad-val $dir/newval
2991             ;;
2992
2993         5)
2994             # Corrupt EC shard
2995             dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2
2996             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2997             ;;
2998
2999         6)
3000             objectstore_tool $dir 0 $objname rm-attr hinfo_key || return 1
3001             echo -n bad-val > $dir/bad-val
3002             objectstore_tool $dir 1 $objname set-attr hinfo_key $dir/bad-val || return 1
3003             ;;
3004
3005         7)
3006             local payload=MAKETHISDIFFERENTFROMOTHEROBJECTS
3007             echo $payload > $dir/DIFFERENT
3008             rados --pool $poolname put $objname $dir/DIFFERENT || return 1
3009
3010             # Get hinfo_key from EOBJ1
3011             objectstore_tool $dir 0 EOBJ1 get-attr hinfo_key > $dir/hinfo
3012             objectstore_tool $dir 0 $objname set-attr hinfo_key $dir/hinfo || return 1
3013             rm -f $dir/hinfo
3014             ;;
3015
3016         esac
3017     done
3018
3019     local pg=$(get_pg $poolname EOBJ0)
3020
3021     pg_scrub $pg
3022
3023     rados list-inconsistent-pg $poolname > $dir/json || return 1
3024     # Check pg count
3025     test $(jq '. | length' $dir/json) = "1" || return 1
3026     # Check pgid
3027     test $(jq -r '.[0]' $dir/json) = $pg || return 1
3028
3029     rados list-inconsistent-obj $pg > $dir/json || return 1
3030     # Get epoch for repair-get requests
3031     epoch=$(jq .epoch $dir/json)
3032
3033     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
3034 {
3035   "inconsistents": [
3036     {
3037       "shards": [
3038         {
3039           "size": 2048,
3040           "errors": [],
3041           "shard": 2,
3042           "osd": 0,
3043           "primary": false
3044         },
3045         {
3046           "object_info": {
3047             "oid": {
3048               "oid": "EOBJ1",
3049               "key": "",
3050               "snapid": -2,
3051               "hash": 560836233,
3052               "max": 0,
3053               "pool": 3,
3054               "namespace": ""
3055             },
3056             "version": "27'1",
3057             "prior_version": "0'0",
3058             "last_reqid": "client.4184.0:1",
3059             "user_version": 1,
3060             "size": 7,
3061             "mtime": "",
3062             "local_mtime": "",
3063             "lost": 0,
3064             "flags": [
3065               "dirty",
3066               "data_digest"
3067             ],
3068             "truncate_seq": 0,
3069             "truncate_size": 0,
3070             "data_digest": "0x2ddbf8f5",
3071             "omap_digest": "0xffffffff",
3072             "expected_object_size": 0,
3073             "expected_write_size": 0,
3074             "alloc_hint_flags": 0,
3075             "manifest": {
3076               "type": 0
3077             },
3078             "watchers": {}
3079           },
3080           "size": 9,
3081           "shard": 0,
3082           "errors": [
3083             "size_mismatch_info",
3084             "obj_size_info_mismatch"
3085           ],
3086           "osd": 1,
3087           "primary": true
3088         },
3089         {
3090           "size": 2048,
3091           "shard": 1,
3092           "errors": [],
3093           "osd": 2,
3094           "primary": false
3095         }
3096       ],
3097       "selected_object_info": {
3098         "oid": {
3099           "oid": "EOBJ1",
3100           "key": "",
3101           "snapid": -2,
3102           "hash": 560836233,
3103           "max": 0,
3104           "pool": 3,
3105           "namespace": ""
3106         },
3107         "version": "27'1",
3108         "prior_version": "0'0",
3109         "last_reqid": "client.4184.0:1",
3110         "user_version": 1,
3111         "size": 7,
3112         "mtime": "",
3113         "local_mtime": "",
3114         "lost": 0,
3115         "flags": [
3116           "dirty",
3117           "data_digest"
3118         ],
3119         "truncate_seq": 0,
3120         "truncate_size": 0,
3121         "data_digest": "0x2ddbf8f5",
3122         "omap_digest": "0xffffffff",
3123         "expected_object_size": 0,
3124         "expected_write_size": 0,
3125         "alloc_hint_flags": 0,
3126         "manifest": {
3127           "type": 0
3128         },
3129         "watchers": {}
3130       },
3131       "union_shard_errors": [
3132         "size_mismatch_info",
3133         "obj_size_info_mismatch"
3134       ],
3135       "errors": [
3136         "size_mismatch"
3137       ],
3138       "object": {
3139         "version": 1,
3140         "snap": "head",
3141         "locator": "",
3142         "nspace": "",
3143         "name": "EOBJ1"
3144       }
3145     },
3146     {
3147       "shards": [
3148         {
3149           "size": 2048,
3150           "errors": [],
3151           "shard": 2,
3152           "osd": 0,
3153           "primary": false
3154         },
3155         {
3156           "shard": 0,
3157           "errors": [
3158             "missing"
3159           ],
3160           "osd": 1,
3161           "primary": true
3162         },
3163         {
3164           "size": 2048,
3165           "shard": 1,
3166           "errors": [],
3167           "osd": 2,
3168           "primary": false
3169         }
3170       ],
3171       "selected_object_info": {
3172         "oid": {
3173           "oid": "EOBJ3",
3174           "key": "",
3175           "snapid": -2,
3176           "hash": 3125668237,
3177           "max": 0,
3178           "pool": 3,
3179           "namespace": ""
3180         },
3181         "version": "39'3",
3182         "prior_version": "0'0",
3183         "last_reqid": "client.4252.0:1",
3184         "user_version": 3,
3185         "size": 7,
3186         "mtime": "",
3187         "local_mtime": "",
3188         "lost": 0,
3189         "flags": [
3190           "dirty",
3191           "data_digest"
3192         ],
3193         "truncate_seq": 0,
3194         "truncate_size": 0,
3195         "data_digest": "0x2ddbf8f5",
3196         "omap_digest": "0xffffffff",
3197         "expected_object_size": 0,
3198         "expected_write_size": 0,
3199         "alloc_hint_flags": 0,
3200         "manifest": {
3201           "type": 0
3202         },
3203         "watchers": {}
3204       },
3205       "union_shard_errors": [
3206         "missing"
3207       ],
3208       "errors": [],
3209       "object": {
3210         "version": 3,
3211         "snap": "head",
3212         "locator": "",
3213         "nspace": "",
3214         "name": "EOBJ3"
3215       }
3216     },
3217     {
3218       "shards": [
3219         {
3220           "attrs": [
3221             {
3222               "Base64": false,
3223               "value": "bad-val",
3224               "name": "key1-EOBJ4"
3225             },
3226             {
3227               "Base64": false,
3228               "value": "val2-EOBJ4",
3229               "name": "key2-EOBJ4"
3230             }
3231           ],
3232           "size": 2048,
3233           "errors": [],
3234           "shard": 2,
3235           "osd": 0,
3236           "primary": false
3237         },
3238         {
3239           "osd": 1,
3240           "primary": true,
3241           "shard": 0,
3242           "errors": [],
3243           "size": 2048,
3244           "attrs": [
3245             {
3246               "Base64": false,
3247               "value": "val1-EOBJ4",
3248               "name": "key1-EOBJ4"
3249             },
3250             {
3251               "Base64": false,
3252               "value": "val2-EOBJ4",
3253               "name": "key2-EOBJ4"
3254             }
3255           ]
3256         },
3257         {
3258           "osd": 2,
3259           "primary": false,
3260           "shard": 1,
3261           "errors": [],
3262           "size": 2048,
3263           "attrs": [
3264             {
3265               "Base64": false,
3266               "value": "val1-EOBJ4",
3267               "name": "key1-EOBJ4"
3268             },
3269             {
3270               "Base64": false,
3271               "value": "val3-EOBJ4",
3272               "name": "key3-EOBJ4"
3273             }
3274           ]
3275         }
3276       ],
3277       "selected_object_info": {
3278         "oid": {
3279           "oid": "EOBJ4",
3280           "key": "",
3281           "snapid": -2,
3282           "hash": 1618759290,
3283           "max": 0,
3284           "pool": 3,
3285           "namespace": ""
3286         },
3287         "version": "45'6",
3288         "prior_version": "45'5",
3289         "last_reqid": "client.4294.0:1",
3290         "user_version": 6,
3291         "size": 7,
3292         "mtime": "",
3293         "local_mtime": "",
3294         "lost": 0,
3295         "flags": [
3296           "dirty",
3297           "data_digest"
3298         ],
3299         "truncate_seq": 0,
3300         "truncate_size": 0,
3301         "data_digest": "0x2ddbf8f5",
3302         "omap_digest": "0xffffffff",
3303         "expected_object_size": 0,
3304         "expected_write_size": 0,
3305         "alloc_hint_flags": 0,
3306         "manifest": {
3307           "type": 0
3308         },
3309         "watchers": {}
3310       },
3311       "union_shard_errors": [],
3312       "errors": [
3313         "attr_value_mismatch",
3314         "attr_name_mismatch"
3315       ],
3316       "object": {
3317         "version": 6,
3318         "snap": "head",
3319         "locator": "",
3320         "nspace": "",
3321         "name": "EOBJ4"
3322       }
3323     },
3324     {
3325       "shards": [
3326         {
3327           "size": 2048,
3328           "errors": [],
3329           "shard": 2,
3330           "osd": 0,
3331           "primary": false
3332         },
3333         {
3334           "object_info": {
3335             "oid": {
3336               "oid": "EOBJ5",
3337               "key": "",
3338               "snapid": -2,
3339               "hash": 2918945441,
3340               "max": 0,
3341               "pool": 3,
3342               "namespace": ""
3343             },
3344             "version": "59'7",
3345             "prior_version": "0'0",
3346             "last_reqid": "client.4382.0:1",
3347             "user_version": 7,
3348             "size": 7,
3349             "mtime": "",
3350             "local_mtime": "",
3351             "lost": 0,
3352             "flags": [
3353               "dirty",
3354               "data_digest"
3355             ],
3356             "truncate_seq": 0,
3357             "truncate_size": 0,
3358             "data_digest": "0x2ddbf8f5",
3359             "omap_digest": "0xffffffff",
3360             "expected_object_size": 0,
3361             "expected_write_size": 0,
3362             "alloc_hint_flags": 0,
3363             "manifest": {
3364               "type": 0
3365             },
3366             "watchers": {}
3367           },
3368           "size": 4096,
3369           "shard": 0,
3370           "errors": [
3371             "size_mismatch_info",
3372             "obj_size_info_mismatch"
3373           ],
3374           "osd": 1,
3375           "primary": true
3376         },
3377         {
3378           "size": 2048,
3379           "shard": 1,
3380           "errors": [],
3381           "osd": 2,
3382           "primary": false
3383         }
3384       ],
3385       "selected_object_info": {
3386         "oid": {
3387           "oid": "EOBJ5",
3388           "key": "",
3389           "snapid": -2,
3390           "hash": 2918945441,
3391           "max": 0,
3392           "pool": 3,
3393           "namespace": ""
3394         },
3395         "version": "59'7",
3396         "prior_version": "0'0",
3397         "last_reqid": "client.4382.0:1",
3398         "user_version": 7,
3399         "size": 7,
3400         "mtime": "",
3401         "local_mtime": "",
3402         "lost": 0,
3403         "flags": [
3404           "dirty",
3405           "data_digest"
3406         ],
3407         "truncate_seq": 0,
3408         "truncate_size": 0,
3409         "data_digest": "0x2ddbf8f5",
3410         "omap_digest": "0xffffffff",
3411         "expected_object_size": 0,
3412         "expected_write_size": 0,
3413         "alloc_hint_flags": 0,
3414         "manifest": {
3415           "type": 0
3416         },
3417         "watchers": {}
3418       },
3419       "union_shard_errors": [
3420         "size_mismatch_info",
3421         "obj_size_info_mismatch"
3422       ],
3423       "errors": [
3424         "size_mismatch"
3425       ],
3426       "object": {
3427         "version": 7,
3428         "snap": "head",
3429         "locator": "",
3430         "nspace": "",
3431         "name": "EOBJ5"
3432       }
3433    },
3434    {
3435      "errors": [],
3436      "object": {
3437        "locator": "",
3438        "name": "EOBJ6",
3439        "nspace": "",
3440        "snap": "head",
3441        "version": 8
3442      },
3443      "selected_object_info": {
3444         "oid": {
3445           "oid": "EOBJ6",
3446           "key": "",
3447           "snapid": -2,
3448           "hash": 3050890866,
3449           "max": 0,
3450           "pool": 3,
3451           "namespace": ""
3452         },
3453         "version": "65'8",
3454         "prior_version": "0'0",
3455         "last_reqid": "client.4418.0:1",
3456         "user_version": 8,
3457         "size": 7,
3458         "mtime": "",
3459         "local_mtime": "",
3460         "lost": 0,
3461         "flags": [
3462           "dirty",
3463           "data_digest"
3464         ],
3465         "truncate_seq": 0,
3466         "truncate_size": 0,
3467         "data_digest": "0x2ddbf8f5",
3468         "omap_digest": "0xffffffff",
3469         "expected_object_size": 0,
3470         "expected_write_size": 0,
3471         "alloc_hint_flags": 0,
3472         "manifest": {
3473           "type": 0
3474         },
3475         "watchers": {}
3476      },
3477      "shards": [
3478        {
3479          "errors": [
3480            "hinfo_missing"
3481          ],
3482          "osd": 0,
3483          "primary": false,
3484          "shard": 2,
3485          "size": 2048
3486        },
3487        {
3488          "errors": [
3489            "hinfo_corrupted"
3490          ],
3491          "osd": 1,
3492          "primary": true,
3493          "shard": 0,
3494          "hashinfo": "bad-val",
3495          "size": 2048
3496        },
3497        {
3498          "errors": [],
3499          "osd": 2,
3500          "primary": false,
3501          "shard": 1,
3502          "size": 2048,
3503          "hashinfo": {
3504            "cumulative_shard_hashes": [
3505             {
3506               "hash": 80717615,
3507               "shard": 0
3508             },
3509             {
3510               "hash": 1534491824,
3511               "shard": 1
3512             },
3513             {
3514               "hash": 80717615,
3515               "shard": 2
3516             }
3517            ],
3518            "total_chunk_size": 2048
3519          }
3520        }
3521      ],
3522      "union_shard_errors": [
3523        "hinfo_missing",
3524        "hinfo_corrupted"
3525      ]
3526    },
3527    {
3528      "errors": [
3529        "hinfo_inconsistency"
3530      ],
3531      "object": {
3532        "locator": "",
3533        "name": "EOBJ7",
3534        "nspace": "",
3535        "snap": "head",
3536        "version": 10
3537      },
3538      "selected_object_info": {
3539         "oid": {
3540           "oid": "EOBJ7",
3541           "key": "",
3542           "snapid": -2,
3543           "hash": 3258066308,
3544           "max": 0,
3545           "pool": 3,
3546           "namespace": ""
3547         },
3548         "version": "75'10",
3549         "prior_version": "75'9",
3550         "last_reqid": "client.4482.0:1",
3551         "user_version": 10,
3552         "size": 34,
3553         "mtime": "",
3554         "local_mtime": "",
3555         "lost": 0,
3556         "flags": [
3557           "dirty",
3558           "data_digest"
3559         ],
3560         "truncate_seq": 0,
3561         "truncate_size": 0,
3562         "data_digest": "0x136e4e27",
3563         "omap_digest": "0xffffffff",
3564         "expected_object_size": 0,
3565         "expected_write_size": 0,
3566         "alloc_hint_flags": 0,
3567         "manifest": {
3568           "type": 0
3569         },
3570         "watchers": {}
3571      },
3572      "shards": [
3573        {
3574          "hashinfo": {
3575            "cumulative_shard_hashes": [
3576             {
3577               "hash": 80717615,
3578               "shard": 0
3579             },
3580             {
3581               "hash": 1534491824,
3582               "shard": 1
3583             },
3584             {
3585               "hash": 80717615,
3586               "shard": 2
3587             }
3588            ],
3589            "total_chunk_size": 2048
3590          },
3591          "errors": [],
3592          "osd": 0,
3593          "primary": false,
3594          "shard": 2,
3595          "size": 2048
3596        },
3597        {
3598          "hashinfo": {
3599            "cumulative_shard_hashes": [
3600             {
3601               "hash": 1534350760,
3602               "shard": 0
3603             },
3604             {
3605               "hash": 1534491824,
3606               "shard": 1
3607             },
3608             {
3609               "hash": 1534350760,
3610               "shard": 2
3611             }
3612            ],
3613            "total_chunk_size": 2048
3614          },
3615          "errors": [],
3616          "osd": 1,
3617          "primary": true,
3618          "shard": 0,
3619          "size": 2048
3620        },
3621        {
3622          "hashinfo": {
3623            "cumulative_shard_hashes": [
3624             {
3625               "hash": 1534350760,
3626               "shard": 0
3627             },
3628             {
3629               "hash": 1534491824,
3630               "shard": 1
3631             },
3632             {
3633               "hash": 1534350760,
3634               "shard": 2
3635             }
3636            ],
3637            "total_chunk_size": 2048
3638          },
3639          "errors": [],
3640          "osd": 2,
3641          "primary": false,
3642          "shard": 1,
3643          "size": 2048
3644        }
3645      ],
3646      "union_shard_errors": []
3647     }
3648   ],
3649   "epoch": 0
3650 }
3651 EOF
3652
3653     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
3654     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
3655     if test $getjson = "yes"
3656     then
3657         jq '.' $dir/json > save3.json
3658     fi
3659
3660     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
3661     then
3662       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
3663     fi
3664
3665     pg_deep_scrub $pg
3666
3667     rados list-inconsistent-pg $poolname > $dir/json || return 1
3668     # Check pg count
3669     test $(jq '. | length' $dir/json) = "1" || return 1
3670     # Check pgid
3671     test $(jq -r '.[0]' $dir/json) = $pg || return 1
3672
3673     rados list-inconsistent-obj $pg > $dir/json || return 1
3674     # Get epoch for repair-get requests
3675     epoch=$(jq .epoch $dir/json)
3676
3677     if [ "$allow_overwrites" = "true" ]
3678     then
3679       jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
3680 {
3681   "inconsistents": [
3682     {
3683       "shards": [
3684         {
3685           "data_digest": "0x00000000",
3686           "omap_digest": "0xffffffff",
3687           "size": 2048,
3688           "errors": [],
3689           "shard": 2,
3690           "osd": 0,
3691           "primary": false
3692         },
3693         {
3694           "object_info": {
3695             "oid": {
3696               "oid": "EOBJ1",
3697               "key": "",
3698               "snapid": -2,
3699               "hash": 560836233,
3700               "max": 0,
3701               "pool": 3,
3702               "namespace": ""
3703             },
3704             "version": "27'1",
3705             "prior_version": "0'0",
3706             "last_reqid": "client.4184.0:1",
3707             "user_version": 1,
3708             "size": 7,
3709             "mtime": "2018-04-05 14:31:33.837147",
3710             "local_mtime": "2018-04-05 14:31:33.840763",
3711             "lost": 0,
3712             "flags": [
3713               "dirty",
3714               "data_digest"
3715             ],
3716             "truncate_seq": 0,
3717             "truncate_size": 0,
3718             "data_digest": "0x2ddbf8f5",
3719             "omap_digest": "0xffffffff",
3720             "expected_object_size": 0,
3721             "expected_write_size": 0,
3722             "alloc_hint_flags": 0,
3723             "manifest": {
3724               "type": 0
3725             },
3726             "watchers": {}
3727           },
3728           "size": 9,
3729           "shard": 0,
3730           "errors": [
3731             "read_error",
3732             "size_mismatch_info",
3733             "obj_size_info_mismatch"
3734           ],
3735           "osd": 1,
3736           "primary": true
3737         },
3738         {
3739           "data_digest": "0x00000000",
3740           "omap_digest": "0xffffffff",
3741           "size": 2048,
3742           "shard": 1,
3743           "errors": [],
3744           "osd": 2,
3745           "primary": false
3746         }
3747       ],
3748       "selected_object_info": {
3749         "oid": {
3750           "oid": "EOBJ1",
3751           "key": "",
3752           "snapid": -2,
3753           "hash": 560836233,
3754           "max": 0,
3755           "pool": 3,
3756           "namespace": ""
3757         },
3758         "version": "27'1",
3759         "prior_version": "0'0",
3760         "last_reqid": "client.4184.0:1",
3761         "user_version": 1,
3762         "size": 7,
3763         "mtime": "2018-04-05 14:31:33.837147",
3764         "local_mtime": "2018-04-05 14:31:33.840763",
3765         "lost": 0,
3766         "flags": [
3767           "dirty",
3768           "data_digest"
3769         ],
3770         "truncate_seq": 0,
3771         "truncate_size": 0,
3772         "data_digest": "0x2ddbf8f5",
3773         "omap_digest": "0xffffffff",
3774         "expected_object_size": 0,
3775         "expected_write_size": 0,
3776         "alloc_hint_flags": 0,
3777         "manifest": {
3778           "type": 0
3779         },
3780         "watchers": {}
3781       },
3782       "union_shard_errors": [
3783         "read_error",
3784         "size_mismatch_info",
3785         "obj_size_info_mismatch"
3786       ],
3787       "errors": [
3788         "size_mismatch"
3789       ],
3790       "object": {
3791         "version": 1,
3792         "snap": "head",
3793         "locator": "",
3794         "nspace": "",
3795         "name": "EOBJ1"
3796       }
3797     },
3798     {
3799       "shards": [
3800         {
3801           "data_digest": "0x00000000",
3802           "omap_digest": "0xffffffff",
3803           "size": 2048,
3804           "errors": [],
3805           "shard": 2,
3806           "osd": 0,
3807           "primary": false
3808         },
3809         {
3810           "shard": 0,
3811           "errors": [
3812             "missing"
3813           ],
3814           "osd": 1,
3815           "primary": true
3816         },
3817         {
3818           "data_digest": "0x00000000",
3819           "omap_digest": "0xffffffff",
3820           "size": 2048,
3821           "shard": 1,
3822           "errors": [],
3823           "osd": 2,
3824           "primary": false
3825         }
3826       ],
3827       "selected_object_info": {
3828         "oid": {
3829           "oid": "EOBJ3",
3830           "key": "",
3831           "snapid": -2,
3832           "hash": 3125668237,
3833           "max": 0,
3834           "pool": 3,
3835           "namespace": ""
3836         },
3837         "version": "39'3",
3838         "prior_version": "0'0",
3839         "last_reqid": "client.4252.0:1",
3840         "user_version": 3,
3841         "size": 7,
3842         "mtime": "2018-04-05 14:31:46.841145",
3843         "local_mtime": "2018-04-05 14:31:46.844996",
3844         "lost": 0,
3845         "flags": [
3846           "dirty",
3847           "data_digest"
3848         ],
3849         "truncate_seq": 0,
3850         "truncate_size": 0,
3851         "data_digest": "0x2ddbf8f5",
3852         "omap_digest": "0xffffffff",
3853         "expected_object_size": 0,
3854         "expected_write_size": 0,
3855         "alloc_hint_flags": 0,
3856         "manifest": {
3857           "type": 0
3858         },
3859         "watchers": {}
3860       },
3861       "union_shard_errors": [
3862         "missing"
3863       ],
3864       "errors": [],
3865       "object": {
3866         "version": 3,
3867         "snap": "head",
3868         "locator": "",
3869         "nspace": "",
3870         "name": "EOBJ3"
3871       }
3872     },
3873     {
3874       "shards": [
3875         {
3876           "attrs": [
3877             {
3878               "Base64": false,
3879               "value": "bad-val",
3880               "name": "key1-EOBJ4"
3881             },
3882             {
3883               "Base64": false,
3884               "value": "val2-EOBJ4",
3885               "name": "key2-EOBJ4"
3886             }
3887           ],
3888           "data_digest": "0x00000000",
3889           "omap_digest": "0xffffffff",
3890           "size": 2048,
3891           "errors": [],
3892           "shard": 2,
3893           "osd": 0,
3894           "primary": false
3895         },
3896         {
3897           "attrs": [
3898             {
3899               "Base64": false,
3900               "value": "val1-EOBJ4",
3901               "name": "key1-EOBJ4"
3902             },
3903             {
3904               "Base64": false,
3905               "value": "val2-EOBJ4",
3906               "name": "key2-EOBJ4"
3907             }
3908           ],
3909           "data_digest": "0x00000000",
3910           "omap_digest": "0xffffffff",
3911           "size": 2048,
3912           "errors": [],
3913           "shard": 0,
3914           "osd": 1,
3915           "primary": true
3916         },
3917         {
3918           "attrs": [
3919             {
3920               "Base64": false,
3921               "value": "val1-EOBJ4",
3922               "name": "key1-EOBJ4"
3923             },
3924             {
3925               "Base64": false,
3926               "value": "val3-EOBJ4",
3927               "name": "key3-EOBJ4"
3928             }
3929           ],
3930           "data_digest": "0x00000000",
3931           "omap_digest": "0xffffffff",
3932           "size": 2048,
3933           "errors": [],
3934           "shard": 1,
3935           "osd": 2,
3936           "primary": false
3937         }
3938       ],
3939       "selected_object_info": {
3940         "oid": {
3941           "oid": "EOBJ4",
3942           "key": "",
3943           "snapid": -2,
3944           "hash": 1618759290,
3945           "max": 0,
3946           "pool": 3,
3947           "namespace": ""
3948         },
3949         "version": "45'6",
3950         "prior_version": "45'5",
3951         "last_reqid": "client.4294.0:1",
3952         "user_version": 6,
3953         "size": 7,
3954         "mtime": "2018-04-05 14:31:54.663622",
3955         "local_mtime": "2018-04-05 14:31:54.664527",
3956         "lost": 0,
3957         "flags": [
3958           "dirty",
3959           "data_digest"
3960         ],
3961         "truncate_seq": 0,
3962         "truncate_size": 0,
3963         "data_digest": "0x2ddbf8f5",
3964         "omap_digest": "0xffffffff",
3965         "expected_object_size": 0,
3966         "expected_write_size": 0,
3967         "alloc_hint_flags": 0,
3968         "manifest": {
3969           "type": 0
3970         },
3971         "watchers": {}
3972       },
3973       "union_shard_errors": [],
3974       "errors": [
3975         "attr_value_mismatch",
3976         "attr_name_mismatch"
3977       ],
3978       "object": {
3979         "version": 6,
3980         "snap": "head",
3981         "locator": "",
3982         "nspace": "",
3983         "name": "EOBJ4"
3984       }
3985     },
3986     {
3987       "shards": [
3988         {
3989           "data_digest": "0x00000000",
3990           "omap_digest": "0xffffffff",
3991           "size": 2048,
3992           "errors": [],
3993           "shard": 2,
3994           "osd": 0,
3995           "primary": false
3996         },
3997         {
3998           "data_digest": "0x00000000",
3999           "omap_digest": "0xffffffff",
4000           "object_info": {
4001             "oid": {
4002               "oid": "EOBJ5",
4003               "key": "",
4004               "snapid": -2,
4005               "hash": 2918945441,
4006               "max": 0,
4007               "pool": 3,
4008               "namespace": ""
4009             },
4010             "version": "59'7",
4011             "prior_version": "0'0",
4012             "last_reqid": "client.4382.0:1",
4013             "user_version": 7,
4014             "size": 7,
4015             "mtime": "2018-04-05 14:32:12.929161",
4016             "local_mtime": "2018-04-05 14:32:12.934707",
4017             "lost": 0,
4018             "flags": [
4019               "dirty",
4020               "data_digest"
4021             ],
4022             "truncate_seq": 0,
4023             "truncate_size": 0,
4024             "data_digest": "0x2ddbf8f5",
4025             "omap_digest": "0xffffffff",
4026             "expected_object_size": 0,
4027             "expected_write_size": 0,
4028             "alloc_hint_flags": 0,
4029             "manifest": {
4030               "type": 0
4031             },
4032             "watchers": {}
4033           },
4034           "size": 4096,
4035           "errors": [
4036             "size_mismatch_info",
4037             "obj_size_info_mismatch"
4038           ],
4039           "shard": 0,
4040           "osd": 1,
4041           "primary": true
4042         },
4043         {
4044           "data_digest": "0x00000000",
4045           "omap_digest": "0xffffffff",
4046           "size": 2048,
4047           "errors": [],
4048           "shard": 1,
4049           "osd": 2,
4050           "primary": false
4051         }
4052       ],
4053       "selected_object_info": {
4054         "oid": {
4055           "oid": "EOBJ5",
4056           "key": "",
4057           "snapid": -2,
4058           "hash": 2918945441,
4059           "max": 0,
4060           "pool": 3,
4061           "namespace": ""
4062         },
4063         "version": "59'7",
4064         "prior_version": "0'0",
4065         "last_reqid": "client.4382.0:1",
4066         "user_version": 7,
4067         "size": 7,
4068         "mtime": "2018-04-05 14:32:12.929161",
4069         "local_mtime": "2018-04-05 14:32:12.934707",
4070         "lost": 0,
4071         "flags": [
4072           "dirty",
4073           "data_digest"
4074         ],
4075         "truncate_seq": 0,
4076         "truncate_size": 0,
4077         "data_digest": "0x2ddbf8f5",
4078         "omap_digest": "0xffffffff",
4079         "expected_object_size": 0,
4080         "expected_write_size": 0,
4081         "alloc_hint_flags": 0,
4082         "manifest": {
4083           "type": 0
4084         },
4085         "watchers": {}
4086       },
4087       "union_shard_errors": [
4088         "size_mismatch_info",
4089         "obj_size_info_mismatch"
4090       ],
4091       "errors": [
4092         "size_mismatch"
4093       ],
4094       "object": {
4095         "version": 7,
4096         "snap": "head",
4097         "locator": "",
4098         "nspace": "",
4099         "name": "EOBJ5"
4100       }
4101     },
4102     {
4103       "object": {
4104         "name": "EOBJ6",
4105         "nspace": "",
4106         "locator": "",
4107         "snap": "head",
4108         "version": 8
4109       },
4110       "errors": [],
4111       "union_shard_errors": [
4112         "read_error",
4113         "hinfo_missing",
4114         "hinfo_corrupted"
4115       ],
4116       "selected_object_info": {
4117         "oid": {
4118           "oid": "EOBJ6",
4119           "key": "",
4120           "snapid": -2,
4121           "hash": 3050890866,
4122           "max": 0,
4123           "pool": 3,
4124           "namespace": ""
4125         },
4126         "version": "65'8",
4127         "prior_version": "0'0",
4128         "last_reqid": "client.4418.0:1",
4129         "user_version": 8,
4130         "size": 7,
4131         "mtime": "2018-04-05 14:32:20.634116",
4132         "local_mtime": "2018-04-05 14:32:20.637999",
4133         "lost": 0,
4134         "flags": [
4135           "dirty",
4136           "data_digest"
4137         ],
4138         "truncate_seq": 0,
4139         "truncate_size": 0,
4140         "data_digest": "0x2ddbf8f5",
4141         "omap_digest": "0xffffffff",
4142         "expected_object_size": 0,
4143         "expected_write_size": 0,
4144         "alloc_hint_flags": 0,
4145         "manifest": {
4146           "type": 0
4147         },
4148         "watchers": {}
4149       },
4150       "shards": [
4151         {
4152           "osd": 0,
4153           "primary": false,
4154           "shard": 2,
4155           "errors": [
4156             "read_error",
4157             "hinfo_missing"
4158           ],
4159           "size": 2048
4160         },
4161         {
4162           "osd": 1,
4163           "primary": true,
4164           "shard": 0,
4165           "errors": [
4166             "read_error",
4167             "hinfo_corrupted"
4168           ],
4169           "size": 2048,
4170           "hashinfo": "bad-val"
4171         },
4172         {
4173           "osd": 2,
4174           "primary": false,
4175           "shard": 1,
4176           "errors": [],
4177           "size": 2048,
4178           "omap_digest": "0xffffffff",
4179           "data_digest": "0x00000000",
4180           "hashinfo": {
4181             "cumulative_shard_hashes": [
4182             {
4183               "hash": 80717615,
4184               "shard": 0
4185             },
4186             {
4187               "hash": 1534491824,
4188               "shard": 1
4189             },
4190             {
4191               "hash": 80717615,
4192               "shard": 2
4193             }
4194            ],
4195            "total_chunk_size": 2048
4196          }
4197         }
4198       ]
4199     },
4200     {
4201       "object": {
4202         "name": "EOBJ7",
4203         "nspace": "",
4204         "locator": "",
4205         "snap": "head",
4206         "version": 10
4207       },
4208       "errors": [
4209         "hinfo_inconsistency"
4210       ],
4211       "union_shard_errors": [],
4212       "selected_object_info": {
4213         "oid": {
4214           "oid": "EOBJ7",
4215           "key": "",
4216           "snapid": -2,
4217           "hash": 3258066308,
4218           "max": 0,
4219           "pool": 3,
4220           "namespace": ""
4221         },
4222         "version": "75'10",
4223         "prior_version": "75'9",
4224         "last_reqid": "client.4482.0:1",
4225         "user_version": 10,
4226         "size": 34,
4227         "mtime": "2018-04-05 14:32:33.058782",
4228         "local_mtime": "2018-04-05 14:32:33.059679",
4229         "lost": 0,
4230         "flags": [
4231           "dirty",
4232           "data_digest"
4233         ],
4234         "truncate_seq": 0,
4235         "truncate_size": 0,
4236         "data_digest": "0x136e4e27",
4237         "omap_digest": "0xffffffff",
4238         "expected_object_size": 0,
4239         "expected_write_size": 0,
4240         "alloc_hint_flags": 0,
4241         "manifest": {
4242           "type": 0
4243         },
4244         "watchers": {}
4245       },
4246       "shards": [
4247         {
4248           "osd": 0,
4249           "primary": false,
4250           "shard": 2,
4251           "errors": [],
4252           "size": 2048,
4253           "omap_digest": "0xffffffff",
4254           "data_digest": "0x00000000",
4255           "hashinfo": {
4256            "cumulative_shard_hashes": [
4257             {
4258               "hash": 80717615,
4259               "shard": 0
4260             },
4261             {
4262               "hash": 1534491824,
4263               "shard": 1
4264             },
4265             {
4266               "hash": 80717615,
4267               "shard": 2
4268             }
4269            ],
4270            "total_chunk_size": 2048
4271           }
4272         },
4273         {
4274           "osd": 1,
4275           "primary": true,
4276           "shard": 0,
4277           "errors": [],
4278           "size": 2048,
4279           "omap_digest": "0xffffffff",
4280           "data_digest": "0x00000000",
4281           "hashinfo": {
4282            "cumulative_shard_hashes": [
4283             {
4284               "hash": 1534350760,
4285               "shard": 0
4286             },
4287             {
4288               "hash": 1534491824,
4289               "shard": 1
4290             },
4291             {
4292               "hash": 1534350760,
4293               "shard": 2
4294             }
4295            ],
4296            "total_chunk_size": 2048
4297           }
4298         },
4299         {
4300           "osd": 2,
4301           "primary": false,
4302           "shard": 1,
4303           "errors": [],
4304           "size": 2048,
4305           "omap_digest": "0xffffffff",
4306           "data_digest": "0x00000000",
4307           "hashinfo": {
4308            "cumulative_shard_hashes": [
4309             {
4310               "hash": 1534350760,
4311               "shard": 0
4312             },
4313             {
4314               "hash": 1534491824,
4315               "shard": 1
4316             },
4317             {
4318               "hash": 1534350760,
4319               "shard": 2
4320             }
4321            ],
4322            "total_chunk_size": 2048
4323           }
4324         }
4325       ]
4326     }
4327   ],
4328   "epoch": 0
4329 }
4330 EOF
4331
4332     else
4333
4334       jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
4335 {
4336   "inconsistents": [
4337     {
4338       "shards": [
4339         {
4340           "data_digest": "0x04cfa72f",
4341           "omap_digest": "0xffffffff",
4342           "size": 2048,
4343           "errors": [],
4344           "shard": 2,
4345           "osd": 0,
4346           "primary": false
4347         },
4348         {
4349           "object_info": {
4350             "oid": {
4351               "oid": "EOBJ1",
4352               "key": "",
4353               "snapid": -2,
4354               "hash": 560836233,
4355               "max": 0,
4356               "pool": 3,
4357               "namespace": ""
4358             },
4359             "version": "27'1",
4360             "prior_version": "0'0",
4361             "last_reqid": "client.4192.0:1",
4362             "user_version": 1,
4363             "size": 7,
4364             "mtime": "2018-04-05 14:30:10.688009",
4365             "local_mtime": "2018-04-05 14:30:10.691774",
4366             "lost": 0,
4367             "flags": [
4368               "dirty",
4369               "data_digest"
4370             ],
4371             "truncate_seq": 0,
4372             "truncate_size": 0,
4373             "data_digest": "0x2ddbf8f5",
4374             "omap_digest": "0xffffffff",
4375             "expected_object_size": 0,
4376             "expected_write_size": 0,
4377             "alloc_hint_flags": 0,
4378             "manifest": {
4379               "type": 0
4380             },
4381             "watchers": {}
4382           },
4383           "size": 9,
4384           "shard": 0,
4385           "errors": [
4386             "read_error",
4387             "size_mismatch_info",
4388             "obj_size_info_mismatch"
4389           ],
4390           "osd": 1,
4391           "primary": true
4392         },
4393         {
4394           "data_digest": "0x04cfa72f",
4395           "omap_digest": "0xffffffff",
4396           "size": 2048,
4397           "shard": 1,
4398           "errors": [],
4399           "osd": 2,
4400           "primary": false
4401         }
4402       ],
4403       "selected_object_info": {
4404         "oid": {
4405           "oid": "EOBJ1",
4406           "key": "",
4407           "snapid": -2,
4408           "hash": 560836233,
4409           "max": 0,
4410           "pool": 3,
4411           "namespace": ""
4412         },
4413         "version": "27'1",
4414         "prior_version": "0'0",
4415         "last_reqid": "client.4192.0:1",
4416         "user_version": 1,
4417         "size": 7,
4418         "mtime": "2018-04-05 14:30:10.688009",
4419         "local_mtime": "2018-04-05 14:30:10.691774",
4420         "lost": 0,
4421         "flags": [
4422           "dirty",
4423           "data_digest"
4424         ],
4425         "truncate_seq": 0,
4426         "truncate_size": 0,
4427         "data_digest": "0x2ddbf8f5",
4428         "omap_digest": "0xffffffff",
4429         "expected_object_size": 0,
4430         "expected_write_size": 0,
4431         "alloc_hint_flags": 0,
4432         "manifest": {
4433           "type": 0
4434         },
4435         "watchers": {}
4436       },
4437       "union_shard_errors": [
4438         "read_error",
4439         "size_mismatch_info",
4440         "obj_size_info_mismatch"
4441       ],
4442       "errors": [
4443         "size_mismatch"
4444       ],
4445       "object": {
4446         "version": 1,
4447         "snap": "head",
4448         "locator": "",
4449         "nspace": "",
4450         "name": "EOBJ1"
4451       }
4452     },
4453     {
4454       "shards": [
4455         {
4456           "size": 2048,
4457           "errors": [
4458             "ec_hash_error"
4459           ],
4460           "shard": 2,
4461           "osd": 0,
4462           "primary": false
4463         },
4464         {
4465           "data_digest": "0x04cfa72f",
4466           "omap_digest": "0xffffffff",
4467           "size": 2048,
4468           "errors": [],
4469           "shard": 0,
4470           "osd": 1,
4471           "primary": true
4472         },
4473         {
4474           "data_digest": "0x04cfa72f",
4475           "omap_digest": "0xffffffff",
4476           "size": 2048,
4477           "errors": [],
4478           "shard": 1,
4479           "osd": 2,
4480           "primary": false
4481         }
4482       ],
4483       "selected_object_info": {
4484         "oid": {
4485           "oid": "EOBJ2",
4486           "key": "",
4487           "snapid": -2,
4488           "hash": 562812377,
4489           "max": 0,
4490           "pool": 3,
4491           "namespace": ""
4492         },
4493         "version": "33'2",
4494         "prior_version": "0'0",
4495         "last_reqid": "client.4224.0:1",
4496         "user_version": 2,
4497         "size": 7,
4498         "mtime": "2018-04-05 14:30:14.152945",
4499         "local_mtime": "2018-04-05 14:30:14.154014",
4500         "lost": 0,
4501         "flags": [
4502           "dirty",
4503           "data_digest"
4504         ],
4505         "truncate_seq": 0,
4506         "truncate_size": 0,
4507         "data_digest": "0x2ddbf8f5",
4508         "omap_digest": "0xffffffff",
4509         "expected_object_size": 0,
4510         "expected_write_size": 0,
4511         "alloc_hint_flags": 0,
4512         "manifest": {
4513           "type": 0
4514         },
4515         "watchers": {}
4516       },
4517       "union_shard_errors": [
4518         "ec_hash_error"
4519       ],
4520       "errors": [],
4521       "object": {
4522         "version": 2,
4523         "snap": "head",
4524         "locator": "",
4525         "nspace": "",
4526         "name": "EOBJ2"
4527       }
4528     },
4529     {
4530       "shards": [
4531         {
4532           "data_digest": "0x04cfa72f",
4533           "omap_digest": "0xffffffff",
4534           "size": 2048,
4535           "errors": [],
4536           "shard": 2,
4537           "osd": 0,
4538           "primary": false
4539         },
4540         {
4541           "osd": 1,
4542           "primary": true,
4543           "shard": 0,
4544           "errors": [
4545             "missing"
4546           ]
4547         },
4548         {
4549           "data_digest": "0x04cfa72f",
4550           "omap_digest": "0xffffffff",
4551           "size": 2048,
4552           "shard": 1,
4553           "errors": [],
4554           "osd": 2,
4555           "primary": false
4556         }
4557       ],
4558       "selected_object_info": {
4559         "oid": {
4560           "oid": "EOBJ3",
4561           "key": "",
4562           "snapid": -2,
4563           "hash": 3125668237,
4564           "max": 0,
4565           "pool": 3,
4566           "namespace": ""
4567         },
4568         "version": "39'3",
4569         "prior_version": "0'0",
4570         "last_reqid": "client.4258.0:1",
4571         "user_version": 3,
4572         "size": 7,
4573         "mtime": "2018-04-05 14:30:18.875544",
4574         "local_mtime": "2018-04-05 14:30:18.880153",
4575         "lost": 0,
4576         "flags": [
4577           "dirty",
4578           "data_digest"
4579         ],
4580         "truncate_seq": 0,
4581         "truncate_size": 0,
4582         "data_digest": "0x2ddbf8f5",
4583         "omap_digest": "0xffffffff",
4584         "expected_object_size": 0,
4585         "expected_write_size": 0,
4586         "alloc_hint_flags": 0,
4587         "manifest": {
4588           "type": 0
4589         },
4590         "watchers": {}
4591       },
4592       "union_shard_errors": [
4593         "missing"
4594       ],
4595       "errors": [],
4596       "object": {
4597         "version": 3,
4598         "snap": "head",
4599         "locator": "",
4600         "nspace": "",
4601         "name": "EOBJ3"
4602       }
4603     },
4604     {
4605       "shards": [
4606         {
4607           "attrs": [
4608             {
4609               "Base64": false,
4610               "value": "bad-val",
4611               "name": "key1-EOBJ4"
4612             },
4613             {
4614               "Base64": false,
4615               "value": "val2-EOBJ4",
4616               "name": "key2-EOBJ4"
4617             }
4618           ],
4619           "data_digest": "0x04cfa72f",
4620           "omap_digest": "0xffffffff",
4621           "size": 2048,
4622           "errors": [],
4623           "shard": 2,
4624           "osd": 0,
4625           "primary": false
4626         },
4627         {
4628           "osd": 1,
4629           "primary": true,
4630           "shard": 0,
4631           "errors": [],
4632           "size": 2048,
4633           "omap_digest": "0xffffffff",
4634           "data_digest": "0x04cfa72f",
4635           "attrs": [
4636             {
4637               "Base64": false,
4638               "value": "val1-EOBJ4",
4639               "name": "key1-EOBJ4"
4640             },
4641             {
4642               "Base64": false,
4643               "value": "val2-EOBJ4",
4644               "name": "key2-EOBJ4"
4645             }
4646           ]
4647         },
4648         {
4649           "osd": 2,
4650           "primary": false,
4651           "shard": 1,
4652           "errors": [],
4653           "size": 2048,
4654           "omap_digest": "0xffffffff",
4655           "data_digest": "0x04cfa72f",
4656           "attrs": [
4657             {
4658               "Base64": false,
4659               "value": "val1-EOBJ4",
4660               "name": "key1-EOBJ4"
4661             },
4662             {
4663               "Base64": false,
4664               "value": "val3-EOBJ4",
4665               "name": "key3-EOBJ4"
4666             }
4667           ]
4668         }
4669       ],
4670       "selected_object_info": {
4671         "oid": {
4672           "oid": "EOBJ4",
4673           "key": "",
4674           "snapid": -2,
4675           "hash": 1618759290,
4676           "max": 0,
4677           "pool": 3,
4678           "namespace": ""
4679         },
4680         "version": "45'6",
4681         "prior_version": "45'5",
4682         "last_reqid": "client.4296.0:1",
4683         "user_version": 6,
4684         "size": 7,
4685         "mtime": "2018-04-05 14:30:22.271983",
4686         "local_mtime": "2018-04-05 14:30:22.272840",
4687         "lost": 0,
4688         "flags": [
4689           "dirty",
4690           "data_digest"
4691         ],
4692         "truncate_seq": 0,
4693         "truncate_size": 0,
4694         "data_digest": "0x2ddbf8f5",
4695         "omap_digest": "0xffffffff",
4696         "expected_object_size": 0,
4697         "expected_write_size": 0,
4698         "alloc_hint_flags": 0,
4699         "manifest": {
4700           "type": 0
4701         },
4702         "watchers": {}
4703       },
4704       "union_shard_errors": [],
4705       "errors": [
4706         "attr_value_mismatch",
4707         "attr_name_mismatch"
4708       ],
4709       "object": {
4710         "version": 6,
4711         "snap": "head",
4712         "locator": "",
4713         "nspace": "",
4714         "name": "EOBJ4"
4715       }
4716     },
4717     {
4718       "shards": [
4719         {
4720           "data_digest": "0x04cfa72f",
4721           "omap_digest": "0xffffffff",
4722           "size": 2048,
4723           "errors": [],
4724           "shard": 2,
4725           "osd": 0,
4726           "primary": false
4727         },
4728         {
4729           "object_info": {
4730             "oid": {
4731               "oid": "EOBJ5",
4732               "key": "",
4733               "snapid": -2,
4734               "hash": 2918945441,
4735               "max": 0,
4736               "pool": 3,
4737               "namespace": ""
4738             },
4739             "version": "59'7",
4740             "prior_version": "0'0",
4741             "last_reqid": "client.4384.0:1",
4742             "user_version": 7,
4743             "size": 7,
4744             "mtime": "2018-04-05 14:30:35.162395",
4745             "local_mtime": "2018-04-05 14:30:35.166390",
4746             "lost": 0,
4747             "flags": [
4748               "dirty",
4749               "data_digest"
4750             ],
4751             "truncate_seq": 0,
4752             "truncate_size": 0,
4753             "data_digest": "0x2ddbf8f5",
4754             "omap_digest": "0xffffffff",
4755             "expected_object_size": 0,
4756             "expected_write_size": 0,
4757             "alloc_hint_flags": 0,
4758             "manifest": {
4759               "type": 0
4760             },
4761             "watchers": {}
4762           },
4763           "size": 4096,
4764           "shard": 0,
4765           "errors": [
4766             "size_mismatch_info",
4767             "ec_size_error",
4768             "obj_size_info_mismatch"
4769           ],
4770           "osd": 1,
4771           "primary": true
4772         },
4773         {
4774           "data_digest": "0x04cfa72f",
4775           "omap_digest": "0xffffffff",
4776           "size": 2048,
4777           "shard": 1,
4778           "errors": [],
4779           "osd": 2,
4780           "primary": false
4781         }
4782       ],
4783       "selected_object_info": {
4784         "oid": {
4785           "oid": "EOBJ5",
4786           "key": "",
4787           "snapid": -2,
4788           "hash": 2918945441,
4789           "max": 0,
4790           "pool": 3,
4791           "namespace": ""
4792         },
4793         "version": "59'7",
4794         "prior_version": "0'0",
4795         "last_reqid": "client.4384.0:1",
4796         "user_version": 7,
4797         "size": 7,
4798         "mtime": "2018-04-05 14:30:35.162395",
4799         "local_mtime": "2018-04-05 14:30:35.166390",
4800         "lost": 0,
4801         "flags": [
4802           "dirty",
4803           "data_digest"
4804         ],
4805         "truncate_seq": 0,
4806         "truncate_size": 0,
4807         "data_digest": "0x2ddbf8f5",
4808         "omap_digest": "0xffffffff",
4809         "expected_object_size": 0,
4810         "expected_write_size": 0,
4811         "alloc_hint_flags": 0,
4812         "manifest": {
4813           "type": 0
4814         },
4815         "watchers": {}
4816       },
4817       "union_shard_errors": [
4818         "size_mismatch_info",
4819         "ec_size_error",
4820         "obj_size_info_mismatch"
4821       ],
4822       "errors": [
4823         "size_mismatch"
4824       ],
4825       "object": {
4826         "version": 7,
4827         "snap": "head",
4828         "locator": "",
4829         "nspace": "",
4830         "name": "EOBJ5"
4831       }
4832     },
4833     {
4834       "object": {
4835         "name": "EOBJ6",
4836         "nspace": "",
4837         "locator": "",
4838         "snap": "head",
4839         "version": 8
4840       },
4841       "errors": [],
4842       "union_shard_errors": [
4843         "read_error",
4844         "hinfo_missing",
4845         "hinfo_corrupted"
4846       ],
4847       "selected_object_info": {
4848         "oid": {
4849           "oid": "EOBJ6",
4850           "key": "",
4851           "snapid": -2,
4852           "hash": 3050890866,
4853           "max": 0,
4854           "pool": 3,
4855           "namespace": ""
4856         },
4857         "version": "65'8",
4858         "prior_version": "0'0",
4859         "last_reqid": "client.4420.0:1",
4860         "user_version": 8,
4861         "size": 7,
4862         "mtime": "2018-04-05 14:30:40.914673",
4863         "local_mtime": "2018-04-05 14:30:40.917705",
4864         "lost": 0,
4865         "flags": [
4866           "dirty",
4867           "data_digest"
4868         ],
4869         "truncate_seq": 0,
4870         "truncate_size": 0,
4871         "data_digest": "0x2ddbf8f5",
4872         "omap_digest": "0xffffffff",
4873         "expected_object_size": 0,
4874         "expected_write_size": 0,
4875         "alloc_hint_flags": 0,
4876         "manifest": {
4877           "type": 0
4878         },
4879         "watchers": {}
4880       },
4881       "shards": [
4882         {
4883           "osd": 0,
4884           "primary": false,
4885           "shard": 2,
4886           "errors": [
4887             "read_error",
4888             "hinfo_missing"
4889           ],
4890           "size": 2048
4891         },
4892         {
4893           "osd": 1,
4894           "primary": true,
4895           "shard": 0,
4896           "errors": [
4897             "read_error",
4898             "hinfo_corrupted"
4899           ],
4900           "size": 2048,
4901           "hashinfo": "bad-val"
4902         },
4903         {
4904           "osd": 2,
4905           "primary": false,
4906           "shard": 1,
4907           "errors": [],
4908           "size": 2048,
4909           "omap_digest": "0xffffffff",
4910           "data_digest": "0x04cfa72f",
4911           "hashinfo": {
4912            "cumulative_shard_hashes": [
4913             {
4914               "hash": 80717615,
4915               "shard": 0
4916             },
4917             {
4918               "hash": 1534491824,
4919               "shard": 1
4920             },
4921             {
4922               "hash": 80717615,
4923               "shard": 2
4924             }
4925            ],
4926            "total_chunk_size": 2048
4927           }
4928         }
4929       ]
4930     },
4931     {
4932       "object": {
4933         "name": "EOBJ7",
4934         "nspace": "",
4935         "locator": "",
4936         "snap": "head",
4937         "version": 10
4938       },
4939       "errors": [
4940         "hinfo_inconsistency"
4941       ],
4942       "union_shard_errors": [
4943         "ec_hash_error"
4944       ],
4945       "selected_object_info": {
4946         "oid": {
4947           "oid": "EOBJ7",
4948           "key": "",
4949           "snapid": -2,
4950           "hash": 3258066308,
4951           "max": 0,
4952           "pool": 3,
4953           "namespace": ""
4954         },
4955         "version": "75'10",
4956         "prior_version": "75'9",
4957         "last_reqid": "client.4486.0:1",
4958         "user_version": 10,
4959         "size": 34,
4960         "mtime": "2018-04-05 14:30:50.995009",
4961         "local_mtime": "2018-04-05 14:30:50.996112",
4962         "lost": 0,
4963         "flags": [
4964           "dirty",
4965           "data_digest"
4966         ],
4967         "truncate_seq": 0,
4968         "truncate_size": 0,
4969         "data_digest": "0x136e4e27",
4970         "omap_digest": "0xffffffff",
4971         "expected_object_size": 0,
4972         "expected_write_size": 0,
4973         "alloc_hint_flags": 0,
4974         "manifest": {
4975           "type": 0
4976         },
4977         "watchers": {}
4978       },
4979       "shards": [
4980         {
4981           "osd": 0,
4982           "primary": false,
4983           "shard": 2,
4984           "errors": [
4985             "ec_hash_error"
4986           ],
4987           "size": 2048,
4988           "hashinfo": {
4989            "cumulative_shard_hashes": [
4990             {
4991               "hash": 80717615,
4992               "shard": 0
4993             },
4994             {
4995               "hash": 1534491824,
4996               "shard": 1
4997             },
4998             {
4999               "hash": 80717615,
5000               "shard": 2
5001             }
5002            ],
5003            "total_chunk_size": 2048
5004          }
5005         },
5006         {
5007           "osd": 1,
5008           "primary": true,
5009           "shard": 0,
5010           "errors": [],
5011           "size": 2048,
5012           "omap_digest": "0xffffffff",
5013           "data_digest": "0x5b7455a8",
5014           "hashinfo": {
5015            "cumulative_shard_hashes": [
5016             {
5017               "hash": 1534350760,
5018               "shard": 0
5019             },
5020             {
5021               "hash": 1534491824,
5022               "shard": 1
5023             },
5024             {
5025               "hash": 1534350760,
5026               "shard": 2
5027             }
5028            ],
5029            "total_chunk_size": 2048
5030           }
5031         },
5032         {
5033           "osd": 2,
5034           "primary": false,
5035           "shard": 1,
5036           "errors": [],
5037           "size": 2048,
5038           "omap_digest": "0xffffffff",
5039           "data_digest": "0x5b7455a8",
5040           "hashinfo": {
5041            "cumulative_shard_hashes": [
5042             {
5043               "hash": 1534350760,
5044               "shard": 0
5045             },
5046             {
5047               "hash": 1534491824,
5048               "shard": 1
5049             },
5050             {
5051               "hash": 1534350760,
5052               "shard": 2
5053             }
5054            ],
5055            "total_chunk_size": 2048
5056           }
5057         }
5058       ]
5059     }
5060   ],
5061   "epoch": 0
5062 }
5063 EOF
5064
5065     fi
5066
5067     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
5068     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
5069     if test $getjson = "yes"
5070     then
5071       if [ "$allow_overwrites" = "true" ]
5072       then
5073         num=4
5074       else
5075         num=5
5076       fi
5077       jq '.' $dir/json > save${num}.json
5078     fi
5079
5080     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
5081     then
5082       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
5083     fi
5084
5085     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
5086     teardown $dir || return 1
5087 }
5088
5089 function TEST_corrupt_scrub_erasure_appends() {
5090     corrupt_scrub_erasure $1 false
5091 }
5092
5093 function TEST_corrupt_scrub_erasure_overwrites() {
5094     if [ "$use_ec_overwrite" = "true" ]; then
5095         corrupt_scrub_erasure $1 true
5096     fi
5097 }
5098
5099 #
5100 # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost
5101 #
5102 function TEST_periodic_scrub_replicated() {
5103     local dir=$1
5104     local poolname=psr_pool
5105     local objname=POBJ
5106
5107     setup $dir || return 1
5108     run_mon $dir a --osd_pool_default_size=2 || return 1
5109     run_mgr $dir x || return 1
5110     local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
5111     ceph_osd_args+="--osd_scrub_backoff_ratio=0"
5112     run_osd $dir 0 $ceph_osd_args || return 1
5113     run_osd $dir 1 $ceph_osd_args || return 1
5114     create_rbd_pool || return 1
5115     wait_for_clean || return 1
5116
5117     create_pool $poolname 1 1 || return 1
5118     wait_for_clean || return 1
5119
5120     local osd=0
5121     add_something $dir $poolname $objname scrub || return 1
5122     local primary=$(get_primary $poolname $objname)
5123     local pg=$(get_pg $poolname $objname)
5124
5125     # Add deep-scrub only error
5126     local payload=UVWXYZ
5127     echo $payload > $dir/CORRUPT
5128     # Uses $ceph_osd_args for osd restart
5129     objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
5130
5131     # No scrub information available, so expect failure
5132     set -o pipefail
5133     !  rados list-inconsistent-obj $pg | jq '.' || return 1
5134     set +o pipefail
5135
5136     pg_deep_scrub $pg || return 1
5137
5138     # Make sure bad object found
5139     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5140
5141     flush_pg_stats
5142     local last_scrub=$(get_last_scrub_stamp $pg)
5143     # Fake a schedule scrub
5144     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
5145              trigger_scrub $pg || return 1
5146     # Wait for schedule regular scrub
5147     wait_for_scrub $pg "$last_scrub"
5148
5149     # It needed to be upgraded
5150     grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1
5151
5152     # Bad object still known
5153     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5154
5155     # Can't upgrade with this set
5156     ceph osd set nodeep-scrub
5157     # Let map change propagate to OSDs
5158     flush pg_stats
5159     sleep 5
5160
5161     # Fake a schedule scrub
5162     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
5163              trigger_scrub $pg || return 1
5164     # Wait for schedule regular scrub
5165     # to notice scrub and skip it
5166     local found=false
5167     for i in $(seq 14 -1 0)
5168     do
5169       sleep 1
5170       ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; }
5171       echo Time left: $i seconds
5172     done
5173     test $found = "true" || return 1
5174
5175     # Bad object still known
5176     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5177
5178     flush_pg_stats
5179     # Request a regular scrub and it will be done
5180     pg_scrub $pg
5181     grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1
5182
5183     # deep-scrub error is no longer present
5184     rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1
5185 }
5186
5187 #
5188 # Corrupt snapset in replicated pool
5189 #
5190 function TEST_corrupt_snapset_scrub_rep() {
5191     local dir=$1
5192     local poolname=csr_pool
5193     local total_objs=2
5194
5195     setup $dir || return 1
5196     run_mon $dir a --osd_pool_default_size=2 || return 1
5197     run_mgr $dir x || return 1
5198     run_osd $dir 0 || return 1
5199     run_osd $dir 1 || return 1
5200     create_rbd_pool || return 1
5201     wait_for_clean || return 1
5202
5203     create_pool foo 1 || return 1
5204     create_pool $poolname 1 1 || return 1
5205     wait_for_clean || return 1
5206
5207     for i in $(seq 1 $total_objs) ; do
5208         objname=ROBJ${i}
5209         add_something $dir $poolname $objname || return 1
5210
5211         rados --pool $poolname setomapheader $objname hdr-$objname || return 1
5212         rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
5213     done
5214
5215     local pg=$(get_pg $poolname ROBJ0)
5216
5217     for i in $(seq 1 $total_objs) ; do
5218         objname=ROBJ${i}
5219
5220         # Alternate corruption between osd.0 and osd.1
5221         local osd=$(expr $i % 2)
5222
5223         rados -p $poolname mksnap snap1
5224         echo -n head_of_snapshot_data > $dir/change
5225
5226         case $i in
5227         1)
5228           rados --pool $poolname put $objname $dir/change
5229           objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
5230           ;;
5231
5232         2)
5233           rados --pool $poolname put $objname $dir/change
5234           objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
5235           ;;
5236
5237         esac
5238     done
5239     rm $dir/change
5240
5241     pg_scrub $pg
5242
5243     rados list-inconsistent-pg $poolname > $dir/json || return 1
5244     # Check pg count
5245     test $(jq '. | length' $dir/json) = "1" || return 1
5246     # Check pgid
5247     test $(jq -r '.[0]' $dir/json) = $pg || return 1
5248
5249     rados list-inconsistent-obj $pg > $dir/json || return 1
5250
5251     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
5252 {
5253   "epoch": 34,
5254   "inconsistents": [
5255     {
5256       "object": {
5257         "name": "ROBJ1",
5258         "nspace": "",
5259         "locator": "",
5260         "snap": "head",
5261         "version": 8
5262       },
5263       "errors": [
5264         "snapset_inconsistency"
5265       ],
5266       "union_shard_errors": [],
5267       "selected_object_info": {
5268         "oid": {
5269           "oid": "ROBJ1",
5270           "key": "",
5271           "snapid": -2,
5272           "hash": 1454963827,
5273           "max": 0,
5274           "pool": 3,
5275           "namespace": ""
5276         },
5277         "version": "24'8",
5278         "prior_version": "21'3",
5279         "last_reqid": "client.4195.0:1",
5280         "user_version": 8,
5281         "size": 21,
5282         "mtime": "2018-04-05 14:35:43.286117",
5283         "local_mtime": "2018-04-05 14:35:43.288990",
5284         "lost": 0,
5285         "flags": [
5286           "dirty",
5287           "omap",
5288           "data_digest"
5289         ],
5290         "truncate_seq": 0,
5291         "truncate_size": 0,
5292         "data_digest": "0x53acb008",
5293         "omap_digest": "0xffffffff",
5294         "expected_object_size": 0,
5295         "expected_write_size": 0,
5296         "alloc_hint_flags": 0,
5297         "manifest": {
5298           "type": 0
5299         },
5300         "watchers": {}
5301       },
5302       "shards": [
5303         {
5304           "osd": 0,
5305           "primary": false,
5306           "errors": [],
5307           "size": 21,
5308           "snapset": {
5309             "clones": [
5310               {
5311                 "overlap": "[]",
5312                 "size": 7,
5313                 "snap": 1,
5314                 "snaps": [
5315                   1
5316                 ]
5317               }
5318             ],
5319             "head_exists": 1,
5320             "snap_context": {
5321               "seq": 1,
5322               "snaps": [
5323                 1
5324               ]
5325             }
5326           }
5327         },
5328         {
5329           "osd": 1,
5330           "primary": true,
5331           "errors": [],
5332           "size": 21,
5333           "snapset": {
5334             "clones": [],
5335             "head_exists": 0,
5336             "snap_context": {
5337               "seq": 0,
5338               "snaps": []
5339             }
5340           }
5341         }
5342       ]
5343     },
5344     {
5345       "object": {
5346         "name": "ROBJ2",
5347         "nspace": "",
5348         "locator": "",
5349         "snap": "head",
5350         "version": 10
5351       },
5352       "errors": [
5353         "snapset_inconsistency"
5354       ],
5355       "union_shard_errors": [],
5356       "selected_object_info": {
5357         "oid": {
5358           "oid": "ROBJ2",
5359           "key": "",
5360           "snapid": -2,
5361           "hash": 2026323607,
5362           "max": 0,
5363           "pool": 3,
5364           "namespace": ""
5365         },
5366         "version": "28'10",
5367         "prior_version": "23'6",
5368         "last_reqid": "client.4223.0:1",
5369         "user_version": 10,
5370         "size": 21,
5371         "mtime": "2018-04-05 14:35:48.326856",
5372         "local_mtime": "2018-04-05 14:35:48.328097",
5373         "lost": 0,
5374         "flags": [
5375           "dirty",
5376           "omap",
5377           "data_digest"
5378         ],
5379         "truncate_seq": 0,
5380         "truncate_size": 0,
5381         "data_digest": "0x53acb008",
5382         "omap_digest": "0xffffffff",
5383         "expected_object_size": 0,
5384         "expected_write_size": 0,
5385         "alloc_hint_flags": 0,
5386         "manifest": {
5387           "type": 0
5388         },
5389         "watchers": {}
5390       },
5391       "shards": [
5392         {
5393           "osd": 0,
5394           "primary": false,
5395           "errors": [],
5396           "size": 21,
5397           "snapset": {
5398             "clones": [],
5399             "head_exists": 0,
5400             "snap_context": {
5401               "seq": 0,
5402               "snaps": []
5403             }
5404           }
5405         },
5406         {
5407           "osd": 1,
5408           "primary": true,
5409           "errors": [],
5410           "size": 21,
5411           "snapset": {
5412             "clones": [
5413               {
5414                 "overlap": "[]",
5415                 "size": 7,
5416                 "snap": 1,
5417                 "snaps": [
5418                   1
5419                 ]
5420               }
5421             ],
5422             "head_exists": 1,
5423             "snap_context": {
5424               "seq": 1,
5425               "snaps": [
5426                 1
5427               ]
5428             }
5429           }
5430         }
5431       ]
5432     }
5433   ]
5434 }
5435 EOF
5436
5437     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
5438     multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
5439     if test $getjson = "yes"
5440     then
5441         jq '.' $dir/json > save6.json
5442     fi
5443
5444     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
5445     then
5446       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
5447     fi
5448
5449     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
5450     teardown $dir || return 1
5451 }
5452
5453 main osd-scrub-repair "$@"
5454
5455 # Local Variables:
5456 # compile-command: "cd build ; make -j4 && \
5457 #    ../qa/run-standalone.sh osd-scrub-repair.sh"
5458 # End: