projects / ceph.git / blame
| Commit | Line | Data |
|---|---|---|
| 7c673cae FG |
1 | #!/bin/bash -x |
| 2 | # | |
| 3 | # Copyright (C) 2014 Red Hat <contact@redhat.com> | |
| 4 | # | |
| 5 | # Author: Loic Dachary <loic@dachary.org> | |
| 6 | # | |
| 7 | # This program is free software; you can redistribute it and/or modify | |
| 8 | # it under the terms of the GNU Library Public License as published by | |
| 9 | # the Free Software Foundation; either version 2, or (at your option) | |
| 10 | # any later version. | |
| 11 | # | |
| 12 | # This program is distributed in the hope that it will be useful, | |
| 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| 15 | # GNU Library Public License for more details. | |
| 16 | # | |
| 17 | source $(dirname $0)/../detect-build-env-vars.sh | |
| 18 | source $CEPH_ROOT/qa/workunits/ceph-helpers.sh | |
| 19 | ||
| 20 | # Test development and debugging | |
| 21 | # Set to "yes" in order to ignore diff errors and save results to update test | |
| 22 | getjson="no" | |
| 23 | ||
| 24 | termwidth=$(stty -a | head -1 | sed -e 's/.*columns \([0-9]*\).*/\1/') | |
| 25 | if test -n "$termwidth" -a "$termwidth" != "0"; then termwidth="-W ${termwidth}"; fi | |
| 26 | ||
| 27 | # Ignore the epoch and filter out the attr '_' value because it has date information and won't match | |
| 28 | jqfilter='.inconsistents | (.[].shards[].attrs[] | select(.name == "_") | .value) |= "----Stripped-by-test----"' | |
| 29 | sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' | |
| 30 | ||
| 31 | # Remove items are not consistent across runs, the pg interval and client | |
| 32 | sedfilter='s/\([ ]*\"\(selected_\)*object_info\":.*head[(]\)[^[:space:]]* [^[:space:]]* \(.*\)/\1\3/' | |
| 33 | ||
| 34 | function run() { | |
| 35 | local dir=$1 | |
| 36 | shift | |
| 37 | ||
| 38 | export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one | |
| 39 | export CEPH_ARGS | |
| 40 | CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " | |
| 41 | CEPH_ARGS+="--mon-host=$CEPH_MON " | |
| 42 | ||
| 43 | local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} | |
| 44 | for func in $funcs ; do | |
| 45 | $func $dir || return 1 | |
| 46 | done | |
| 47 | } | |
| 48 | ||
| 49 | function add_something() { | |
| 50 | local dir=$1 | |
| 51 | local poolname=$2 | |
| 52 | local obj=${3:-SOMETHING} | |
| 53 | local scrub=${4:-noscrub} | |
| 54 | ||
| 55 | if [ "$scrub" = "noscrub" ]; | |
| 56 | then | |
| 57 | ceph osd set noscrub || return 1 | |
| 58 | ceph osd set nodeep-scrub || return 1 | |
| 59 | else | |
| 60 | ceph osd unset noscrub || return 1 | |
| 61 | ceph osd unset nodeep-scrub || return 1 | |
| 62 | fi | |
| 63 | ||
| 64 | local payload=ABCDEF | |
| 65 | echo $payload > $dir/ORIGINAL | |
| 66 | rados --pool $poolname put $obj $dir/ORIGINAL || return 1 | |
| 67 | } | |
| 68 | ||
| 69 | # | |
| 70 | # Corrupt one copy of a replicated pool | |
| 71 | # | |
| 72 | function TEST_corrupt_and_repair_replicated() { | |
| 73 | local dir=$1 | |
| 74 | local poolname=rbd | |
| 75 | ||
| 76 | setup $dir || return 1 | |
| 77 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
| 78 | run_mgr $dir x || return 1 | |
| 79 | run_osd $dir 0 || return 1 | |
| 80 | run_osd $dir 1 || return 1 | |
| 81 | wait_for_clean || return 1 | |
| 82 | ||
| 83 | add_something $dir $poolname || return 1 | |
| 84 | corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1 | |
| 85 | # Reproduces http://tracker.ceph.com/issues/8914 | |
| 86 | corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1 | |
| 87 | ||
| 88 | teardown $dir || return 1 | |
| 89 | } | |
| 90 | ||
| 91 | function corrupt_and_repair_two() { | |
| 92 | local dir=$1 | |
| 93 | local poolname=$2 | |
| 94 | local first=$3 | |
| 95 | local second=$4 | |
| 96 | ||
| 97 | # | |
| 98 | # 1) remove the corresponding file from the OSDs | |
| 99 | # | |
| 100 | pids="" | |
| 101 | run_in_background pids objectstore_tool $dir $first SOMETHING remove | |
| 102 | run_in_background pids objectstore_tool $dir $second SOMETHING remove | |
| 103 | wait_background pids | |
| 104 | return_code=$? | |
| 105 | if [ $return_code -ne 0 ]; then return $return_code; fi | |
| 106 | ||
| 107 | # | |
| 108 | # 2) repair the PG | |
| 109 | # | |
| 110 | local pg=$(get_pg $poolname SOMETHING) | |
| 111 | repair $pg | |
| 112 | # | |
| 113 | # 3) The files must be back | |
| 114 | # | |
| 115 | pids="" | |
| 116 | run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs | |
| 117 | run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs | |
| 118 | wait_background pids | |
| 119 | return_code=$? | |
| 120 | if [ $return_code -ne 0 ]; then return $return_code; fi | |
| 121 | ||
| 122 | rados --pool $poolname get SOMETHING $dir/COPY || return 1 | |
| 123 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
| 124 | } | |
| 125 | ||
| 126 | # | |
| 127 | # 1) add an object | |
| 128 | # 2) remove the corresponding file from a designated OSD | |
| 129 | # 3) repair the PG | |
| 130 | # 4) check that the file has been restored in the designated OSD | |
| 131 | # | |
| 132 | function corrupt_and_repair_one() { | |
| 133 | local dir=$1 | |
| 134 | local poolname=$2 | |
| 135 | local osd=$3 | |
| 136 | ||
| 137 | # | |
| 138 | # 1) remove the corresponding file from the OSD | |
| 139 | # | |
| 140 | objectstore_tool $dir $osd SOMETHING remove || return 1 | |
| 141 | # | |
| 142 | # 2) repair the PG | |
| 143 | # | |
| 144 | local pg=$(get_pg $poolname SOMETHING) | |
| 145 | repair $pg | |
| 146 | # | |
| 147 | # 3) The file must be back | |
| 148 | # | |
| 149 | objectstore_tool $dir $osd SOMETHING list-attrs || return 1 | |
| 150 | rados --pool $poolname get SOMETHING $dir/COPY || return 1 | |
| 151 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
| 152 | } | |
| 153 | ||
| 154 | function corrupt_and_repair_erasure_coded() { | |
| 155 | local dir=$1 | |
| 156 | local poolname=$2 | |
| 157 | ||
| 158 | add_something $dir $poolname || return 1 | |
| 159 | ||
| 160 | local primary=$(get_primary $poolname SOMETHING) | |
| 161 | local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//")) | |
| 162 | local not_primary_first=${osds[0]} | |
| 163 | local not_primary_second=${osds[1]} | |
| 164 | ||
| 165 | # Reproduces http://tracker.ceph.com/issues/10017 | |
| 166 | corrupt_and_repair_one $dir $poolname $primary || return 1 | |
| 167 | # Reproduces http://tracker.ceph.com/issues/10409 | |
| 168 | corrupt_and_repair_one $dir $poolname $not_primary_first || return 1 | |
| 169 | corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1 | |
| 170 | corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1 | |
| 171 | ||
| 172 | } | |
| 173 | ||
| 174 | function create_ec_pool() { | |
| 175 | local pool_name=$1 | |
| 176 | local allow_overwrites=$2 | |
| 177 | ||
| 178 | ceph osd erasure-code-profile set myprofile ruleset-failure-domain=osd $3 $4 $5 $6 $7 || return 1 | |
| 179 | ||
| 180 | ceph osd pool create "$poolname" 1 1 erasure myprofile || return 1 | |
| 181 | ||
| 182 | if [ "$allow_overwrites" = "true" ]; then | |
| 183 | ceph osd pool set "$poolname" allow_ec_overwrites true || return 1 | |
| 184 | fi | |
| 185 | ||
| 186 | wait_for_clean || return 1 | |
| 187 | return 0 | |
| 188 | } | |
| 189 | ||
| 190 | function auto_repair_erasure_coded() { | |
| 191 | local dir=$1 | |
| 192 | local allow_overwrites=$2 | |
| 193 | local poolname=ecpool | |
| 194 | ||
| 195 | # Launch a cluster with 5 seconds scrub interval | |
| 196 | setup $dir || return 1 | |
| 197 | run_mon $dir a || return 1 | |
| 198 | run_mgr $dir x || return 1 | |
| 199 | local ceph_osd_args="--osd-scrub-auto-repair=true \ | |
| 200 | --osd-deep-scrub-interval=5 \ | |
| 201 | --osd-scrub-max-interval=5 \ | |
| 202 | --osd-scrub-min-interval=5 \ | |
| 203 | --osd-scrub-interval-randomize-ratio=0" | |
| 204 | for id in $(seq 0 2) ; do | |
| 205 | if [ "$allow_overwrites" = "true" ]; then | |
| 206 | run_osd_bluestore $dir $id $ceph_osd_args || return 1 | |
| 207 | else | |
| 208 | run_osd $dir $id $ceph_osd_args || return 1 | |
| 209 | fi | |
| 210 | done | |
| 211 | wait_for_clean || return 1 | |
| 212 | ||
| 213 | # Create an EC pool | |
| 214 | create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1 | |
| 215 | ||
| 216 | # Put an object | |
| 217 | local payload=ABCDEF | |
| 218 | echo $payload > $dir/ORIGINAL | |
| 219 | rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1 | |
| 220 | ||
| 221 | # Remove the object from one shard physically | |
| 222 | # Restarted osd get $ceph_osd_args passed | |
| 223 | objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1 | |
| 224 | # Wait for auto repair | |
| 225 | local pgid=$(get_pg $poolname SOMETHING) | |
| 226 | wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)" | |
| 227 | wait_for_clean || return 1 | |
| 228 | # Verify - the file should be back | |
| 229 | # Restarted osd get $ceph_osd_args passed | |
| 230 | objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1 | |
| 231 | rados --pool $poolname get SOMETHING $dir/COPY || return 1 | |
| 232 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
| 233 | ||
| 234 | # Tear down | |
| 235 | teardown $dir || return 1 | |
| 236 | } | |
| 237 | ||
| 238 | function TEST_auto_repair_erasure_coded_appends() { | |
| 239 | auto_repair_erasure_coded $1 false | |
| 240 | } | |
| 241 | ||
| 242 | function TEST_auto_repair_erasure_coded_overwrites() { | |
| 243 | auto_repair_erasure_coded $1 true | |
| 244 | } | |
| 245 | ||
| 246 | function corrupt_and_repair_jerasure() { | |
| 247 | local dir=$1 | |
| 248 | local allow_overwrites=$2 | |
| 249 | local poolname=ecpool | |
| 250 | ||
| 251 | setup $dir || return 1 | |
| 252 | run_mon $dir a || return 1 | |
| 253 | run_mgr $dir x || return 1 | |
| 254 | for id in $(seq 0 3) ; do | |
| 255 | if [ "$allow_overwrites" = "true" ]; then | |
| 256 | run_osd_bluestore $dir $id || return 1 | |
| 257 | else | |
| 258 | run_osd $dir $id || return 1 | |
| 259 | fi | |
| 260 | done | |
| 261 | wait_for_clean || return 1 | |
| 262 | ||
| 263 | create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 | |
| 264 | corrupt_and_repair_erasure_coded $dir $poolname || return 1 | |
| 265 | ||
| 266 | teardown $dir || return 1 | |
| 267 | } | |
| 268 | ||
| 269 | function TEST_corrupt_and_repair_jerasure_appends() { | |
| 270 | corrupt_and_repair_jerasure $1 | |
| 271 | } | |
| 272 | ||
| 273 | function TEST_corrupt_and_repair_jerasure_overwrites() { | |
| 274 | corrupt_and_repair_jerasure $1 true | |
| 275 | } | |
| 276 | ||
| 277 | function corrupt_and_repair_lrc() { | |
| 278 | local dir=$1 | |
| 279 | local allow_overwrites=$2 | |
| 280 | local poolname=ecpool | |
| 281 | ||
| 282 | setup $dir || return 1 | |
| 283 | run_mon $dir a || return 1 | |
| 284 | run_mgr $dir x || return 1 | |
| 285 | for id in $(seq 0 9) ; do | |
| 286 | if [ "$allow_overwrites" = "true" ]; then | |
| 287 | run_osd_bluestore $dir $id || return 1 | |
| 288 | else | |
| 289 | run_osd $dir $id || return 1 | |
| 290 | fi | |
| 291 | done | |
| 292 | wait_for_clean || return 1 | |
| 293 | ||
| 294 | create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1 | |
| 295 | corrupt_and_repair_erasure_coded $dir $poolname || return 1 | |
| 296 | ||
| 297 | teardown $dir || return 1 | |
| 298 | } | |
| 299 | ||
| 300 | function TEST_corrupt_and_repair_lrc_appends() { | |
| 301 | corrupt_and_repair_jerasure $1 | |
| 302 | } | |
| 303 | ||
| 304 | function TEST_corrupt_and_repair_lrc_overwrites() { | |
| 305 | corrupt_and_repair_jerasure $1 true | |
| 306 | } | |
| 307 | ||
| 308 | function unfound_erasure_coded() { | |
| 309 | local dir=$1 | |
| 310 | local allow_overwrites=$2 | |
| 311 | local poolname=ecpool | |
| 312 | local payload=ABCDEF | |
| 313 | ||
| 314 | setup $dir || return 1 | |
| 315 | run_mon $dir a || return 1 | |
| 316 | run_mgr $dir x || return 1 | |
| 317 | for id in $(seq 0 3) ; do | |
| 318 | if [ "$allow_overwrites" = "true" ]; then | |
| 319 | run_osd_bluestore $dir $id || return 1 | |
| 320 | else | |
| 321 | run_osd $dir $id || return 1 | |
| 322 | fi | |
| 323 | done | |
| 324 | wait_for_clean || return 1 | |
| 325 | ||
| 326 | create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 | |
| 327 | ||
| 328 | add_something $dir $poolname || return 1 | |
| 329 | ||
| 330 | local primary=$(get_primary $poolname SOMETHING) | |
| 331 | local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//")) | |
| 332 | local not_primary_first=${osds[0]} | |
| 333 | local not_primary_second=${osds[1]} | |
| 334 | local not_primary_third=${osds[2]} | |
| 335 | ||
| 336 | # | |
| 337 | # 1) remove the corresponding file from the OSDs | |
| 338 | # | |
| 339 | pids="" | |
| 340 | run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove | |
| 341 | run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove | |
| 342 | run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove | |
| 343 | wait_background pids | |
| 344 | return_code=$? | |
| 345 | if [ $return_code -ne 0 ]; then return $return_code; fi | |
| 346 | ||
| 347 | # | |
| 348 | # 2) repair the PG | |
| 349 | # | |
| 350 | local pg=$(get_pg $poolname SOMETHING) | |
| 351 | repair $pg | |
| 352 | # | |
| 353 | # 3) check pg state | |
| 354 | # | |
| 355 | # it may take a bit to appear due to mon/mgr asynchrony | |
| 356 | for f in `seq 1 60`; do | |
| 357 | ceph -s | grep "1/1 unfound" && break | |
| 358 | sleep 1 | |
| 359 | done | |
| 360 | ceph -s|grep "4 osds: 4 up, 4 in" || return 1 | |
| 361 | ceph -s|grep "1/1 unfound" || return 1 | |
| 362 | ||
| 363 | teardown $dir || return 1 | |
| 364 | } | |
| 365 | ||
| 366 | function TEST_unfound_erasure_coded_appends() { | |
| 367 | unfound_erasure_coded $1 | |
| 368 | } | |
| 369 | ||
| 370 | function TEST_unfound_erasure_coded_overwrites() { | |
| 371 | unfound_erasure_coded $1 true | |
| 372 | } | |
| 373 | ||
| 374 | # | |
| 375 | # list_missing for EC pool | |
| 376 | # | |
| 377 | function list_missing_erasure_coded() { | |
| 378 | local dir=$1 | |
| 379 | local allow_overwrites=$2 | |
| 380 | local poolname=ecpool | |
| 381 | ||
| 382 | setup $dir || return 1 | |
| 383 | run_mon $dir a || return 1 | |
| 384 | run_mgr $dir x || return 1 | |
| 385 | for id in $(seq 0 2) ; do | |
| 386 | if [ "$allow_overwrites" = "true" ]; then | |
| 387 | run_osd_bluestore $dir $id || return 1 | |
| 388 | else | |
| 389 | run_osd $dir $id || return 1 | |
| 390 | fi | |
| 391 | done | |
| 392 | wait_for_clean || return 1 | |
| 393 | ||
| 394 | create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1 | |
| 395 | ||
| 396 | # Put an object and remove the two shards (including primary) | |
| 397 | add_something $dir $poolname MOBJ0 || return 1 | |
| 398 | local -a osds0=($(get_osds $poolname MOBJ0)) | |
| 399 | ||
| 400 | # Put another object and remove two shards (excluding primary) | |
| 401 | add_something $dir $poolname MOBJ1 || return 1 | |
| 402 | local -a osds1=($(get_osds $poolname MOBJ1)) | |
| 403 | ||
| 404 | # Stop all osd daemons | |
| 405 | for id in $(seq 0 2) ; do | |
| 406 | kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1 | |
| 407 | done | |
| 408 | ||
| 409 | id=${osds0[0]} | |
| 410 | ceph-objectstore-tool --data-path $dir/$id --enable-experimental-unrecoverable-data-corrupting-features=bluestore \ | |
| 411 | MOBJ0 remove || return 1 | |
| 412 | id=${osds0[1]} | |
| 413 | ceph-objectstore-tool --data-path $dir/$id --enable-experimental-unrecoverable-data-corrupting-features=bluestore \ | |
| 414 | MOBJ0 remove || return 1 | |
| 415 | ||
| 416 | id=${osds1[1]} | |
| 417 | ceph-objectstore-tool --data-path $dir/$id --enable-experimental-unrecoverable-data-corrupting-features=bluestore \ | |
| 418 | MOBJ1 remove || return 1 | |
| 419 | id=${osds1[2]} | |
| 420 | ceph-objectstore-tool --data-path $dir/$id --enable-experimental-unrecoverable-data-corrupting-features=bluestore \ | |
| 421 | MOBJ1 remove || return 1 | |
| 422 | ||
| 423 | for id in $(seq 0 2) ; do | |
| 424 | activate_osd $dir $id >&2 || return 1 | |
| 425 | done | |
| 426 | wait_for_clean || return 1 | |
| 427 | ||
| 428 | # Get get - both objects should in the same PG | |
| 429 | local pg=$(get_pg $poolname MOBJ0) | |
| 430 | ||
| 431 | # Repair the PG, which triggers the recovering, | |
| 432 | # and should mark the object as unfound | |
| 433 | repair $pg | |
| 434 | ||
| 435 | for i in $(seq 0 120) ; do | |
| 436 | [ $i -lt 60 ] || return 1 | |
| 437 | matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l) | |
| 438 | [ $matches -eq 2 ] && break | |
| 439 | done | |
| 440 | ||
| 441 | teardown $dir || return 1 | |
| 442 | } | |
| 443 | ||
| 444 | function TEST_list_missing_erasure_coded_appends() { | |
| 445 | list_missing_erasure_coded $1 false | |
| 446 | } | |
| 447 | ||
| 448 | function TEST_list_missing_erasure_coded_overwrites() { | |
| 449 | list_missing_erasure_coded $1 true | |
| 450 | } | |
| 451 | ||
| 452 | # | |
| 453 | # Corrupt one copy of a replicated pool | |
| 454 | # | |
| 455 | function TEST_corrupt_scrub_replicated() { | |
| 456 | local dir=$1 | |
| 457 | local poolname=csr_pool | |
| 458 | local total_objs=15 | |
| 459 | ||
| 460 | setup $dir || return 1 | |
| 461 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
| 462 | run_mgr $dir x || return 1 | |
| 463 | run_osd $dir 0 || return 1 | |
| 464 | run_osd $dir 1 || return 1 | |
| 465 | wait_for_clean || return 1 | |
| 466 | ||
| 467 | ceph osd pool create $poolname 1 1 || return 1 | |
| 468 | wait_for_clean || return 1 | |
| 469 | ||
| 470 | for i in $(seq 1 $total_objs) ; do | |
| 471 | objname=ROBJ${i} | |
| 472 | add_something $dir $poolname $objname || return 1 | |
| 473 | ||
| 474 | rados --pool $poolname setomapheader $objname hdr-$objname || return 1 | |
| 475 | rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1 | |
| 476 | ||
| 477 | # Alternate corruption between osd.0 and osd.1 | |
| 478 | local osd=$(expr $i % 2) | |
| 479 | ||
| 480 | case $i in | |
| 481 | 1) | |
| 482 | # Size (deep scrub data_digest too) | |
| 483 | local payload=UVWXYZZZ | |
| 484 | echo $payload > $dir/CORRUPT | |
| 485 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
| 486 | ;; | |
| 487 | ||
| 488 | 2) | |
| 489 | # digest (deep scrub only) | |
| 490 | local payload=UVWXYZ | |
| 491 | echo $payload > $dir/CORRUPT | |
| 492 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
| 493 | ;; | |
| 494 | ||
| 495 | 3) | |
| 496 | # missing | |
| 497 | objectstore_tool $dir $osd $objname remove || return 1 | |
| 498 | ;; | |
| 499 | ||
| 500 | 4) | |
| 501 | # Modify omap value (deep scrub only) | |
| 502 | objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1 | |
| 503 | ;; | |
| 504 | ||
| 505 | 5) | |
| 506 | # Delete omap key (deep scrub only) | |
| 507 | objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1 | |
| 508 | ;; | |
| 509 | ||
| 510 | 6) | |
| 511 | # Add extra omap key (deep scrub only) | |
| 512 | echo extra > $dir/extra-val | |
| 513 | objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1 | |
| 514 | rm $dir/extra-val | |
| 515 | ;; | |
| 516 | ||
| 517 | 7) | |
| 518 | # Modify omap header (deep scrub only) | |
| 519 | echo -n newheader > $dir/hdr | |
| 520 | objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1 | |
| 521 | rm $dir/hdr | |
| 522 | ;; | |
| 523 | ||
| 524 | 8) | |
| 525 | rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1 | |
| 526 | rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1 | |
| 527 | ||
| 528 | # Break xattrs | |
| 529 | echo -n bad-val > $dir/bad-val | |
| 530 | objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1 | |
| 531 | objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1 | |
| 532 | echo -n val3-$objname > $dir/newval | |
| 533 | objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1 | |
| 534 | rm $dir/bad-val $dir/newval | |
| 535 | ;; | |
| 536 | ||
| 537 | 9) | |
| 538 | objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi | |
| 539 | echo -n D > $dir/change | |
| 540 | rados --pool $poolname put $objname $dir/change | |
| 541 | objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi | |
| 542 | rm $dir/oi $dir/change | |
| 543 | ;; | |
| 544 | ||
| 545 | # ROBJ10 must be handled after digests are re-computed by a deep scrub below | |
| 546 | # ROBJ11 must be handled with config change before deep scrub | |
| 547 | # ROBJ12 must be handled with config change before scrubs | |
| 548 | # ROBJ13 must be handled before scrubs | |
| 549 | ||
| 550 | 14) | |
| 551 | echo -n bad-val > $dir/bad-val | |
| 552 | objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1 | |
| 553 | objectstore_tool $dir 1 $objname rm-attr _ || return 1 | |
| 554 | rm $dir/bad-val | |
| 555 | ;; | |
| 556 | ||
| 557 | 15) | |
| 558 | objectstore_tool $dir $osd $objname rm-attr _ || return 1 | |
| 559 | ||
| 560 | esac | |
| 561 | done | |
| 562 | ||
| 563 | local pg=$(get_pg $poolname ROBJ0) | |
| 564 | ||
| 565 | set_config osd 0 filestore_debug_inject_read_err true || return 1 | |
| 566 | set_config osd 1 filestore_debug_inject_read_err true || return 1 | |
| 567 | CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \ | |
| 568 | injectdataerr $poolname ROBJ11 || return 1 | |
| 569 | CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \ | |
| 570 | injectmdataerr $poolname ROBJ12 || return 1 | |
| 571 | CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \ | |
| 572 | injectmdataerr $poolname ROBJ13 || return 1 | |
| 573 | CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \ | |
| 574 | injectdataerr $poolname ROBJ13 || return 1 | |
| 575 | ||
| 576 | pg_scrub $pg | |
| 577 | ||
| 578 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
| 579 | # Check pg count | |
| 580 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
| 581 | # Check pgid | |
| 582 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
| 583 | ||
| 584 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
| 585 | # Get epoch for repair-get requests | |
| 586 | epoch=$(jq .epoch $dir/json) | |
| 587 | ||
| 588 | jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson | |
| 589 | { | |
| 590 | "inconsistents": [ | |
| 591 | { | |
| 592 | "shards": [ | |
| 593 | { | |
| 594 | "size": 7, | |
| 595 | "errors": [], | |
| 596 | "osd": 0 | |
| 597 | }, | |
| 598 | { | |
| 599 | "size": 9, | |
| 600 | "errors": [ | |
| 601 | "size_mismatch_oi" | |
| 602 | ], | |
| 603 | "osd": 1 | |
| 604 | } | |
| 605 | ], | |
| 606 | "selected_object_info": "2:ce3f1d6a:::ROBJ1:head(15'3 client.4169.0:1 dirty|omap|data_digest s 7 uv 3 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 607 | "union_shard_errors": [ | |
| 608 | "size_mismatch_oi" | |
| 609 | ], | |
| 610 | "errors": [ | |
| 611 | "size_mismatch" | |
| 612 | ], | |
| 613 | "object": { | |
| 614 | "version": 3, | |
| 615 | "snap": "head", | |
| 616 | "locator": "", | |
| 617 | "nspace": "", | |
| 618 | "name": "ROBJ1" | |
| 619 | } | |
| 620 | }, | |
| 621 | { | |
| 622 | "shards": [ | |
| 623 | { | |
| 624 | "errors": [ | |
| 625 | "stat_error" | |
| 626 | ], | |
| 627 | "osd": 0 | |
| 628 | }, | |
| 629 | { | |
| 630 | "size": 7, | |
| 631 | "errors": [], | |
| 632 | "osd": 1 | |
| 633 | } | |
| 634 | ], | |
| 635 | "selected_object_info": "2:bc819597:::ROBJ12:head(110'39 client.4732.0:1 dirty|omap|data_digest s 7 uv 39 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 636 | "union_shard_errors": [ | |
| 637 | "stat_error" | |
| 638 | ], | |
| 639 | "errors": [], | |
| 640 | "object": { | |
| 641 | "version": 39, | |
| 642 | "snap": "head", | |
| 643 | "locator": "", | |
| 644 | "nspace": "", | |
| 645 | "name": "ROBJ12" | |
| 646 | } | |
| 647 | }, | |
| 648 | { | |
| 649 | "shards": [ | |
| 650 | { | |
| 651 | "errors": [ | |
| 652 | "stat_error" | |
| 653 | ], | |
| 654 | "osd": 0 | |
| 655 | }, | |
| 656 | { | |
| 657 | "size": 7, | |
| 658 | "errors": [], | |
| 659 | "osd": 1 | |
| 660 | } | |
| 661 | ], | |
| 662 | "selected_object_info": "2:d60617f9:::ROBJ13:head(112'42 client.4737.0:1 dirty|omap|data_digest s 7 uv 42 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 663 | "union_shard_errors": [ | |
| 664 | "stat_error" | |
| 665 | ], | |
| 666 | "errors": [], | |
| 667 | "object": { | |
| 668 | "version": 42, | |
| 669 | "snap": "head", | |
| 670 | "locator": "", | |
| 671 | "nspace": "", | |
| 672 | "name": "ROBJ13" | |
| 673 | } | |
| 674 | }, | |
| 675 | { | |
| 676 | "shards": [ | |
| 677 | { | |
| 678 | "size": 7, | |
| 679 | "errors": [ | |
| 680 | "oi_attr_corrupted" | |
| 681 | ], | |
| 682 | "osd": 0 | |
| 683 | }, | |
| 684 | { | |
| 685 | "size": 7, | |
| 686 | "errors": [ | |
| 687 | "oi_attr_missing" | |
| 688 | ], | |
| 689 | "osd": 1 | |
| 690 | } | |
| 691 | ], | |
| 692 | "union_shard_errors": [ | |
| 693 | "oi_attr_missing", | |
| 694 | "oi_attr_corrupted" | |
| 695 | ], | |
| 696 | "errors": [], | |
| 697 | "object": { | |
| 698 | "version": 0, | |
| 699 | "snap": "head", | |
| 700 | "locator": "", | |
| 701 | "nspace": "", | |
| 702 | "name": "ROBJ14" | |
| 703 | } | |
| 704 | }, | |
| 705 | { | |
| 706 | "shards": [ | |
| 707 | { | |
| 708 | "attrs": [ | |
| 709 | { | |
| 710 | "Base64": true, | |
| 711 | "value": "", | |
| 712 | "name": "_" | |
| 713 | }, | |
| 714 | { | |
| 715 | "Base64": true, | |
| 716 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 717 | "name": "snapset" | |
| 718 | } | |
| 719 | ], | |
| 720 | "size": 7, | |
| 721 | "errors": [], | |
| 722 | "osd": 0 | |
| 723 | }, | |
| 724 | { | |
| 725 | "attrs": [ | |
| 726 | { | |
| 727 | "Base64": true, | |
| 728 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 729 | "name": "snapset" | |
| 730 | } | |
| 731 | ], | |
| 732 | "size": 7, | |
| 733 | "errors": [ | |
| 734 | "oi_attr_missing" | |
| 735 | ], | |
| 736 | "osd": 1 | |
| 737 | } | |
| 738 | ], | |
| 739 | "selected_object_info": "2:30259878:::ROBJ15:head(127'48 client.4820.0:1 dirty|omap|data_digest s 7 uv 48 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 740 | "union_shard_errors": [ | |
| 741 | "oi_attr_missing" | |
| 742 | ], | |
| 743 | "errors": [ | |
| 744 | "attr_name_mismatch" | |
| 745 | ], | |
| 746 | "object": { | |
| 747 | "version": 48, | |
| 748 | "snap": "head", | |
| 749 | "locator": "", | |
| 750 | "nspace": "", | |
| 751 | "name": "ROBJ15" | |
| 752 | } | |
| 753 | }, | |
| 754 | { | |
| 755 | "shards": [ | |
| 756 | { | |
| 757 | "size": 7, | |
| 758 | "errors": [], | |
| 759 | "osd": 0 | |
| 760 | }, | |
| 761 | { | |
| 762 | "errors": [ | |
| 763 | "missing" | |
| 764 | ], | |
| 765 | "osd": 1 | |
| 766 | } | |
| 767 | ], | |
| 768 | "selected_object_info": "2:f2a5b2a4:::ROBJ3:head(29'9 client.4251.0:1 dirty|omap|data_digest s 7 uv 9 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 769 | "union_shard_errors": [ | |
| 770 | "missing" | |
| 771 | ], | |
| 772 | "errors": [], | |
| 773 | "object": { | |
| 774 | "version": 9, | |
| 775 | "snap": "head", | |
| 776 | "locator": "", | |
| 777 | "nspace": "", | |
| 778 | "name": "ROBJ3" | |
| 779 | } | |
| 780 | }, | |
| 781 | { | |
| 782 | "shards": [ | |
| 783 | { | |
| 784 | "attrs": [ | |
| 785 | { | |
| 786 | "Base64": true, | |
| 787 | "value": "", | |
| 788 | "name": "_" | |
| 789 | }, | |
| 790 | { | |
| 791 | "Base64": false, | |
| 792 | "value": "bad-val", | |
| 793 | "name": "_key1-ROBJ8" | |
| 794 | }, | |
| 795 | { | |
| 796 | "Base64": false, | |
| 797 | "value": "val3-ROBJ8", | |
| 798 | "name": "_key3-ROBJ8" | |
| 799 | }, | |
| 800 | { | |
| 801 | "Base64": true, | |
| 802 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 803 | "name": "snapset" | |
| 804 | } | |
| 805 | ], | |
| 806 | "size": 7, | |
| 807 | "errors": [], | |
| 808 | "osd": 0 | |
| 809 | }, | |
| 810 | { | |
| 811 | "attrs": [ | |
| 812 | { | |
| 813 | "Base64": true, | |
| 814 | "value": "", | |
| 815 | "name": "_" | |
| 816 | }, | |
| 817 | { | |
| 818 | "Base64": false, | |
| 819 | "value": "val1-ROBJ8", | |
| 820 | "name": "_key1-ROBJ8" | |
| 821 | }, | |
| 822 | { | |
| 823 | "Base64": false, | |
| 824 | "value": "val2-ROBJ8", | |
| 825 | "name": "_key2-ROBJ8" | |
| 826 | }, | |
| 827 | { | |
| 828 | "Base64": true, | |
| 829 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 830 | "name": "snapset" | |
| 831 | } | |
| 832 | ], | |
| 833 | "size": 7, | |
| 834 | "errors": [], | |
| 835 | "osd": 1 | |
| 836 | } | |
| 837 | ], | |
| 838 | "selected_object_info": "2:86586531:::ROBJ8:head(70'26 client.4495.0:1 dirty|omap|data_digest s 7 uv 26 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 839 | "union_shard_errors": [], | |
| 840 | "errors": [ | |
| 841 | "attr_value_mismatch", | |
| 842 | "attr_name_mismatch" | |
| 843 | ], | |
| 844 | "object": { | |
| 845 | "version": 26, | |
| 846 | "snap": "head", | |
| 847 | "locator": "", | |
| 848 | "nspace": "", | |
| 849 | "name": "ROBJ8" | |
| 850 | } | |
| 851 | }, | |
| 852 | { | |
| 853 | "shards": [ | |
| 854 | { | |
| 855 | "attrs": [ | |
| 856 | { | |
| 857 | "Base64": true, | |
| 858 | "value": "", | |
| 859 | "name": "_" | |
| 860 | }, | |
| 861 | { | |
| 862 | "Base64": true, | |
| 863 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 864 | "name": "snapset" | |
| 865 | } | |
| 866 | ], | |
| 867 | "object_info": "2:ffdb2004:::ROBJ9:head(94'30 client.4649.0:1 dirty|omap|data_digest s 1 uv 30 dd 2b63260d alloc_hint [0 0 0])", | |
| 868 | "size": 1, | |
| 869 | "errors": [], | |
| 870 | "osd": 0 | |
| 871 | }, | |
| 872 | { | |
| 873 | "attrs": [ | |
| 874 | { | |
| 875 | "Base64": true, | |
| 876 | "value": "", | |
| 877 | "name": "_" | |
| 878 | }, | |
| 879 | { | |
| 880 | "Base64": true, | |
| 881 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 882 | "name": "snapset" | |
| 883 | } | |
| 884 | ], | |
| 885 | "object_info": "2:ffdb2004:::ROBJ9:head(89'29 client.4612.0:1 dirty|omap|data_digest s 7 uv 29 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 886 | "size": 1, | |
| 887 | "errors": [], | |
| 888 | "osd": 1 | |
| 889 | } | |
| 890 | ], | |
| 891 | "selected_object_info": "2:ffdb2004:::ROBJ9:head(94'30 client.4649.0:1 dirty|omap|data_digest s 1 uv 30 dd 2b63260d alloc_hint [0 0 0])", | |
| 892 | "union_shard_errors": [], | |
| 893 | "errors": [ | |
| 894 | "object_info_inconsistency", | |
| 895 | "attr_value_mismatch" | |
| 896 | ], | |
| 897 | "object": { | |
| 898 | "version": 30, | |
| 899 | "snap": "head", | |
| 900 | "locator": "", | |
| 901 | "nspace": "", | |
| 902 | "name": "ROBJ9" | |
| 903 | } | |
| 904 | } | |
| 905 | ], | |
| 906 | "epoch": 0 | |
| 907 | } | |
| 908 | EOF | |
| 909 | ||
| 910 | jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson | |
| 911 | diff -y $termwidth $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 | |
| 912 | if test $getjson = "yes" | |
| 913 | then | |
| 914 | jq '.' $dir/json > save1.json | |
| 915 | fi | |
| 916 | ||
| 917 | if which jsonschema > /dev/null; | |
| 918 | then | |
| 919 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
| 920 | fi | |
| 921 | ||
| 922 | # Compute an old omap digest and save oi | |
| 923 | CEPH_ARGS='' ceph daemon $dir//ceph-osd.0.asok \ | |
| 924 | config set osd_deep_scrub_update_digest_min_age 0 | |
| 925 | CEPH_ARGS='' ceph daemon $dir//ceph-osd.1.asok \ | |
| 926 | config set osd_deep_scrub_update_digest_min_age 0 | |
| 927 | pg_deep_scrub $pg | |
| 928 | ||
| 929 | objname=ROBJ9 | |
| 930 | # Change data and size again because digest was recomputed | |
| 931 | echo -n ZZZ > $dir/change | |
| 932 | rados --pool $poolname put $objname $dir/change | |
| 933 | # Set one to an even older value | |
| 934 | objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi | |
| 935 | rm $dir/oi $dir/change | |
| 936 | ||
| 937 | objname=ROBJ10 | |
| 938 | objectstore_tool $dir 1 $objname get-attr _ > $dir/oi | |
| 939 | rados --pool $poolname setomapval $objname key2-$objname val2-$objname | |
| 940 | objectstore_tool $dir 0 $objname set-attr _ $dir/oi | |
| 941 | objectstore_tool $dir 1 $objname set-attr _ $dir/oi | |
| 942 | rm $dir/oi | |
| 943 | ||
| 944 | set_config osd 0 filestore_debug_inject_read_err true || return 1 | |
| 945 | set_config osd 1 filestore_debug_inject_read_err true || return 1 | |
| 946 | CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \ | |
| 947 | injectdataerr $poolname ROBJ11 || return 1 | |
| 948 | CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \ | |
| 949 | injectmdataerr $poolname ROBJ12 || return 1 | |
| 950 | CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \ | |
| 951 | injectmdataerr $poolname ROBJ13 || return 1 | |
| 952 | CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \ | |
| 953 | injectdataerr $poolname ROBJ13 || return 1 | |
| 954 | pg_deep_scrub $pg | |
| 955 | ||
| 956 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
| 957 | # Check pg count | |
| 958 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
| 959 | # Check pgid | |
| 960 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
| 961 | ||
| 962 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
| 963 | # Get epoch for repair-get requests | |
| 964 | epoch=$(jq .epoch $dir/json) | |
| 965 | ||
| 966 | jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson | |
| 967 | { | |
| 968 | "inconsistents": [ | |
| 969 | { | |
| 970 | "shards": [ | |
| 971 | { | |
| 972 | "data_digest": "0x2ddbf8f5", | |
| 973 | "omap_digest": "0xf5fba2c6", | |
| 974 | "size": 7, | |
| 975 | "errors": [], | |
| 976 | "osd": 0 | |
| 977 | }, | |
| 978 | { | |
| 979 | "data_digest": "0x2d4a11c2", | |
| 980 | "omap_digest": "0xf5fba2c6", | |
| 981 | "size": 9, | |
| 982 | "errors": [ | |
| 983 | "data_digest_mismatch_oi", | |
| 984 | "size_mismatch_oi" | |
| 985 | ], | |
| 986 | "osd": 1 | |
| 987 | } | |
| 988 | ], | |
| 989 | "selected_object_info": "2:ce3f1d6a:::ROBJ1:head(15'3 client.4171.0:1 dirty|omap|data_digest s 7 uv 3 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 990 | "union_shard_errors": [ | |
| 991 | "data_digest_mismatch_oi", | |
| 992 | "size_mismatch_oi" | |
| 993 | ], | |
| 994 | "errors": [ | |
| 995 | "data_digest_mismatch", | |
| 996 | "size_mismatch" | |
| 997 | ], | |
| 998 | "object": { | |
| 999 | "version": 3, | |
| 1000 | "snap": "head", | |
| 1001 | "locator": "", | |
| 1002 | "nspace": "", | |
| 1003 | "name": "ROBJ1" | |
| 1004 | } | |
| 1005 | }, | |
| 1006 | { | |
| 1007 | "shards": [ | |
| 1008 | { | |
| 1009 | "data_digest": "0x2ddbf8f5", | |
| 1010 | "omap_digest": "0xa8dd5adc", | |
| 1011 | "size": 7, | |
| 1012 | "errors": [ | |
| 1013 | "omap_digest_mismatch_oi" | |
| 1014 | ], | |
| 1015 | "osd": 0 | |
| 1016 | }, | |
| 1017 | { | |
| 1018 | "data_digest": "0x2ddbf8f5", | |
| 1019 | "omap_digest": "0xa8dd5adc", | |
| 1020 | "size": 7, | |
| 1021 | "errors": [ | |
| 1022 | "omap_digest_mismatch_oi" | |
| 1023 | ], | |
| 1024 | "osd": 1 | |
| 1025 | } | |
| 1026 | ], | |
| 1027 | "selected_object_info": "2:b1f19cbd:::ROBJ10:head(136'51 osd.0.0:8 dirty|omap|data_digest|omap_digest s 7 uv 33 dd 2ddbf8f5 od c2025a24 alloc_hint [0 0 0])", | |
| 1028 | "union_shard_errors": [ | |
| 1029 | "omap_digest_mismatch_oi" | |
| 1030 | ], | |
| 1031 | "errors": [], | |
| 1032 | "object": { | |
| 1033 | "version": 33, | |
| 1034 | "snap": "head", | |
| 1035 | "locator": "", | |
| 1036 | "nspace": "", | |
| 1037 | "name": "ROBJ10" | |
| 1038 | } | |
| 1039 | }, | |
| 1040 | { | |
| 1041 | "shards": [ | |
| 1042 | { | |
| 1043 | "data_digest": "0x2ddbf8f5", | |
| 1044 | "omap_digest": "0xa03cef03", | |
| 1045 | "size": 7, | |
| 1046 | "errors": [], | |
| 1047 | "osd": 0 | |
| 1048 | }, | |
| 1049 | { | |
| 1050 | "size": 7, | |
| 1051 | "errors": [ | |
| 1052 | "read_error" | |
| 1053 | ], | |
| 1054 | "osd": 1 | |
| 1055 | } | |
| 1056 | ], | |
| 1057 | "selected_object_info": "2:87abbf36:::ROBJ11:head(105'36 client.4699.0:1 dirty|omap|data_digest s 7 uv 36 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 1058 | "union_shard_errors": [ | |
| 1059 | "read_error" | |
| 1060 | ], | |
| 1061 | "errors": [], | |
| 1062 | "object": { | |
| 1063 | "version": 36, | |
| 1064 | "snap": "head", | |
| 1065 | "locator": "", | |
| 1066 | "nspace": "", | |
| 1067 | "name": "ROBJ11" | |
| 1068 | } | |
| 1069 | }, | |
| 1070 | { | |
| 1071 | "shards": [ | |
| 1072 | { | |
| 1073 | "errors": [ | |
| 1074 | "stat_error" | |
| 1075 | ], | |
| 1076 | "osd": 0 | |
| 1077 | }, | |
| 1078 | { | |
| 1079 | "data_digest": "0x2ddbf8f5", | |
| 1080 | "omap_digest": "0x067f306a", | |
| 1081 | "size": 7, | |
| 1082 | "errors": [], | |
| 1083 | "osd": 1 | |
| 1084 | } | |
| 1085 | ], | |
| 1086 | "selected_object_info": "2:bc819597:::ROBJ12:head(107'39 client.4704.0:1 dirty|omap|data_digest s 7 uv 39 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 1087 | "union_shard_errors": [ | |
| 1088 | "stat_error" | |
| 1089 | ], | |
| 1090 | "errors": [], | |
| 1091 | "object": { | |
| 1092 | "version": 39, | |
| 1093 | "snap": "head", | |
| 1094 | "locator": "", | |
| 1095 | "nspace": "", | |
| 1096 | "name": "ROBJ12" | |
| 1097 | } | |
| 1098 | }, | |
| 1099 | { | |
| 1100 | "shards": [ | |
| 1101 | { | |
| 1102 | "errors": [ | |
| 1103 | "stat_error" | |
| 1104 | ], | |
| 1105 | "osd": 0 | |
| 1106 | }, | |
| 1107 | { | |
| 1108 | "size": 7, | |
| 1109 | "errors": [ | |
| 1110 | "read_error" | |
| 1111 | ], | |
| 1112 | "osd": 1 | |
| 1113 | } | |
| 1114 | ], | |
| 1115 | "union_shard_errors": [ | |
| 1116 | "stat_error", | |
| 1117 | "read_error" | |
| 1118 | ], | |
| 1119 | "errors": [], | |
| 1120 | "object": { | |
| 1121 | "version": 0, | |
| 1122 | "snap": "head", | |
| 1123 | "locator": "", | |
| 1124 | "nspace": "", | |
| 1125 | "name": "ROBJ13" | |
| 1126 | } | |
| 1127 | }, | |
| 1128 | { | |
| 1129 | "shards": [ | |
| 1130 | { | |
| 1131 | "data_digest": "0x2ddbf8f5", | |
| 1132 | "omap_digest": "0x4f14f849", | |
| 1133 | "size": 7, | |
| 1134 | "errors": [ | |
| 1135 | "oi_attr_corrupted" | |
| 1136 | ], | |
| 1137 | "osd": 0 | |
| 1138 | }, | |
| 1139 | { | |
| 1140 | "data_digest": "0x2ddbf8f5", | |
| 1141 | "omap_digest": "0x4f14f849", | |
| 1142 | "size": 7, | |
| 1143 | "errors": [ | |
| 1144 | "oi_attr_missing" | |
| 1145 | ], | |
| 1146 | "osd": 1 | |
| 1147 | } | |
| 1148 | ], | |
| 1149 | "union_shard_errors": [ | |
| 1150 | "oi_attr_missing", | |
| 1151 | "oi_attr_corrupted" | |
| 1152 | ], | |
| 1153 | "errors": [], | |
| 1154 | "object": { | |
| 1155 | "version": 0, | |
| 1156 | "snap": "head", | |
| 1157 | "locator": "", | |
| 1158 | "nspace": "", | |
| 1159 | "name": "ROBJ14" | |
| 1160 | } | |
| 1161 | }, | |
| 1162 | { | |
| 1163 | "shards": [ | |
| 1164 | { | |
| 1165 | "attrs": [ | |
| 1166 | { | |
| 1167 | "Base64": true, | |
| 1168 | "value": "", | |
| 1169 | "name": "_" | |
| 1170 | }, | |
| 1171 | { | |
| 1172 | "Base64": true, | |
| 1173 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 1174 | "name": "snapset" | |
| 1175 | } | |
| 1176 | ], | |
| 1177 | "data_digest": "0x2ddbf8f5", | |
| 1178 | "omap_digest": "0x2d2a4d6e", | |
| 1179 | "size": 7, | |
| 1180 | "errors": [], | |
| 1181 | "osd": 0 | |
| 1182 | }, | |
| 1183 | { | |
| 1184 | "attrs": [ | |
| 1185 | { | |
| 1186 | "Base64": true, | |
| 1187 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 1188 | "name": "snapset" | |
| 1189 | } | |
| 1190 | ], | |
| 1191 | "data_digest": "0x2ddbf8f5", | |
| 1192 | "omap_digest": "0x2d2a4d6e", | |
| 1193 | "size": 7, | |
| 1194 | "errors": [ | |
| 1195 | "oi_attr_missing" | |
| 1196 | ], | |
| 1197 | "osd": 1 | |
| 1198 | } | |
| 1199 | ], | |
| 1200 | "selected_object_info": "2:30259878:::ROBJ15:head(124'48 client.4792.0:1 dirty|omap|data_digest s 7 uv 48 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 1201 | "union_shard_errors": [ | |
| 1202 | "oi_attr_missing" | |
| 1203 | ], | |
| 1204 | "errors": [ | |
| 1205 | "attr_name_mismatch" | |
| 1206 | ], | |
| 1207 | "object": { | |
| 1208 | "version": 48, | |
| 1209 | "snap": "head", | |
| 1210 | "locator": "", | |
| 1211 | "nspace": "", | |
| 1212 | "name": "ROBJ15" | |
| 1213 | } | |
| 1214 | }, | |
| 1215 | { | |
| 1216 | "shards": [ | |
| 1217 | { | |
| 1218 | "data_digest": "0x578a4830", | |
| 1219 | "omap_digest": "0xf8e11918", | |
| 1220 | "size": 7, | |
| 1221 | "errors": [ | |
| 1222 | "data_digest_mismatch_oi" | |
| 1223 | ], | |
| 1224 | "osd": 0 | |
| 1225 | }, | |
| 1226 | { | |
| 1227 | "data_digest": "0x2ddbf8f5", | |
| 1228 | "omap_digest": "0xf8e11918", | |
| 1229 | "size": 7, | |
| 1230 | "errors": [], | |
| 1231 | "osd": 1 | |
| 1232 | } | |
| 1233 | ], | |
| 1234 | "selected_object_info": "2:e97ce31e:::ROBJ2:head(22'6 client.4214.0:1 dirty|omap|data_digest s 7 uv 6 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 1235 | "union_shard_errors": [ | |
| 1236 | "data_digest_mismatch_oi" | |
| 1237 | ], | |
| 1238 | "errors": [ | |
| 1239 | "data_digest_mismatch" | |
| 1240 | ], | |
| 1241 | "object": { | |
| 1242 | "version": 6, | |
| 1243 | "snap": "head", | |
| 1244 | "locator": "", | |
| 1245 | "nspace": "", | |
| 1246 | "name": "ROBJ2" | |
| 1247 | } | |
| 1248 | }, | |
| 1249 | { | |
| 1250 | "shards": [ | |
| 1251 | { | |
| 1252 | "data_digest": "0x2ddbf8f5", | |
| 1253 | "omap_digest": "0x00b35dfd", | |
| 1254 | "size": 7, | |
| 1255 | "errors": [], | |
| 1256 | "osd": 0 | |
| 1257 | }, | |
| 1258 | { | |
| 1259 | "errors": [ | |
| 1260 | "missing" | |
| 1261 | ], | |
| 1262 | "osd": 1 | |
| 1263 | } | |
| 1264 | ], | |
| 1265 | "selected_object_info": "2:f2a5b2a4:::ROBJ3:head(29'9 client.4255.0:1 dirty|omap|data_digest s 7 uv 9 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 1266 | "union_shard_errors": [ | |
| 1267 | "missing" | |
| 1268 | ], | |
| 1269 | "errors": [], | |
| 1270 | "object": { | |
| 1271 | "version": 9, | |
| 1272 | "snap": "head", | |
| 1273 | "locator": "", | |
| 1274 | "nspace": "", | |
| 1275 | "name": "ROBJ3" | |
| 1276 | } | |
| 1277 | }, | |
| 1278 | { | |
| 1279 | "shards": [ | |
| 1280 | { | |
| 1281 | "data_digest": "0x2ddbf8f5", | |
| 1282 | "omap_digest": "0xd7178dfe", | |
| 1283 | "size": 7, | |
| 1284 | "errors": [], | |
| 1285 | "osd": 0 | |
| 1286 | }, | |
| 1287 | { | |
| 1288 | "data_digest": "0x2ddbf8f5", | |
| 1289 | "omap_digest": "0xe2d46ea4", | |
| 1290 | "size": 7, | |
| 1291 | "errors": [ | |
| 1292 | "omap_digest_mismatch_oi" | |
| 1293 | ], | |
| 1294 | "osd": 1 | |
| 1295 | } | |
| 1296 | ], | |
| 1297 | "selected_object_info": "2:f4981d31:::ROBJ4:head(136'52 osd.0.0:9 dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od d7178dfe alloc_hint [0 0 0])", | |
| 1298 | "union_shard_errors": [ | |
| 1299 | "omap_digest_mismatch_oi" | |
| 1300 | ], | |
| 1301 | "errors": [ | |
| 1302 | "omap_digest_mismatch" | |
| 1303 | ], | |
| 1304 | "object": { | |
| 1305 | "version": 12, | |
| 1306 | "snap": "head", | |
| 1307 | "locator": "", | |
| 1308 | "nspace": "", | |
| 1309 | "name": "ROBJ4" | |
| 1310 | } | |
| 1311 | }, | |
| 1312 | { | |
| 1313 | "shards": [ | |
| 1314 | { | |
| 1315 | "data_digest": "0x2ddbf8f5", | |
| 1316 | "omap_digest": "0x1a862a41", | |
| 1317 | "size": 7, | |
| 1318 | "errors": [], | |
| 1319 | "osd": 0 | |
| 1320 | }, | |
| 1321 | { | |
| 1322 | "data_digest": "0x2ddbf8f5", | |
| 1323 | "omap_digest": "0x06cac8f6", | |
| 1324 | "size": 7, | |
| 1325 | "errors": [ | |
| 1326 | "omap_digest_mismatch_oi" | |
| 1327 | ], | |
| 1328 | "osd": 1 | |
| 1329 | } | |
| 1330 | ], | |
| 1331 | "selected_object_info": "2:f4bfd4d1:::ROBJ5:head(136'53 osd.0.0:10 dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [0 0 0])", | |
| 1332 | "union_shard_errors": [ | |
| 1333 | "omap_digest_mismatch_oi" | |
| 1334 | ], | |
| 1335 | "errors": [ | |
| 1336 | "omap_digest_mismatch" | |
| 1337 | ], | |
| 1338 | "object": { | |
| 1339 | "version": 15, | |
| 1340 | "snap": "head", | |
| 1341 | "locator": "", | |
| 1342 | "nspace": "", | |
| 1343 | "name": "ROBJ5" | |
| 1344 | } | |
| 1345 | }, | |
| 1346 | { | |
| 1347 | "shards": [ | |
| 1348 | { | |
| 1349 | "data_digest": "0x2ddbf8f5", | |
| 1350 | "omap_digest": "0x689ee887", | |
| 1351 | "size": 7, | |
| 1352 | "errors": [], | |
| 1353 | "osd": 0 | |
| 1354 | }, | |
| 1355 | { | |
| 1356 | "data_digest": "0x2ddbf8f5", | |
| 1357 | "omap_digest": "0x179c919f", | |
| 1358 | "size": 7, | |
| 1359 | "errors": [ | |
| 1360 | "omap_digest_mismatch_oi" | |
| 1361 | ], | |
| 1362 | "osd": 1 | |
| 1363 | } | |
| 1364 | ], | |
| 1365 | "selected_object_info": "2:a53c12e8:::ROBJ6:head(136'50 osd.0.0:7 dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 689ee887 alloc_hint [0 0 0])", | |
| 1366 | "union_shard_errors": [ | |
| 1367 | "omap_digest_mismatch_oi" | |
| 1368 | ], | |
| 1369 | "errors": [ | |
| 1370 | "omap_digest_mismatch" | |
| 1371 | ], | |
| 1372 | "object": { | |
| 1373 | "version": 18, | |
| 1374 | "snap": "head", | |
| 1375 | "locator": "", | |
| 1376 | "nspace": "", | |
| 1377 | "name": "ROBJ6" | |
| 1378 | } | |
| 1379 | }, | |
| 1380 | { | |
| 1381 | "shards": [ | |
| 1382 | { | |
| 1383 | "data_digest": "0x2ddbf8f5", | |
| 1384 | "omap_digest": "0xefced57a", | |
| 1385 | "size": 7, | |
| 1386 | "errors": [], | |
| 1387 | "osd": 0 | |
| 1388 | }, | |
| 1389 | { | |
| 1390 | "data_digest": "0x2ddbf8f5", | |
| 1391 | "omap_digest": "0x6a73cc07", | |
| 1392 | "size": 7, | |
| 1393 | "errors": [ | |
| 1394 | "omap_digest_mismatch_oi" | |
| 1395 | ], | |
| 1396 | "osd": 1 | |
| 1397 | } | |
| 1398 | ], | |
| 1399 | "selected_object_info": "2:8b55fa4b:::ROBJ7:head(123'50 osd.0.0:7 dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [0 0 0])", | |
| 1400 | "union_shard_errors": [ | |
| 1401 | "omap_digest_mismatch_oi" | |
| 1402 | ], | |
| 1403 | "errors": [ | |
| 1404 | "omap_digest_mismatch" | |
| 1405 | ], | |
| 1406 | "object": { | |
| 1407 | "version": 21, | |
| 1408 | "snap": "head", | |
| 1409 | "locator": "", | |
| 1410 | "nspace": "", | |
| 1411 | "name": "ROBJ7" | |
| 1412 | } | |
| 1413 | }, | |
| 1414 | { | |
| 1415 | "shards": [ | |
| 1416 | { | |
| 1417 | "attrs": [ | |
| 1418 | { | |
| 1419 | "Base64": true, | |
| 1420 | "value": "", | |
| 1421 | "name": "_" | |
| 1422 | }, | |
| 1423 | { | |
| 1424 | "Base64": false, | |
| 1425 | "value": "bad-val", | |
| 1426 | "name": "_key1-ROBJ8" | |
| 1427 | }, | |
| 1428 | { | |
| 1429 | "Base64": false, | |
| 1430 | "value": "val3-ROBJ8", | |
| 1431 | "name": "_key3-ROBJ8" | |
| 1432 | }, | |
| 1433 | { | |
| 1434 | "Base64": true, | |
| 1435 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 1436 | "name": "snapset" | |
| 1437 | } | |
| 1438 | ], | |
| 1439 | "data_digest": "0x2ddbf8f5", | |
| 1440 | "omap_digest": "0xd6be81dc", | |
| 1441 | "size": 7, | |
| 1442 | "errors": [], | |
| 1443 | "osd": 0 | |
| 1444 | }, | |
| 1445 | { | |
| 1446 | "attrs": [ | |
| 1447 | { | |
| 1448 | "Base64": true, | |
| 1449 | "value": "", | |
| 1450 | "name": "_" | |
| 1451 | }, | |
| 1452 | { | |
| 1453 | "Base64": false, | |
| 1454 | "value": "val1-ROBJ8", | |
| 1455 | "name": "_key1-ROBJ8" | |
| 1456 | }, | |
| 1457 | { | |
| 1458 | "Base64": false, | |
| 1459 | "value": "val2-ROBJ8", | |
| 1460 | "name": "_key2-ROBJ8" | |
| 1461 | }, | |
| 1462 | { | |
| 1463 | "Base64": true, | |
| 1464 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 1465 | "name": "snapset" | |
| 1466 | } | |
| 1467 | ], | |
| 1468 | "data_digest": "0x2ddbf8f5", | |
| 1469 | "omap_digest": "0xd6be81dc", | |
| 1470 | "size": 7, | |
| 1471 | "errors": [], | |
| 1472 | "osd": 1 | |
| 1473 | } | |
| 1474 | ], | |
| 1475 | "selected_object_info": "2:86586531:::ROBJ8:head(136'49 osd.0.0:6 dirty|omap|data_digest|omap_digest s 7 uv 26 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])", | |
| 1476 | "union_shard_errors": [], | |
| 1477 | "errors": [ | |
| 1478 | "attr_value_mismatch", | |
| 1479 | "attr_name_mismatch" | |
| 1480 | ], | |
| 1481 | "object": { | |
| 1482 | "version": 26, | |
| 1483 | "snap": "head", | |
| 1484 | "locator": "", | |
| 1485 | "nspace": "", | |
| 1486 | "name": "ROBJ8" | |
| 1487 | } | |
| 1488 | }, | |
| 1489 | { | |
| 1490 | "shards": [ | |
| 1491 | { | |
| 1492 | "attrs": [ | |
| 1493 | { | |
| 1494 | "Base64": true, | |
| 1495 | "value": "", | |
| 1496 | "name": "_" | |
| 1497 | }, | |
| 1498 | { | |
| 1499 | "Base64": true, | |
| 1500 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 1501 | "name": "snapset" | |
| 1502 | } | |
| 1503 | ], | |
| 1504 | "object_info": "2:ffdb2004:::ROBJ9:head(90'29 client.4615.0:1 dirty|omap|data_digest s 7 uv 29 dd 2ddbf8f5 alloc_hint [0 0 0])", | |
| 1505 | "data_digest": "0x1f26fb26", | |
| 1506 | "omap_digest": "0x2eecc539", | |
| 1507 | "size": 3, | |
| 1508 | "errors": [], | |
| 1509 | "osd": 0 | |
| 1510 | }, | |
| 1511 | { | |
| 1512 | "attrs": [ | |
| 1513 | { | |
| 1514 | "Base64": true, | |
| 1515 | "value": "", | |
| 1516 | "name": "_" | |
| 1517 | }, | |
| 1518 | { | |
| 1519 | "Base64": true, | |
| 1520 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 1521 | "name": "snapset" | |
| 1522 | } | |
| 1523 | ], | |
| 1524 | "object_info": "2:ffdb2004:::ROBJ9:head(123'56 client.4891.0:1 dirty|omap|data_digest|omap_digest s 3 uv 56 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])", | |
| 1525 | "data_digest": "0x1f26fb26", | |
| 1526 | "omap_digest": "0x2eecc539", | |
| 1527 | "size": 3, | |
| 1528 | "errors": [], | |
| 1529 | "osd": 1 | |
| 1530 | } | |
| 1531 | ], | |
| 1532 | "selected_object_info": "2:ffdb2004:::ROBJ9:head(123'56 client.4891.0:1 dirty|omap|data_digest|omap_digest s 3 uv 56 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])", | |
| 1533 | "union_shard_errors": [], | |
| 1534 | "errors": [ | |
| 1535 | "object_info_inconsistency", | |
| 1536 | "attr_value_mismatch" | |
| 1537 | ], | |
| 1538 | "object": { | |
| 1539 | "version": 56, | |
| 1540 | "snap": "head", | |
| 1541 | "locator": "", | |
| 1542 | "nspace": "", | |
| 1543 | "name": "ROBJ9" | |
| 1544 | } | |
| 1545 | } | |
| 1546 | ], | |
| 1547 | "epoch": 0 | |
| 1548 | } | |
| 1549 | EOF | |
| 1550 | ||
| 1551 | jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson | |
| 1552 | diff -y $termwidth $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 | |
| 1553 | if test $getjson = "yes" | |
| 1554 | then | |
| 1555 | jq '.' $dir/json > save2.json | |
| 1556 | fi | |
| 1557 | ||
| 1558 | if which jsonschema > /dev/null; | |
| 1559 | then | |
| 1560 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
| 1561 | fi | |
| 1562 | ||
| 1563 | rados rmpool $poolname $poolname --yes-i-really-really-mean-it | |
| 1564 | teardown $dir || return 1 | |
| 1565 | } | |
| 1566 | ||
| 1567 | ||
| 1568 | # | |
| 1569 | # Test scrub errors for an erasure coded pool | |
| 1570 | # | |
| 1571 | function corrupt_scrub_erasure() { | |
| 1572 | local dir=$1 | |
| 1573 | local allow_overwrites=$2 | |
| 1574 | local poolname=ecpool | |
| 1575 | local total_objs=5 | |
| 1576 | ||
| 1577 | setup $dir || return 1 | |
| 1578 | run_mon $dir a || return 1 | |
| 1579 | run_mgr $dir x || return 1 | |
| 1580 | for id in $(seq 0 2) ; do | |
| 1581 | if [ "$allow_overwrites" = "true" ]; then | |
| 1582 | run_osd_bluestore $dir $id || return 1 | |
| 1583 | else | |
| 1584 | run_osd $dir $id || return 1 | |
| 1585 | fi | |
| 1586 | done | |
| 1587 | wait_for_clean || return 1 | |
| 1588 | ||
| 1589 | create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1 | |
| 1590 | ||
| 1591 | for i in $(seq 1 $total_objs) ; do | |
| 1592 | objname=EOBJ${i} | |
| 1593 | add_something $dir $poolname $objname || return 1 | |
| 1594 | ||
| 1595 | local osd=$(expr $i % 2) | |
| 1596 | ||
| 1597 | case $i in | |
| 1598 | 1) | |
| 1599 | # Size (deep scrub data_digest too) | |
| 1600 | local payload=UVWXYZZZ | |
| 1601 | echo $payload > $dir/CORRUPT | |
| 1602 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
| 1603 | ;; | |
| 1604 | ||
| 1605 | 2) | |
| 1606 | # Corrupt EC shard | |
| 1607 | dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1 | |
| 1608 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
| 1609 | ;; | |
| 1610 | ||
| 1611 | 3) | |
| 1612 | # missing | |
| 1613 | objectstore_tool $dir $osd $objname remove || return 1 | |
| 1614 | ;; | |
| 1615 | ||
| 1616 | 4) | |
| 1617 | rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1 | |
| 1618 | rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1 | |
| 1619 | ||
| 1620 | # Break xattrs | |
| 1621 | echo -n bad-val > $dir/bad-val | |
| 1622 | objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1 | |
| 1623 | objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1 | |
| 1624 | echo -n val3-$objname > $dir/newval | |
| 1625 | objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1 | |
| 1626 | rm $dir/bad-val $dir/newval | |
| 1627 | ;; | |
| 1628 | ||
| 1629 | 5) | |
| 1630 | # Corrupt EC shard | |
| 1631 | dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2 | |
| 1632 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
| 1633 | ;; | |
| 1634 | ||
| 1635 | esac | |
| 1636 | done | |
| 1637 | ||
| 1638 | local pg=$(get_pg $poolname EOBJ0) | |
| 1639 | ||
| 1640 | pg_scrub $pg | |
| 1641 | ||
| 1642 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
| 1643 | # Check pg count | |
| 1644 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
| 1645 | # Check pgid | |
| 1646 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
| 1647 | ||
| 1648 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
| 1649 | # Get epoch for repair-get requests | |
| 1650 | epoch=$(jq .epoch $dir/json) | |
| 1651 | ||
| 1652 | jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson | |
| 1653 | { | |
| 1654 | "inconsistents": [ | |
| 1655 | { | |
| 1656 | "shards": [ | |
| 1657 | { | |
| 1658 | "size": 2048, | |
| 1659 | "errors": [], | |
| 1660 | "shard": 2, | |
| 1661 | "osd": 0 | |
| 1662 | }, | |
| 1663 | { | |
| 1664 | "size": 9, | |
| 1665 | "errors": [ | |
| 1666 | "size_mismatch_oi" | |
| 1667 | ], | |
| 1668 | "shard": 1, | |
| 1669 | "osd": 1 | |
| 1670 | }, | |
| 1671 | { | |
| 1672 | "size": 2048, | |
| 1673 | "errors": [], | |
| 1674 | "shard": 0, | |
| 1675 | "osd": 2 | |
| 1676 | } | |
| 1677 | ], | |
| 1678 | "selected_object_info": "2:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", | |
| 1679 | "union_shard_errors": [ | |
| 1680 | "size_mismatch_oi" | |
| 1681 | ], | |
| 1682 | "errors": [ | |
| 1683 | "size_mismatch" | |
| 1684 | ], | |
| 1685 | "object": { | |
| 1686 | "version": 1, | |
| 1687 | "snap": "head", | |
| 1688 | "locator": "", | |
| 1689 | "nspace": "", | |
| 1690 | "name": "EOBJ1" | |
| 1691 | } | |
| 1692 | }, | |
| 1693 | { | |
| 1694 | "shards": [ | |
| 1695 | { | |
| 1696 | "size": 2048, | |
| 1697 | "errors": [], | |
| 1698 | "shard": 2, | |
| 1699 | "osd": 0 | |
| 1700 | }, | |
| 1701 | { | |
| 1702 | "errors": [ | |
| 1703 | "missing" | |
| 1704 | ], | |
| 1705 | "shard": 1, | |
| 1706 | "osd": 1 | |
| 1707 | }, | |
| 1708 | { | |
| 1709 | "size": 2048, | |
| 1710 | "errors": [], | |
| 1711 | "shard": 0, | |
| 1712 | "osd": 2 | |
| 1713 | } | |
| 1714 | ], | |
| 1715 | "selected_object_info": "2:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", | |
| 1716 | "union_shard_errors": [ | |
| 1717 | "missing" | |
| 1718 | ], | |
| 1719 | "errors": [], | |
| 1720 | "object": { | |
| 1721 | "version": 3, | |
| 1722 | "snap": "head", | |
| 1723 | "locator": "", | |
| 1724 | "nspace": "", | |
| 1725 | "name": "EOBJ3" | |
| 1726 | } | |
| 1727 | }, | |
| 1728 | { | |
| 1729 | "shards": [ | |
| 1730 | { | |
| 1731 | "attrs": [ | |
| 1732 | { | |
| 1733 | "Base64": true, | |
| 1734 | "value": "", | |
| 1735 | "name": "_" | |
| 1736 | }, | |
| 1737 | { | |
| 1738 | "Base64": false, | |
| 1739 | "value": "bad-val", | |
| 1740 | "name": "_key1-EOBJ4" | |
| 1741 | }, | |
| 1742 | { | |
| 1743 | "Base64": false, | |
| 1744 | "value": "val3-EOBJ4", | |
| 1745 | "name": "_key3-EOBJ4" | |
| 1746 | }, | |
| 1747 | { | |
| 1748 | "Base64": true, | |
| 1749 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
| 1750 | "name": "hinfo_key" | |
| 1751 | }, | |
| 1752 | { | |
| 1753 | "Base64": true, | |
| 1754 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 1755 | "name": "snapset" | |
| 1756 | } | |
| 1757 | ], | |
| 1758 | "size": 2048, | |
| 1759 | "errors": [], | |
| 1760 | "shard": 2, | |
| 1761 | "osd": 0 | |
| 1762 | }, | |
| 1763 | { | |
| 1764 | "attrs": [ | |
| 1765 | { | |
| 1766 | "Base64": true, | |
| 1767 | "value": "", | |
| 1768 | "name": "_" | |
| 1769 | }, | |
| 1770 | { | |
| 1771 | "Base64": false, | |
| 1772 | "value": "val1-EOBJ4", | |
| 1773 | "name": "_key1-EOBJ4" | |
| 1774 | }, | |
| 1775 | { | |
| 1776 | "Base64": false, | |
| 1777 | "value": "val2-EOBJ4", | |
| 1778 | "name": "_key2-EOBJ4" | |
| 1779 | }, | |
| 1780 | { | |
| 1781 | "Base64": true, | |
| 1782 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
| 1783 | "name": "hinfo_key" | |
| 1784 | }, | |
| 1785 | { | |
| 1786 | "Base64": true, | |
| 1787 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 1788 | "name": "snapset" | |
| 1789 | } | |
| 1790 | ], | |
| 1791 | "size": 2048, | |
| 1792 | "errors": [], | |
| 1793 | "shard": 1, | |
| 1794 | "osd": 1 | |
| 1795 | }, | |
| 1796 | { | |
| 1797 | "attrs": [ | |
| 1798 | { | |
| 1799 | "Base64": true, | |
| 1800 | "value": "", | |
| 1801 | "name": "_" | |
| 1802 | }, | |
| 1803 | { | |
| 1804 | "Base64": false, | |
| 1805 | "value": "val1-EOBJ4", | |
| 1806 | "name": "_key1-EOBJ4" | |
| 1807 | }, | |
| 1808 | { | |
| 1809 | "Base64": false, | |
| 1810 | "value": "val2-EOBJ4", | |
| 1811 | "name": "_key2-EOBJ4" | |
| 1812 | }, | |
| 1813 | { | |
| 1814 | "Base64": true, | |
| 1815 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
| 1816 | "name": "hinfo_key" | |
| 1817 | }, | |
| 1818 | { | |
| 1819 | "Base64": true, | |
| 1820 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 1821 | "name": "snapset" | |
| 1822 | } | |
| 1823 | ], | |
| 1824 | "size": 2048, | |
| 1825 | "errors": [], | |
| 1826 | "shard": 0, | |
| 1827 | "osd": 2 | |
| 1828 | } | |
| 1829 | ], | |
| 1830 | "selected_object_info": "2:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", | |
| 1831 | "union_shard_errors": [], | |
| 1832 | "errors": [ | |
| 1833 | "attr_value_mismatch", | |
| 1834 | "attr_name_mismatch" | |
| 1835 | ], | |
| 1836 | "object": { | |
| 1837 | "version": 6, | |
| 1838 | "snap": "head", | |
| 1839 | "locator": "", | |
| 1840 | "nspace": "", | |
| 1841 | "name": "EOBJ4" | |
| 1842 | } | |
| 1843 | }, | |
| 1844 | { | |
| 1845 | "shards": [ | |
| 1846 | { | |
| 1847 | "size": 2048, | |
| 1848 | "errors": [], | |
| 1849 | "shard": 2, | |
| 1850 | "osd": 0 | |
| 1851 | }, | |
| 1852 | { | |
| 1853 | "size": 4096, | |
| 1854 | "errors": [ | |
| 1855 | "size_mismatch_oi" | |
| 1856 | ], | |
| 1857 | "shard": 1, | |
| 1858 | "osd": 1 | |
| 1859 | }, | |
| 1860 | { | |
| 1861 | "size": 2048, | |
| 1862 | "errors": [], | |
| 1863 | "shard": 0, | |
| 1864 | "osd": 2 | |
| 1865 | } | |
| 1866 | ], | |
| 1867 | "selected_object_info": "2:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", | |
| 1868 | "union_shard_errors": [ | |
| 1869 | "size_mismatch_oi" | |
| 1870 | ], | |
| 1871 | "errors": [ | |
| 1872 | "size_mismatch" | |
| 1873 | ], | |
| 1874 | "object": { | |
| 1875 | "version": 7, | |
| 1876 | "snap": "head", | |
| 1877 | "locator": "", | |
| 1878 | "nspace": "", | |
| 1879 | "name": "EOBJ5" | |
| 1880 | } | |
| 1881 | } | |
| 1882 | ], | |
| 1883 | "epoch": 0 | |
| 1884 | } | |
| 1885 | EOF | |
| 1886 | ||
| 1887 | jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson | |
| 1888 | diff -y $termwidth $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 | |
| 1889 | if test $getjson = "yes" | |
| 1890 | then | |
| 1891 | jq '.' $dir/json > save3.json | |
| 1892 | fi | |
| 1893 | ||
| 1894 | if which jsonschema > /dev/null; | |
| 1895 | then | |
| 1896 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
| 1897 | fi | |
| 1898 | ||
| 1899 | pg_deep_scrub $pg | |
| 1900 | ||
| 1901 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
| 1902 | # Check pg count | |
| 1903 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
| 1904 | # Check pgid | |
| 1905 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
| 1906 | ||
| 1907 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
| 1908 | # Get epoch for repair-get requests | |
| 1909 | epoch=$(jq .epoch $dir/json) | |
| 1910 | ||
| 1911 | jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson | |
| 1912 | { | |
| 1913 | "inconsistents": [ | |
| 1914 | { | |
| 1915 | "shards": [ | |
| 1916 | { | |
| 1917 | "data_digest": "0x04cfa72f", | |
| 1918 | "omap_digest": "0xffffffff", | |
| 1919 | "size": 2048, | |
| 1920 | "errors": [], | |
| 1921 | "shard": 2, | |
| 1922 | "osd": 0 | |
| 1923 | }, | |
| 1924 | { | |
| 1925 | "size": 9, | |
| 1926 | "errors": [ | |
| 1927 | "read_error", | |
| 1928 | "size_mismatch_oi" | |
| 1929 | ], | |
| 1930 | "shard": 1, | |
| 1931 | "osd": 1 | |
| 1932 | }, | |
| 1933 | { | |
| 1934 | "data_digest": "0x04cfa72f", | |
| 1935 | "omap_digest": "0xffffffff", | |
| 1936 | "size": 2048, | |
| 1937 | "errors": [], | |
| 1938 | "shard": 0, | |
| 1939 | "osd": 2 | |
| 1940 | } | |
| 1941 | ], | |
| 1942 | "selected_object_info": "2:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", | |
| 1943 | "union_shard_errors": [ | |
| 1944 | "read_error", | |
| 1945 | "size_mismatch_oi" | |
| 1946 | ], | |
| 1947 | "errors": [ | |
| 1948 | "size_mismatch" | |
| 1949 | ], | |
| 1950 | "object": { | |
| 1951 | "version": 1, | |
| 1952 | "snap": "head", | |
| 1953 | "locator": "", | |
| 1954 | "nspace": "", | |
| 1955 | "name": "EOBJ1" | |
| 1956 | } | |
| 1957 | }, | |
| 1958 | { | |
| 1959 | "shards": [ | |
| 1960 | { | |
| 1961 | "size": 2048, | |
| 1962 | "errors": [ | |
| 1963 | "ec_hash_error" | |
| 1964 | ], | |
| 1965 | "shard": 2, | |
| 1966 | "osd": 0 | |
| 1967 | }, | |
| 1968 | { | |
| 1969 | "data_digest": "0x04cfa72f", | |
| 1970 | "omap_digest": "0xffffffff", | |
| 1971 | "size": 2048, | |
| 1972 | "errors": [], | |
| 1973 | "shard": 1, | |
| 1974 | "osd": 1 | |
| 1975 | }, | |
| 1976 | { | |
| 1977 | "data_digest": "0x04cfa72f", | |
| 1978 | "omap_digest": "0xffffffff", | |
| 1979 | "size": 2048, | |
| 1980 | "errors": [], | |
| 1981 | "shard": 0, | |
| 1982 | "osd": 2 | |
| 1983 | } | |
| 1984 | ], | |
| 1985 | "selected_object_info": "2:9babd184:::EOBJ2:head(29'2 client.4217.0:1 dirty|data_digest|omap_digest s 7 uv 2 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", | |
| 1986 | "union_shard_errors": [ | |
| 1987 | "ec_hash_error" | |
| 1988 | ], | |
| 1989 | "errors": [], | |
| 1990 | "object": { | |
| 1991 | "version": 2, | |
| 1992 | "snap": "head", | |
| 1993 | "locator": "", | |
| 1994 | "nspace": "", | |
| 1995 | "name": "EOBJ2" | |
| 1996 | } | |
| 1997 | }, | |
| 1998 | { | |
| 1999 | "shards": [ | |
| 2000 | { | |
| 2001 | "data_digest": "0x04cfa72f", | |
| 2002 | "omap_digest": "0xffffffff", | |
| 2003 | "size": 2048, | |
| 2004 | "errors": [], | |
| 2005 | "shard": 2, | |
| 2006 | "osd": 0 | |
| 2007 | }, | |
| 2008 | { | |
| 2009 | "errors": [ | |
| 2010 | "missing" | |
| 2011 | ], | |
| 2012 | "shard": 1, | |
| 2013 | "osd": 1 | |
| 2014 | }, | |
| 2015 | { | |
| 2016 | "data_digest": "0x04cfa72f", | |
| 2017 | "omap_digest": "0xffffffff", | |
| 2018 | "size": 2048, | |
| 2019 | "errors": [], | |
| 2020 | "shard": 0, | |
| 2021 | "osd": 2 | |
| 2022 | } | |
| 2023 | ], | |
| 2024 | "selected_object_info": "2:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", | |
| 2025 | "union_shard_errors": [ | |
| 2026 | "missing" | |
| 2027 | ], | |
| 2028 | "errors": [], | |
| 2029 | "object": { | |
| 2030 | "version": 3, | |
| 2031 | "snap": "head", | |
| 2032 | "locator": "", | |
| 2033 | "nspace": "", | |
| 2034 | "name": "EOBJ3" | |
| 2035 | } | |
| 2036 | }, | |
| 2037 | { | |
| 2038 | "shards": [ | |
| 2039 | { | |
| 2040 | "attrs": [ | |
| 2041 | { | |
| 2042 | "Base64": true, | |
| 2043 | "value": "", | |
| 2044 | "name": "_" | |
| 2045 | }, | |
| 2046 | { | |
| 2047 | "Base64": false, | |
| 2048 | "value": "bad-val", | |
| 2049 | "name": "_key1-EOBJ4" | |
| 2050 | }, | |
| 2051 | { | |
| 2052 | "Base64": false, | |
| 2053 | "value": "val3-EOBJ4", | |
| 2054 | "name": "_key3-EOBJ4" | |
| 2055 | }, | |
| 2056 | { | |
| 2057 | "Base64": true, | |
| 2058 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
| 2059 | "name": "hinfo_key" | |
| 2060 | }, | |
| 2061 | { | |
| 2062 | "Base64": true, | |
| 2063 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 2064 | "name": "snapset" | |
| 2065 | } | |
| 2066 | ], | |
| 2067 | "data_digest": "0x04cfa72f", | |
| 2068 | "omap_digest": "0xffffffff", | |
| 2069 | "size": 2048, | |
| 2070 | "errors": [], | |
| 2071 | "shard": 2, | |
| 2072 | "osd": 0 | |
| 2073 | }, | |
| 2074 | { | |
| 2075 | "attrs": [ | |
| 2076 | { | |
| 2077 | "Base64": true, | |
| 2078 | "value": "", | |
| 2079 | "name": "_" | |
| 2080 | }, | |
| 2081 | { | |
| 2082 | "Base64": false, | |
| 2083 | "value": "val1-EOBJ4", | |
| 2084 | "name": "_key1-EOBJ4" | |
| 2085 | }, | |
| 2086 | { | |
| 2087 | "Base64": false, | |
| 2088 | "value": "val2-EOBJ4", | |
| 2089 | "name": "_key2-EOBJ4" | |
| 2090 | }, | |
| 2091 | { | |
| 2092 | "Base64": true, | |
| 2093 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
| 2094 | "name": "hinfo_key" | |
| 2095 | }, | |
| 2096 | { | |
| 2097 | "Base64": true, | |
| 2098 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 2099 | "name": "snapset" | |
| 2100 | } | |
| 2101 | ], | |
| 2102 | "data_digest": "0x04cfa72f", | |
| 2103 | "omap_digest": "0xffffffff", | |
| 2104 | "size": 2048, | |
| 2105 | "errors": [], | |
| 2106 | "shard": 1, | |
| 2107 | "osd": 1 | |
| 2108 | }, | |
| 2109 | { | |
| 2110 | "attrs": [ | |
| 2111 | { | |
| 2112 | "Base64": true, | |
| 2113 | "value": "", | |
| 2114 | "name": "_" | |
| 2115 | }, | |
| 2116 | { | |
| 2117 | "Base64": false, | |
| 2118 | "value": "val1-EOBJ4", | |
| 2119 | "name": "_key1-EOBJ4" | |
| 2120 | }, | |
| 2121 | { | |
| 2122 | "Base64": false, | |
| 2123 | "value": "val2-EOBJ4", | |
| 2124 | "name": "_key2-EOBJ4" | |
| 2125 | }, | |
| 2126 | { | |
| 2127 | "Base64": true, | |
| 2128 | "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", | |
| 2129 | "name": "hinfo_key" | |
| 2130 | }, | |
| 2131 | { | |
| 2132 | "Base64": true, | |
| 2133 | "value": "AgIZAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAA==", | |
| 2134 | "name": "snapset" | |
| 2135 | } | |
| 2136 | ], | |
| 2137 | "data_digest": "0x04cfa72f", | |
| 2138 | "omap_digest": "0xffffffff", | |
| 2139 | "size": 2048, | |
| 2140 | "errors": [], | |
| 2141 | "shard": 0, | |
| 2142 | "osd": 2 | |
| 2143 | } | |
| 2144 | ], | |
| 2145 | "selected_object_info": "2:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", | |
| 2146 | "union_shard_errors": [], | |
| 2147 | "errors": [ | |
| 2148 | "attr_value_mismatch", | |
| 2149 | "attr_name_mismatch" | |
| 2150 | ], | |
| 2151 | "object": { | |
| 2152 | "version": 6, | |
| 2153 | "snap": "head", | |
| 2154 | "locator": "", | |
| 2155 | "nspace": "", | |
| 2156 | "name": "EOBJ4" | |
| 2157 | } | |
| 2158 | }, | |
| 2159 | { | |
| 2160 | "shards": [ | |
| 2161 | { | |
| 2162 | "data_digest": "0x04cfa72f", | |
| 2163 | "omap_digest": "0xffffffff", | |
| 2164 | "size": 2048, | |
| 2165 | "errors": [], | |
| 2166 | "shard": 2, | |
| 2167 | "osd": 0 | |
| 2168 | }, | |
| 2169 | { | |
| 2170 | "size": 4096, | |
| 2171 | "errors": [ | |
| 2172 | "size_mismatch_oi", | |
| 2173 | "ec_size_error" | |
| 2174 | ], | |
| 2175 | "shard": 1, | |
| 2176 | "osd": 1 | |
| 2177 | }, | |
| 2178 | { | |
| 2179 | "data_digest": "0x04cfa72f", | |
| 2180 | "omap_digest": "0xffffffff", | |
| 2181 | "size": 2048, | |
| 2182 | "errors": [], | |
| 2183 | "shard": 0, | |
| 2184 | "osd": 2 | |
| 2185 | } | |
| 2186 | ], | |
| 2187 | "selected_object_info": "2:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", | |
| 2188 | "union_shard_errors": [ | |
| 2189 | "size_mismatch_oi", | |
| 2190 | "ec_size_error" | |
| 2191 | ], | |
| 2192 | "errors": [ | |
| 2193 | "size_mismatch" | |
| 2194 | ], | |
| 2195 | "object": { | |
| 2196 | "version": 7, | |
| 2197 | "snap": "head", | |
| 2198 | "locator": "", | |
| 2199 | "nspace": "", | |
| 2200 | "name": "EOBJ5" | |
| 2201 | } | |
| 2202 | } | |
| 2203 | ], | |
| 2204 | "epoch": 0 | |
| 2205 | } | |
| 2206 | EOF | |
| 2207 | ||
| 2208 | jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson | |
| 2209 | if [ "$allow_overwrites" = "true" ] | |
| 2210 | then | |
| 2211 | grep -v data_digest $dir/csjson | grep -v ec_size_error > $dir/csjson | |
| 2212 | grep -v data_digest $dir/checkcsjson | grep -v ec_size_error > $dir/checkcsjson | |
| 2213 | fi | |
| 2214 | diff -y $termwidth $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 | |
| 2215 | if test $getjson = "yes" | |
| 2216 | then | |
| 2217 | jq '.' $dir/json > save4.json | |
| 2218 | fi | |
| 2219 | ||
| 2220 | if which jsonschema > /dev/null; | |
| 2221 | then | |
| 2222 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
| 2223 | fi | |
| 2224 | ||
| 2225 | rados rmpool $poolname $poolname --yes-i-really-really-mean-it | |
| 2226 | teardown $dir || return 1 | |
| 2227 | } | |
| 2228 | ||
| 2229 | function TEST_corrupt_scrub_erasure_appends() { | |
| 2230 | corrupt_scrub_erasure $1 false | |
| 2231 | } | |
| 2232 | ||
| 2233 | function TEST_corrupt_scrub_erasure_overwrites() { | |
| 2234 | corrupt_scrub_erasure $1 true | |
| 2235 | } | |
| 2236 | ||
| 2237 | # | |
| 2238 | # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost | |
| 2239 | # | |
| 2240 | function TEST_periodic_scrub_replicated() { | |
| 2241 | local dir=$1 | |
| 2242 | local poolname=psr_pool | |
| 2243 | local objname=POBJ | |
| 2244 | ||
| 2245 | setup $dir || return 1 | |
| 2246 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
| 2247 | run_mgr $dir x || return 1 | |
| 2248 | local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0" | |
| 2249 | run_osd $dir 0 $ceph_osd_args || return 1 | |
| 2250 | run_osd $dir 1 $ceph_osd_args || return 1 | |
| 2251 | wait_for_clean || return 1 | |
| 2252 | ||
| 2253 | ceph osd pool create $poolname 1 1 || return 1 | |
| 2254 | wait_for_clean || return 1 | |
| 2255 | ||
| 2256 | local osd=0 | |
| 2257 | add_something $dir $poolname $objname scrub || return 1 | |
| 2258 | local primary=$(get_primary $poolname $objname) | |
| 2259 | local pg=$(get_pg $poolname $objname) | |
| 2260 | ||
| 2261 | # Add deep-scrub only error | |
| 2262 | local payload=UVWXYZ | |
| 2263 | echo $payload > $dir/CORRUPT | |
| 2264 | # Uses $ceph_osd_args for osd restart | |
| 2265 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
| 2266 | ||
| 2267 | # No scrub information available, so expect failure | |
| 2268 | set -o pipefail | |
| 2269 | ! rados list-inconsistent-obj $pg | jq '.' || return 1 | |
| 2270 | set +o pipefail | |
| 2271 | ||
| 2272 | pg_deep_scrub $pg || return 1 | |
| 2273 | ||
| 2274 | # Make sure bad object found | |
| 2275 | rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 | |
| 2276 | ||
| 2277 | local last_scrub=$(get_last_scrub_stamp $pg) | |
| 2278 | # Fake a schedule scrub | |
| 2279 | CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.${primary}.asok \ | |
| 2280 | trigger_scrub $pg || return 1 | |
| 2281 | # Wait for schedule regular scrub | |
| 2282 | wait_for_scrub $pg "$last_scrub" | |
| 2283 | ||
| 2284 | # It needed to be upgraded | |
| 2285 | grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1 | |
| 2286 | ||
| 2287 | # Bad object still known | |
| 2288 | rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 | |
| 2289 | ||
| 2290 | # Can't upgrade with this set | |
| 2291 | ceph osd set nodeep-scrub | |
| 2292 | ||
| 2293 | # Fake a schedule scrub | |
| 2294 | local last_scrub=$(get_last_scrub_stamp $pg) | |
| 2295 | CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.${primary}.asok \ | |
| 2296 | trigger_scrub $pg || return 1 | |
| 2297 | # Wait for schedule regular scrub | |
| 2298 | # to notice scrub and skip it | |
| 2299 | local found=false | |
| 2300 | for i in $(seq 14 -1 0) | |
| 2301 | do | |
| 2302 | sleep 1 | |
| 2303 | ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; } | |
| 2304 | echo Time left: $i seconds | |
| 2305 | done | |
| 2306 | test $found = "true" || return 1 | |
| 2307 | ||
| 2308 | # Bad object still known | |
| 2309 | rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 | |
| 2310 | ||
| 2311 | # Request a regular scrub and it will be done | |
| 2312 | local scrub_backoff_ratio=$(get_config osd ${primary} osd_scrub_backoff_ratio) | |
| 2313 | set_config osd ${primary} osd_scrub_backoff_ratio 0 | |
| 2314 | pg_scrub $pg | |
| 2315 | sleep 1 | |
| 2316 | set_config osd ${primary} osd_scrub_backoff_ratio $scrub_backoff_ratio | |
| 2317 | grep -q "Regular scrub request, losing deep-scrub details" $dir/osd.${primary}.log || return 1 | |
| 2318 | ||
| 2319 | # deep-scrub error is no longer present | |
| 2320 | rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1 | |
| 2321 | } | |
| 2322 | ||
| 2323 | ||
| 2324 | main osd-scrub-repair "$@" | |
| 2325 | ||
| 2326 | # Local Variables: | |
| 2327 | # compile-command: "cd ../.. ; make -j4 && \ | |
| 2328 | # test/osd/osd-scrub-repair.sh # TEST_corrupt_and_repair_replicated" | |
| 2329 | # End: |