ceph/qa/standalone/scrub/osd-scrub-snaps.sh

   1 #! /bin/bash
   2 #
   3 # Copyright (C) 2015 Red Hat <contact@redhat.com>
   4 #
   5 # Author: David Zafman <dzafman@redhat.com>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  18
  19 # Test development and debugging
  20 # Set to "yes" in order to ignore diff errors and save results to update test
  21 getjson="no"
  22
  23 function run() {
  24     local dir=$1
  25     shift
  26
  27     export CEPH_MON="127.0.0.1:7121" # git grep '\<7121\>' : there must be only one
  28     export CEPH_ARGS
  29     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  30     CEPH_ARGS+="--mon-host=$CEPH_MON "
  31
  32     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  33     for func in $funcs ; do
  34         setup $dir || return 1
  35         $func $dir || return 1
  36         teardown $dir || return 1
  37     done
  38 }
  39
  40 function TEST_scrub_snaps() {
  41     local dir=$1
  42     local poolname=test
  43     local OBJS=15
  44     local OSDS=1
  45
  46     TESTDATA="testdata.$$"
  47
  48     run_mon $dir a --osd_pool_default_size=$OSDS || return 1
  49     run_mgr $dir x || return 1
  50     for osd in $(seq 0 $(expr $OSDS - 1))
  51     do
  52       run_osd $dir $osd || return 1
  53     done
  54
  55     # Create a pool with a single pg
  56     create_pool $poolname 1 1
  57     wait_for_clean || return 1
  58     poolid=$(ceph osd dump | grep "^pool.*[']test[']" | awk '{ print $2 }')
  59
  60     dd if=/dev/urandom of=$TESTDATA bs=1032 count=1
  61     for i in `seq 1 $OBJS`
  62     do
  63         rados -p $poolname put obj${i} $TESTDATA
  64     done
  65
  66     local primary=$(get_primary $poolname obj1)
  67     SNAP=1
  68     rados -p $poolname mksnap snap${SNAP}
  69     dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP}
  70     rados -p $poolname put obj1 $TESTDATA
  71     rados -p $poolname put obj5 $TESTDATA
  72     rados -p $poolname put obj3 $TESTDATA
  73     for i in `seq 6 14`
  74      do rados -p $poolname put obj${i} $TESTDATA
  75     done
  76
  77     SNAP=2
  78     rados -p $poolname mksnap snap${SNAP}
  79     dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP}
  80     rados -p $poolname put obj5 $TESTDATA
  81
  82     SNAP=3
  83     rados -p $poolname mksnap snap${SNAP}
  84     dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP}
  85     rados -p $poolname put obj3 $TESTDATA
  86
  87     SNAP=4
  88     rados -p $poolname mksnap snap${SNAP}
  89     dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP}
  90     rados -p $poolname put obj5 $TESTDATA
  91     rados -p $poolname put obj2 $TESTDATA
  92
  93     SNAP=5
  94     rados -p $poolname mksnap snap${SNAP}
  95     SNAP=6
  96     rados -p $poolname mksnap snap${SNAP}
  97     dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP}
  98     rados -p $poolname put obj5 $TESTDATA
  99
 100     SNAP=7
 101     rados -p $poolname mksnap snap${SNAP}
 102
 103     rados -p $poolname rm obj4
 104     rados -p $poolname rm obj2
 105
 106     kill_daemons $dir TERM osd || return 1
 107
 108     # Don't need to use ceph_objectstore_tool() function because osd stopped
 109
 110     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj1)"
 111     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" --force remove
 112
 113     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":2)"
 114     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove
 115
 116     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":1)"
 117     OBJ5SAVE="$JSON"
 118     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove
 119
 120     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":4)"
 121     dd if=/dev/urandom of=$TESTDATA bs=256 count=18
 122     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA
 123
 124     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj3)"
 125     dd if=/dev/urandom of=$TESTDATA bs=256 count=15
 126     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA
 127
 128     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj4 | grep \"snapid\":7)"
 129     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove
 130
 131     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj2)"
 132     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" rm-attr snapset
 133
 134     # Create a clone which isn't in snapset and doesn't have object info
 135     JSON="$(echo "$OBJ5SAVE" | sed s/snapid\":1/snapid\":7/)"
 136     dd if=/dev/urandom of=$TESTDATA bs=256 count=7
 137     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA
 138
 139     rm -f $TESTDATA
 140
 141     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj6)"
 142     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset
 143     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj7)"
 144     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset corrupt
 145     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj8)"
 146     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset seq
 147     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj9)"
 148     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clone_size
 149     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj10)"
 150     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clone_overlap
 151     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj11)"
 152     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clones
 153     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj12)"
 154     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset head
 155     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj13)"
 156     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset snaps
 157     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj14)"
 158     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset size
 159
 160     echo "garbage" > $dir/bad
 161     JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj15)"
 162     ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-attr snapset $dir/bad
 163     rm -f $dir/bad
 164
 165     for osd in $(seq 0 $(expr $OSDS - 1))
 166     do
 167       run_osd $dir $osd || return 1
 168     done
 169
 170     local pgid="${poolid}.0"
 171     if ! pg_scrub "$pgid" ; then
 172         cat $dir/osd.0.log
 173         return 1
 174     fi
 175     grep 'log_channel' $dir/osd.0.log
 176
 177     rados list-inconsistent-pg $poolname > $dir/json || return 1
 178     # Check pg count
 179     test $(jq '. | length' $dir/json) = "1" || return 1
 180     # Check pgid
 181     test $(jq -r '.[0]' $dir/json) = $pgid || return 1
 182
 183     rados list-inconsistent-snapset $pgid > $dir/json || return 1
 184
 185     local jqfilter='.inconsistents'
 186     local sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
 187
 188     jq "$jqfilter" << EOF | python -c "$sortkeys" > $dir/checkcsjson
 189 {
 190   "inconsistents": [
 191     {
 192       "errors": [
 193         "headless"
 194       ],
 195       "snap": 1,
 196       "locator": "",
 197       "nspace": "",
 198       "name": "obj1"
 199     },
 200     {
 201       "errors": [
 202         "size_mismatch"
 203       ],
 204       "snap": 1,
 205       "locator": "",
 206       "nspace": "",
 207       "name": "obj10"
 208     },
 209     {
 210       "errors": [
 211         "headless"
 212       ],
 213       "snap": 1,
 214       "locator": "",
 215       "nspace": "",
 216       "name": "obj11"
 217     },
 218     {
 219       "errors": [
 220         "size_mismatch"
 221       ],
 222       "snap": 1,
 223       "locator": "",
 224       "nspace": "",
 225       "name": "obj14"
 226     },
 227     {
 228       "errors": [
 229         "headless"
 230       ],
 231       "snap": 1,
 232       "locator": "",
 233       "nspace": "",
 234       "name": "obj6"
 235     },
 236     {
 237       "errors": [
 238         "headless"
 239       ],
 240       "snap": 1,
 241       "locator": "",
 242       "nspace": "",
 243       "name": "obj7"
 244     },
 245     {
 246       "errors": [
 247         "size_mismatch"
 248       ],
 249       "snap": 1,
 250       "locator": "",
 251       "nspace": "",
 252       "name": "obj9"
 253     },
 254     {
 255       "errors": [
 256         "headless"
 257       ],
 258       "snap": 4,
 259       "locator": "",
 260       "nspace": "",
 261       "name": "obj2"
 262     },
 263     {
 264       "errors": [
 265         "size_mismatch"
 266       ],
 267       "snap": 4,
 268       "locator": "",
 269       "nspace": "",
 270       "name": "obj5"
 271     },
 272     {
 273       "errors": [
 274         "headless"
 275       ],
 276       "snap": 7,
 277       "locator": "",
 278       "nspace": "",
 279       "name": "obj2"
 280     },
 281     {
 282       "errors": [
 283         "info_missing",
 284         "headless"
 285       ],
 286       "snap": 7,
 287       "locator": "",
 288       "nspace": "",
 289       "name": "obj5"
 290     },
 291     {
 292       "name": "obj10",
 293       "nspace": "",
 294       "locator": "",
 295       "snap": "head",
 296       "snapset": {
 297         "head_exists": 1,
 298         "snap_context": {
 299           "seq": 1,
 300           "snaps": [
 301             1
 302           ]
 303         },
 304         "clones": [
 305           {
 306             "snap": 1,
 307             "size": 1032,
 308             "overlap": "????",
 309             "snaps": [
 310               1
 311             ]
 312           }
 313         ]
 314       },
 315       "errors": []
 316     },
 317     {
 318       "extra clones": [
 319         1
 320       ],
 321       "errors": [
 322         "extra_clones"
 323       ],
 324       "snap": "head",
 325       "locator": "",
 326       "nspace": "",
 327       "name": "obj11",
 328       "snapset": {
 329         "head_exists": 1,
 330         "snap_context": {
 331           "seq": 1,
 332           "snaps": [
 333             1
 334           ]
 335         },
 336         "clones": []
 337       }
 338     },
 339     {
 340       "errors": [
 341         "head_mismatch"
 342       ],
 343       "snap": "head",
 344       "locator": "",
 345       "nspace": "",
 346       "name": "obj12",
 347       "snapset": {
 348         "head_exists": 0,
 349         "snap_context": {
 350           "seq": 1,
 351           "snaps": [
 352             1
 353           ]
 354         },
 355         "clones": [
 356           {
 357             "snap": 1,
 358             "size": 1032,
 359             "overlap": "[]",
 360             "snaps": [
 361               1
 362             ]
 363           }
 364         ]
 365       }
 366     },
 367     {
 368       "name": "obj14",
 369       "nspace": "",
 370       "locator": "",
 371       "snap": "head",
 372       "snapset": {
 373         "head_exists": 1,
 374         "snap_context": {
 375           "seq": 1,
 376           "snaps": [
 377             1
 378           ]
 379         },
 380         "clones": [
 381           {
 382             "snap": 1,
 383             "size": 1033,
 384             "overlap": "[]",
 385             "snaps": [
 386               1
 387             ]
 388           }
 389         ]
 390       },
 391       "errors": []
 392     },
 393     {
 394       "errors": [
 395         "snapset_corrupted"
 396       ],
 397       "snap": "head",
 398       "locator": "",
 399       "nspace": "",
 400       "name": "obj15"
 401     },
 402     {
 403       "extra clones": [
 404         7,
 405         4
 406       ],
 407       "errors": [
 408         "snapset_missing",
 409         "extra_clones"
 410       ],
 411       "snap": "head",
 412       "locator": "",
 413       "nspace": "",
 414       "name": "obj2"
 415     },
 416     {
 417       "errors": [
 418         "size_mismatch"
 419       ],
 420       "snap": "head",
 421       "locator": "",
 422       "nspace": "",
 423       "name": "obj3",
 424       "snapset": {
 425         "head_exists": 1,
 426         "snap_context": {
 427           "seq": 3,
 428           "snaps": [
 429             3,
 430             2,
 431             1
 432           ]
 433         },
 434         "clones": [
 435           {
 436             "snap": 1,
 437             "size": 1032,
 438             "overlap": "[]",
 439             "snaps": [
 440               1
 441             ]
 442           },
 443           {
 444             "snap": 3,
 445             "size": 256,
 446             "overlap": "[]",
 447             "snaps": [
 448               3,
 449               2
 450             ]
 451           }
 452         ]
 453       }
 454     },
 455     {
 456       "missing": [
 457         7
 458       ],
 459       "errors": [
 460         "clone_missing"
 461       ],
 462       "snap": "head",
 463       "locator": "",
 464       "nspace": "",
 465       "name": "obj4",
 466       "snapset": {
 467         "head_exists": 1,
 468         "snap_context": {
 469           "seq": 7,
 470           "snaps": [
 471             7,
 472             6,
 473             5,
 474             4,
 475             3,
 476             2,
 477             1
 478           ]
 479         },
 480         "clones": [
 481           {
 482             "snap": 7,
 483             "size": 1032,
 484             "overlap": "[]",
 485             "snaps": [
 486               7,
 487               6,
 488               5,
 489               4,
 490               3,
 491               2,
 492               1
 493             ]
 494           }
 495         ]
 496       }
 497     },
 498     {
 499       "missing": [
 500         2,
 501         1
 502       ],
 503       "extra clones": [
 504         7
 505       ],
 506       "errors": [
 507         "extra_clones",
 508         "clone_missing"
 509       ],
 510       "snap": "head",
 511       "locator": "",
 512       "nspace": "",
 513       "name": "obj5",
 514       "snapset": {
 515         "head_exists": 1,
 516         "snap_context": {
 517           "seq": 6,
 518           "snaps": [
 519             6,
 520             5,
 521             4,
 522             3,
 523             2,
 524             1
 525           ]
 526         },
 527         "clones": [
 528           {
 529             "snap": 1,
 530             "size": 1032,
 531             "overlap": "[]",
 532             "snaps": [
 533               1
 534             ]
 535           },
 536           {
 537             "snap": 2,
 538             "size": 256,
 539             "overlap": "[]",
 540             "snaps": [
 541               2
 542             ]
 543           },
 544           {
 545             "snap": 4,
 546             "size": 512,
 547             "overlap": "[]",
 548             "snaps": [
 549               4,
 550               3
 551             ]
 552           },
 553           {
 554             "snap": 6,
 555             "size": 1024,
 556             "overlap": "[]",
 557             "snaps": [
 558               6,
 559               5
 560             ]
 561           }
 562         ]
 563       }
 564     },
 565     {
 566       "extra clones": [
 567         1
 568       ],
 569       "errors": [
 570         "extra_clones"
 571       ],
 572       "snap": "head",
 573       "locator": "",
 574       "nspace": "",
 575       "name": "obj6",
 576       "snapset": {
 577         "head_exists": 1,
 578         "snap_context": {
 579           "seq": 1,
 580           "snaps": [
 581             1
 582           ]
 583         },
 584         "clones": []
 585       }
 586     },
 587     {
 588       "extra clones": [
 589         1
 590       ],
 591       "errors": [
 592         "head_mismatch",
 593         "extra_clones"
 594       ],
 595       "snap": "head",
 596       "locator": "",
 597       "nspace": "",
 598       "name": "obj7",
 599       "snapset": {
 600         "head_exists": 0,
 601         "snap_context": {
 602           "seq": 0,
 603           "snaps": []
 604         },
 605         "clones": []
 606       }
 607     },
 608     {
 609       "errors": [
 610         "snapset_error"
 611       ],
 612       "snap": "head",
 613       "locator": "",
 614       "nspace": "",
 615       "name": "obj8",
 616       "snapset": {
 617         "head_exists": 1,
 618         "snap_context": {
 619           "seq": 0,
 620           "snaps": [
 621             1
 622           ]
 623         },
 624         "clones": [
 625           {
 626             "snap": 1,
 627             "size": 1032,
 628             "overlap": "[]",
 629             "snaps": [
 630               1
 631             ]
 632           }
 633         ]
 634       }
 635     },
 636     {
 637       "name": "obj9",
 638       "nspace": "",
 639       "locator": "",
 640       "snap": "head",
 641       "snapset": {
 642         "head_exists": 1,
 643         "snap_context": {
 644           "seq": 1,
 645           "snaps": [
 646             1
 647           ]
 648         },
 649         "clones": [
 650           {
 651             "snap": 1,
 652             "size": "????",
 653             "overlap": "[]",
 654             "snaps": [
 655               1
 656             ]
 657           }
 658         ]
 659       },
 660       "errors": []
 661     }
 662   ],
 663   "epoch": 20
 664 }
 665 EOF
 666
 667     jq "$jqfilter" $dir/json | python -c "$sortkeys" > $dir/csjson
 668     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
 669     if test $getjson = "yes"
 670     then
 671         jq '.' $dir/json > save1.json
 672     fi
 673
 674     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
 675     then
 676       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-snap.json || return 1
 677     fi
 678
 679     for i in `seq 1 7`
 680     do
 681         rados -p $poolname rmsnap snap$i
 682     done
 683
 684     ERRORS=0
 685
 686     pidfile=$(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid')
 687     pid=$(cat $pidfile)
 688     if ! kill -0 $pid
 689     then
 690         echo "OSD crash occurred"
 691         tail -100 $dir/osd.0.log
 692         ERRORS=$(expr $ERRORS + 1)
 693     fi
 694
 695     kill_daemons $dir || return 1
 696
 697     declare -a err_strings
 698     err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj10:.* is missing in clone_overlap"
 699     err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:7 no '_' attr"
 700     err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:7 is an unexpected clone"
 701     err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:4 on disk size [(]4608[)] does not match object info size [(]512[)] adjusted for ondisk to [(]512[)]"
 702     err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj5:head expected clone .*:::obj5:2"
 703     err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj5:head expected clone .*:::obj5:1"
 704     err_strings[6]="log_channel[(]cluster[)] log [[]INF[]] : scrub [0-9]*[.]0 .*:::obj5:head 2 missing clone[(]s[)]"
 705     err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj12:head snapset.head_exists=false, but head exists"
 706     err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj8:head snaps.seq not set"
 707     err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj7:head snapset.head_exists=false, but head exists"
 708     err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj7:1 is an unexpected clone"
 709     err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj3:head on disk size [(]3840[)] does not match object info size [(]768[)] adjusted for ondisk to [(]768[)]"
 710     err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj6:1 is an unexpected clone"
 711     err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:head no 'snapset' attr"
 712     err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:7 clone ignored due to missing snapset"
 713     err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:4 clone ignored due to missing snapset"
 714     err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj4:head expected clone .*:::obj4:7"
 715     err_strings[17]="log_channel[(]cluster[)] log [[]INF[]] : scrub [0-9]*[.]0 .*:::obj4:head 1 missing clone[(]s[)]"
 716     err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj1:1 is an unexpected clone"
 717     err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj9:1 is missing in clone_size"
 718     err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj11:1 is an unexpected clone"
 719     err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj14:1 size 1032 != clone_size 1033"
 720     err_strings[22]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 22 errors"
 721     err_strings[23]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj15:head can't decode 'snapset' attr buffer"
 722
 723     for err_string in "${err_strings[@]}"
 724     do
 725         if ! grep "$err_string" $dir/osd.${primary}.log > /dev/null;
 726         then
 727             echo "Missing log message '$err_string'"
 728             ERRORS=$(expr $ERRORS + 1)
 729         fi
 730     done
 731
 732     if [ $ERRORS != "0" ];
 733     then
 734         echo "TEST FAILED WITH $ERRORS ERRORS"
 735         return 1
 736     fi
 737
 738     echo "TEST PASSED"
 739     return 0
 740 }
 741
 742 main osd-scrub-snaps "$@"
 743
 744 # Local Variables:
 745 # compile-command: "cd ../.. ; make -j4 && \
 746 #    test/osd/osd-scrub-snaps.sh"