]> git.proxmox.com Git - ceph.git/blobdiff - ceph/qa/standalone/scrub/osd-scrub-snaps.sh
update sources to 12.2.7
[ceph.git] / ceph / qa / standalone / scrub / osd-scrub-snaps.sh
index cf6f67b6dbe2d1b6d3694b940d182a8de738ea0a..a83cfe75c9a8ee49daf15e722e3e18f8bbfcc629 100755 (executable)
@@ -20,6 +20,9 @@ source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
 # Set to "yes" in order to ignore diff errors and save results to update test
 getjson="no"
 
+jqfilter='.inconsistents'
+sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
+
 function run() {
     local dir=$1
     shift
@@ -37,33 +40,12 @@ function run() {
     done
 }
 
-function TEST_scrub_snaps() {
+function create_scenario() {
     local dir=$1
-    local poolname=test
-    local OBJS=15
-    local OSDS=1
-
-    TESTDATA="testdata.$$"
+    local poolname=$2
+    local TESTDATA=$3
+    local osd=$4
 
-    run_mon $dir a --osd_pool_default_size=$OSDS || return 1
-    run_mgr $dir x || return 1
-    for osd in $(seq 0 $(expr $OSDS - 1))
-    do
-      run_osd $dir $osd || return 1
-    done
-
-    # Create a pool with a single pg
-    create_pool $poolname 1 1
-    wait_for_clean || return 1
-    poolid=$(ceph osd dump | grep "^pool.*[']test[']" | awk '{ print $2 }')
-
-    dd if=/dev/urandom of=$TESTDATA bs=1032 count=1
-    for i in `seq 1 $OBJS`
-    do
-        rados -p $poolname put obj${i} $TESTDATA
-    done
-
-    local primary=$(get_primary $poolname obj1)
     SNAP=1
     rados -p $poolname mksnap snap${SNAP}
     dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP}
@@ -107,60 +89,91 @@ function TEST_scrub_snaps() {
 
     # Don't need to use ceph_objectstore_tool() function because osd stopped
 
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj1)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" --force remove
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj1)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" --force remove
 
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":2)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj5 | grep \"snapid\":2)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" remove
 
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":1)"
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj5 | grep \"snapid\":1)"
     OBJ5SAVE="$JSON"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" remove
 
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":4)"
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj5 | grep \"snapid\":4)"
     dd if=/dev/urandom of=$TESTDATA bs=256 count=18
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" set-bytes $TESTDATA
 
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj3)"
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj3)"
     dd if=/dev/urandom of=$TESTDATA bs=256 count=15
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" set-bytes $TESTDATA
 
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj4 | grep \"snapid\":7)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj4 | grep \"snapid\":7)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" remove
 
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj2)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" rm-attr snapset
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj2)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" rm-attr snapset
 
     # Create a clone which isn't in snapset and doesn't have object info
     JSON="$(echo "$OBJ5SAVE" | sed s/snapid\":1/snapid\":7/)"
     dd if=/dev/urandom of=$TESTDATA bs=256 count=7
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" set-bytes $TESTDATA
 
-    rm -f $TESTDATA
-
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj6)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj7)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset corrupt
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj8)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset seq
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj9)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clone_size
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj10)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clone_overlap
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj11)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clones
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj12)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset head
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj13)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset snaps
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj14)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset size
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj6)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj7)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset corrupt
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj8)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset seq
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj9)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset clone_size
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj10)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset clone_overlap
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj11)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset clones
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj12)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset head
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj13)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset snaps
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj14)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset size
 
     echo "garbage" > $dir/bad
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj15)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-attr snapset $dir/bad
+    JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj15)"
+    ceph-objectstore-tool --data-path $dir/${osd} "$JSON" set-attr snapset $dir/bad
     rm -f $dir/bad
+}
+
+function TEST_scrub_snaps() {
+    local dir=$1
+    local poolname=test
+    local OBJS=15
+    local OSDS=1
+
+    TESTDATA="testdata.$$"
+
+    run_mon $dir a --osd_pool_default_size=$OSDS || return 1
+    run_mgr $dir x || return 1
+    for osd in $(seq 0 $(expr $OSDS - 1))
+    do
+      run_osd $dir $osd || return 1
+    done
+
+    # Create a pool with a single pg
+    create_pool $poolname 1 1
+    wait_for_clean || return 1
+    poolid=$(ceph osd dump | grep "^pool.*[']test[']" | awk '{ print $2 }')
+
+    dd if=/dev/urandom of=$TESTDATA bs=1032 count=1
+    for i in `seq 1 $OBJS`
+    do
+        rados -p $poolname put obj${i} $TESTDATA
+    done
+
+    local primary=$(get_primary $poolname obj1)
+
+    create_scenario $dir $poolname $TESTDATA $primary
+
+    rm -f $TESTDATA
 
     for osd in $(seq 0 $(expr $OSDS - 1))
     do
@@ -172,7 +185,8 @@ function TEST_scrub_snaps() {
         cat $dir/osd.0.log
         return 1
     fi
-    grep 'log_channel' $dir/osd.0.log
+
+    test "$(grep "_scan_snaps start" $dir/osd.${primary}.log | wc -l)" = "2" || return 1
 
     rados list-inconsistent-pg $poolname > $dir/json || return 1
     # Check pg count
@@ -180,10 +194,22 @@ function TEST_scrub_snaps() {
     # Check pgid
     test $(jq -r '.[0]' $dir/json) = $pgid || return 1
 
-    rados list-inconsistent-snapset $pgid > $dir/json || return 1
+    rados list-inconsistent-obj $pgid > $dir/json || return 1
 
-    local jqfilter='.inconsistents'
-    local sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
+    # The injected snapshot errors with a single copy pool doesn't
+    # see object errors because all the issues are detected by
+    # comparing copies.
+    jq "$jqfilter" << EOF | python -c "$sortkeys" > $dir/checkcsjson
+{
+    "epoch": 17,
+    "inconsistents": []
+}
+EOF
+
+    jq "$jqfilter" $dir/json | python -c "$sortkeys" > $dir/csjson
+    diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
+
+    rados list-inconsistent-snapset $pgid > $dir/json || return 1
 
     jq "$jqfilter" << EOF | python -c "$sortkeys" > $dir/checkcsjson
 {
@@ -676,6 +702,13 @@ EOF
       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-snap.json || return 1
     fi
 
+    pidfiles=$(find $dir 2>/dev/null | grep 'osd[^/]*\.pid')
+    pids=""
+    for pidfile in ${pidfiles}
+    do
+        pids+="$(cat $pidfile) "
+    done
+
     for i in `seq 1 7`
     do
         rados -p $poolname rmsnap snap$i
@@ -683,14 +716,14 @@ EOF
 
     ERRORS=0
 
-    pidfile=$(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid')
-    pid=$(cat $pidfile)
-    if ! kill -0 $pid
-    then
-        echo "OSD crash occurred"
-        tail -100 $dir/osd.0.log
-        ERRORS=$(expr $ERRORS + 1)
-    fi
+    for pid in $pids
+    do
+        if ! kill -0 $pid
+        then
+            echo "OSD Crash occurred"
+            ERRORS=$(expr $ERRORS + 1)
+        fi
+    done
 
     kill_daemons $dir || return 1
 
@@ -739,8 +772,505 @@ EOF
     return 0
 }
 
+function _scrub_snaps_multi() {
+    local dir=$1
+    local poolname=test
+    local OBJS=15
+    local OSDS=2
+    local which=$2
+
+    TESTDATA="testdata.$$"
+
+    run_mon $dir a --osd_pool_default_size=$OSDS || return 1
+    run_mgr $dir x || return 1
+    for osd in $(seq 0 $(expr $OSDS - 1))
+    do
+      run_osd $dir $osd || return 1
+    done
+
+    # Create a pool with a single pg
+    create_pool $poolname 1 1
+    wait_for_clean || return 1
+    poolid=$(ceph osd dump | grep "^pool.*[']test[']" | awk '{ print $2 }')
+
+    dd if=/dev/urandom of=$TESTDATA bs=1032 count=1
+    for i in `seq 1 $OBJS`
+    do
+        rados -p $poolname put obj${i} $TESTDATA
+    done
+
+    local primary=$(get_primary $poolname obj1)
+    local replica=$(get_not_primary $poolname obj1)
+
+    eval create_scenario $dir $poolname $TESTDATA \$$which
+
+    rm -f $TESTDATA
+
+    for osd in $(seq 0 $(expr $OSDS - 1))
+    do
+      run_osd $dir $osd || return 1
+    done
+
+    local pgid="${poolid}.0"
+    if ! pg_scrub "$pgid" ; then
+        cat $dir/osd.0.log
+        return 1
+    fi
+
+    test "$(grep "_scan_snaps start" $dir/osd.${primary}.log | wc -l)" -gt "3" || return 1
+    test "$(grep "_scan_snaps start" $dir/osd.${replica}.log | wc -l)" -gt "3" || return 1
+
+    rados list-inconsistent-pg $poolname > $dir/json || return 1
+    # Check pg count
+    test $(jq '. | length' $dir/json) = "1" || return 1
+    # Check pgid
+    test $(jq -r '.[0]' $dir/json) = $pgid || return 1
+
+    rados list-inconsistent-obj $pgid --format=json-pretty
+
+    rados list-inconsistent-snapset $pgid > $dir/json || return 1
+
+    # Since all of the snapshots on the primary is consistent there are no errors here
+    if [ $which = "replica" ];
+    then
+        scruberrors="21"
+        jq "$jqfilter" << EOF | python -c "$sortkeys" > $dir/checkcsjson
+{
+    "epoch": 23,
+    "inconsistents": []
+}
+EOF
+
+else
+        scruberrors="33"
+        jq "$jqfilter" << EOF | python -c "$sortkeys" > $dir/checkcsjson
+{
+    "epoch": 23,
+    "inconsistents": [
+        {
+            "name": "obj10",
+            "nspace": "",
+            "locator": "",
+            "snap": 1,
+            "errors": [
+                "size_mismatch"
+            ]
+        },
+        {
+            "name": "obj11",
+            "nspace": "",
+            "locator": "",
+            "snap": 1,
+            "errors": [
+                "headless"
+            ]
+        },
+        {
+            "name": "obj14",
+            "nspace": "",
+            "locator": "",
+            "snap": 1,
+            "errors": [
+                "size_mismatch"
+            ]
+        },
+        {
+            "name": "obj6",
+            "nspace": "",
+            "locator": "",
+            "snap": 1,
+            "errors": [
+                "headless"
+            ]
+        },
+        {
+            "name": "obj7",
+            "nspace": "",
+            "locator": "",
+            "snap": 1,
+            "errors": [
+                "headless"
+            ]
+        },
+        {
+            "name": "obj9",
+            "nspace": "",
+            "locator": "",
+            "snap": 1,
+            "errors": [
+                "size_mismatch"
+            ]
+        },
+        {
+            "name": "obj5",
+            "nspace": "",
+            "locator": "",
+            "snap": 7,
+            "errors": [
+                "info_missing",
+                "headless"
+            ]
+        },
+        {
+            "name": "obj10",
+            "nspace": "",
+            "locator": "",
+            "snap": "head",
+            "snapset": {
+                "head_exists": 1,
+                "snap_context": {
+                    "seq": 1,
+                    "snaps": [
+                        1
+                    ]
+                },
+                "clones": [
+                    {
+                        "snap": 1,
+                        "size": 1032,
+                        "overlap": "????",
+                        "snaps": [
+                            1
+                        ]
+                    }
+                ]
+            },
+            "errors": []
+        },
+        {
+            "name": "obj11",
+            "nspace": "",
+            "locator": "",
+            "snap": "head",
+            "snapset": {
+                "head_exists": 1,
+                "snap_context": {
+                    "seq": 1,
+                    "snaps": [
+                        1
+                    ]
+                },
+                "clones": []
+            },
+            "errors": [
+                "extra_clones"
+            ],
+            "extra clones": [
+                1
+            ]
+        },
+        {
+           "errors": [
+             "head_mismatch"
+           ],
+           "locator": "",
+           "name": "obj12",
+           "nspace": "",
+           "snap": "head",
+           "snapset": {
+             "clones": [
+               {
+                 "overlap": "[]",
+                 "size": 1032,
+                 "snap": 1,
+                 "snaps": [
+                   1
+                 ]
+               }
+             ],
+             "head_exists": 0,
+              "snap_context": {
+              "seq": 1,
+              "snaps": [
+                1
+              ]
+            }
+          }
+        },
+        {
+            "name": "obj14",
+            "nspace": "",
+            "locator": "",
+            "snap": "head",
+            "snapset": {
+                "head_exists": 1,
+                "snap_context": {
+                    "seq": 1,
+                    "snaps": [
+                        1
+                    ]
+                },
+                "clones": [
+                    {
+                        "snap": 1,
+                        "size": 1033,
+                        "overlap": "[]",
+                        "snaps": [
+                            1
+                        ]
+                    }
+                ]
+            },
+            "errors": []
+        },
+        {
+            "name": "obj5",
+            "nspace": "",
+            "locator": "",
+            "snap": "head",
+            "snapset": {
+                "head_exists": 1,
+                "snap_context": {
+                    "seq": 6,
+                    "snaps": [
+                        6,
+                        5,
+                        4,
+                        3,
+                        2,
+                        1
+                    ]
+                },
+                "clones": [
+                    {
+                        "snap": 1,
+                        "size": 1032,
+                        "overlap": "[]",
+                        "snaps": [
+                            1
+                        ]
+                    },
+                    {
+                        "snap": 2,
+                        "size": 256,
+                        "overlap": "[]",
+                        "snaps": [
+                            2
+                        ]
+                    },
+                    {
+                        "snap": 4,
+                        "size": 512,
+                        "overlap": "[]",
+                        "snaps": [
+                            4,
+                            3
+                        ]
+                    },
+                    {
+                        "snap": 6,
+                        "size": 1024,
+                        "overlap": "[]",
+                        "snaps": [
+                            6,
+                            5
+                        ]
+                    }
+                ]
+            },
+            "errors": [
+                "extra_clones"
+            ],
+            "extra clones": [
+                7
+            ]
+        },
+        {
+            "name": "obj6",
+            "nspace": "",
+            "locator": "",
+            "snap": "head",
+            "snapset": {
+                "head_exists": 1,
+                "snap_context": {
+                    "seq": 1,
+                    "snaps": [
+                        1
+                    ]
+                },
+                "clones": []
+            },
+            "errors": [
+                "extra_clones"
+            ],
+            "extra clones": [
+                1
+            ]
+        },
+        {
+            "name": "obj7",
+            "nspace": "",
+            "locator": "",
+            "snap": "head",
+            "snapset": {
+                "head_exists": 0,
+                "snap_context": {
+                    "seq": 0,
+                    "snaps": []
+                },
+                "clones": []
+            },
+            "errors": [
+                "head_mismatch",
+                "extra_clones"
+            ],
+            "extra clones": [
+                1
+            ]
+        },
+        {
+            "name": "obj8",
+            "nspace": "",
+            "locator": "",
+            "snap": "head",
+            "snapset": {
+                "head_exists": 1,
+                "snap_context": {
+                    "seq": 0,
+                    "snaps": [
+                        1
+                    ]
+                },
+                "clones": [
+                    {
+                        "snap": 1,
+                        "size": 1032,
+                        "overlap": "[]",
+                        "snaps": [
+                            1
+                        ]
+                    }
+                ]
+            },
+            "errors": [
+                "snapset_error"
+            ]
+        },
+        {
+            "name": "obj9",
+            "nspace": "",
+            "locator": "",
+            "snap": "head",
+            "snapset": {
+                "head_exists": 1,
+                "snap_context": {
+                    "seq": 1,
+                    "snaps": [
+                        1
+                    ]
+                },
+                "clones": [
+                    {
+                        "snap": 1,
+                        "size": "????",
+                        "overlap": "[]",
+                        "snaps": [
+                            1
+                        ]
+                    }
+                ]
+            },
+            "errors": []
+        }
+    ]
+}
+EOF
+fi
+
+    jq "$jqfilter" $dir/json | python -c "$sortkeys" > $dir/csjson
+    diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
+    if test $getjson = "yes"
+    then
+        jq '.' $dir/json > save1.json
+    fi
+
+    if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
+    then
+      jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-snap.json || return 1
+    fi
+
+    pidfiles=$(find $dir 2>/dev/null | grep 'osd[^/]*\.pid')
+    pids=""
+    for pidfile in ${pidfiles}
+    do
+        pids+="$(cat $pidfile) "
+    done
+
+    # When removing snapshots with a corrupt replica, it crashes.
+    # See http://tracker.ceph.com/issues/23875
+    if [ $which = "primary" ];
+    then
+        for i in `seq 1 7`
+        do
+            rados -p $poolname rmsnap snap$i
+        done
+    fi
+
+    ERRORS=0
+
+    for pid in $pids
+    do
+        if ! kill -0 $pid
+        then
+            echo "OSD Crash occurred"
+            ERRORS=$(expr $ERRORS + 1)
+        fi
+    done
+
+    kill_daemons $dir || return 1
+
+    declare -a err_strings
+    err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] missing .*:::obj4:7"
+    err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1]: soid .*:::obj3:head size 3840 != size 768 from auth oi"
+    err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] missing .*:::obj5:1"
+    err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] missing .*:::obj5:2"
+    err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1]: soid .*:::obj5:4 size 4608 != size 512 from auth oi"
+    err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid .*:::obj5:7: failed to pick suitable object info"
+    err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] missing .*:::obj1:head"
+    err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub ${scruberrors} errors"
+
+    for err_string in "${err_strings[@]}"
+    do
+        if ! grep "$err_string" $dir/osd.${primary}.log > /dev/null;
+        then
+            echo "Missing log message '$err_string'"
+            ERRORS=$(expr $ERRORS + 1)
+        fi
+    done
+
+    if [ $ERRORS != "0" ];
+    then
+        echo "TEST FAILED WITH $ERRORS ERRORS"
+        return 1
+    fi
+
+    echo "TEST PASSED"
+    return 0
+}
+
+function TEST_scrub_snaps_replica() {
+    local dir=$1
+    ORIG_ARGS=$CEPH_ARGS
+    CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=3"
+    _scrub_snaps_multi $dir replica
+    err=$?
+    CEPH_ARGS=$ORIG_ARGS
+    return $err
+}
+
+function TEST_scrub_snaps_primary() {
+    local dir=$1
+    ORIG_ARGS=$CEPH_ARGS
+    CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=3"
+    _scrub_snaps_multi $dir primary
+    err=$?
+    CEPH_ARGS=$ORIG_ARGS
+    return $err
+}
+
 main osd-scrub-snaps "$@"
 
 # Local Variables:
-# compile-command: "cd ../.. ; make -j4 && \
-#    test/osd/osd-scrub-snaps.sh"
+# compile-command: "cd build ; make -j4 && \
+#    ../qa/run-standalone.sh osd-scrub-snaps.sh"
+# End: