update sources to v12.2.0

[ceph.git] / ceph / qa / standalone / scrub / osd-scrub-repair.sh

diff --git a/ceph/qa/standalone/scrub/osd-scrub-repair.sh b/ceph/qa/standalone/scrub/osd-scrub-repair.sh
index 28db9476b1b83e08437f53a09d6b67a98c5b5358..2aaaebd6e9701f1d7b6b3368c6be9c8811bce266 100755 (executable)
--- a/ceph/qa/standalone/scrub/osd-scrub-repair.sh
+++ b/ceph/qa/standalone/scrub/osd-scrub-repair.sh
@@ -30,7 +30,11 @@ fi
 getjson="no"
 
 # Ignore the epoch and filter out the attr '_' value because it has date information and won't match
-jqfilter='.inconsistents | (.[].shards[].attrs[]? | select(.name == "_") | .value) |= "----Stripped-by-test----"'
+if [ "$(jq --version 2>&1 | awk '{ print $3}')" = "1.3" ]; then # Not sure all versions that apply here
+    jqfilter='.inconsistents | (.[].shards[].attrs[] | select(.name == "_") | .value) |= "----Stripped-by-test----"'
+else
+    jqfilter='.inconsistents | (.[].shards[].attrs[]? | select(.name == "_") | .value) |= "----Stripped-by-test----"'
+fi
 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
 
 # Remove items are not consistent across runs, the pg interval and client
@@ -183,7 +187,7 @@ function create_ec_pool() {
 
     ceph osd erasure-code-profile set myprofile crush-failure-domain=osd $3 $4 $5 $6 $7 || return 1
 
-    ceph osd pool create "$poolname" 1 1 erasure myprofile || return 1
+    create_pool "$poolname" 1 1 erasure myprofile || return 1
 
     if [ "$allow_overwrites" = "true" ]; then
         ceph osd pool set "$poolname" allow_ec_overwrites true || return 1
@@ -370,11 +374,11 @@ function unfound_erasure_coded() {
     #
     # it may take a bit to appear due to mon/mgr asynchrony
     for f in `seq 1 60`; do
-       ceph -s | grep "1/1 unfound" && break
+       ceph -s | grep "1/1 objects unfound" && break
        sleep 1
     done
     ceph -s|grep "4 osds: 4 up, 4 in" || return 1
-    ceph -s|grep "1/1 unfound" || return 1
+    ceph -s|grep "1/1 objects unfound" || return 1
 
     teardown $dir || return 1
 }
@@ -477,7 +481,7 @@ function TEST_list_missing_erasure_coded_overwrites() {
 function TEST_corrupt_scrub_replicated() {
     local dir=$1
     local poolname=csr_pool
-    local total_objs=15
+    local total_objs=16
 
     setup $dir || return 1
     run_mon $dir a --osd_pool_default_size=2 || return 1
@@ -487,8 +491,8 @@ function TEST_corrupt_scrub_replicated() {
     create_rbd_pool || return 1
     wait_for_clean || return 1
 
-    ceph osd pool create foo 1 || return 1
-    ceph osd pool create $poolname 1 1 || return 1
+    create_pool foo 1 || return 1
+    create_pool $poolname 1 1 || return 1
     wait_for_clean || return 1
 
     for i in $(seq 1 $total_objs) ; do
@@ -593,22 +597,22 @@ function TEST_corrupt_scrub_replicated() {
 
         15)
             objectstore_tool $dir $osd $objname rm-attr _ || return 1
+            ;;
+
+        16)
+            objectstore_tool $dir 0 $objname rm-attr snapset || return 1
+            echo -n bad-val > $dir/bad-val
+            objectstore_tool $dir 1 $objname set-attr snapset $dir/bad-val || return 1
 
         esac
     done
 
     local pg=$(get_pg $poolname ROBJ0)
 
-    set_config osd 0 filestore_debug_inject_read_err true || return 1
-    set_config osd 1 filestore_debug_inject_read_err true || return 1
-    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \
-             injectdataerr $poolname ROBJ11 || return 1
-    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \
-             injectmdataerr $poolname ROBJ12 || return 1
-    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \
-             injectmdataerr $poolname ROBJ13 || return 1
-    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \
-             injectdataerr $poolname ROBJ13 || return 1
+    inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
+    inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
+    inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
+    inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
 
     pg_scrub $pg
 
@@ -630,19 +634,23 @@ function TEST_corrupt_scrub_replicated() {
         {
           "size": 7,
           "errors": [],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "size": 9,
           "errors": [
-            "size_mismatch_oi"
+            "size_mismatch_oi",
+            "obj_size_oi_mismatch"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])",
       "union_shard_errors": [
-        "size_mismatch_oi"
+        "size_mismatch_oi",
+        "obj_size_oi_mismatch"
       ],
       "errors": [
         "size_mismatch"
@@ -661,12 +669,14 @@ function TEST_corrupt_scrub_replicated() {
           "errors": [
             "stat_error"
           ],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "size": 7,
           "errors": [],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])",
@@ -688,12 +698,14 @@ function TEST_corrupt_scrub_replicated() {
           "errors": [
             "stat_error"
           ],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "size": 7,
           "errors": [],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:d60617f9:::ROBJ13:head(47'55 osd.0.0:54 dirty|omap|data_digest|omap_digest s 7 uv 39 dd 2ddbf8f5 od 6441854d alloc_hint [0 0 0])",
@@ -713,17 +725,38 @@ function TEST_corrupt_scrub_replicated() {
       "shards": [
         {
           "size": 7,
+          "attrs": [
+            {
+              "Base64": false,
+              "value": "",
+              "name": "_"
+            },
+            {
+              "Base64": true,
+              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "name": "snapset"
+            }
+          ],
           "errors": [
             "oi_attr_corrupted"
           ],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "size": 7,
+          "attrs": [
+            {
+              "Base64": true,
+              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "name": "snapset"
+            }
+          ],
           "errors": [
             "oi_attr_missing"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "union_shard_errors": [
@@ -756,7 +789,8 @@ function TEST_corrupt_scrub_replicated() {
           ],
           "size": 7,
           "errors": [],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "attrs": [
@@ -770,16 +804,15 @@ function TEST_corrupt_scrub_replicated() {
           "errors": [
             "oi_attr_missing"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])",
       "union_shard_errors": [
         "oi_attr_missing"
       ],
-      "errors": [
-        "attr_name_mismatch"
-      ],
+      "errors": [],
       "object": {
         "version": 45,
         "snap": "head",
@@ -788,18 +821,71 @@ function TEST_corrupt_scrub_replicated() {
         "name": "ROBJ15"
       }
     },
+    {
+      "errors": [],
+      "object": {
+      "locator": "",
+      "name": "ROBJ16",
+      "nspace": "",
+      "snap": "head",
+      "version": 0
+       },
+        "shards": [
+      {
+        "attrs": [
+          {
+            "Base64": true,
+            "name": "_",
+            "value": ""
+          }
+        ],
+        "errors": [
+          "ss_attr_missing"
+        ],
+        "osd": 0,
+        "primary": false,
+        "size": 7
+      },
+      {
+        "attrs": [
+          {
+            "Base64": true,
+            "name": "_",
+            "value": ""
+          },
+          {
+            "Base64": false,
+            "name": "snapset",
+            "value": "bad-val"
+          }
+        ],
+        "errors": [
+          "ss_attr_corrupted"
+        ],
+        "osd": 1,
+        "primary": true,
+        "size": 7
+      }
+      ],
+      "union_shard_errors": [
+        "ss_attr_missing",
+        "ss_attr_corrupted"
+      ]
+    },
     {
       "shards": [
         {
           "size": 7,
           "errors": [],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "errors": [
             "missing"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])",
@@ -842,7 +928,8 @@ function TEST_corrupt_scrub_replicated() {
           ],
           "size": 7,
           "errors": [],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "attrs": [
@@ -869,17 +956,18 @@ function TEST_corrupt_scrub_replicated() {
           ],
           "size": 7,
           "errors": [],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
-      "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 62 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
+      "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 66 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
       "union_shard_errors": [],
       "errors": [
         "attr_value_mismatch",
         "attr_name_mismatch"
       ],
       "object": {
-        "version": 62,
+        "version": 66,
         "snap": "head",
         "locator": "",
         "nspace": "",
@@ -889,50 +977,31 @@ function TEST_corrupt_scrub_replicated() {
     {
       "shards": [
         {
-          "attrs": [
-            {
-              "Base64": true,
-              "value": "",
-              "name": "_"
-            },
-            {
-              "Base64": true,
-              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
-              "name": "snapset"
-            }
-          ],
-          "object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 63 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
+          "object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 67 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
           "size": 1,
           "errors": [],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
-          "attrs": [
-            {
-              "Base64": true,
-              "value": "",
-              "name": "_"
-            },
-            {
-              "Base64": true,
-              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
-              "name": "snapset"
-            }
-          ],
           "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])",
           "size": 1,
-          "errors": [],
-          "osd": 1
+          "errors": [
+            "obj_size_oi_mismatch"
+          ],
+          "osd": 1,
+          "primary": true
         }
       ],
-      "selected_object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 63 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
-      "union_shard_errors": [],
+      "selected_object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 67 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
+      "union_shard_errors": [
+         "obj_size_oi_mismatch"
+      ],
       "errors": [
-        "object_info_inconsistency",
-        "attr_value_mismatch"
+        "object_info_inconsistency"
       ],
       "object": {
-        "version": 63,
+        "version": 67,
         "snap": "head",
         "locator": "",
         "nspace": "",
@@ -971,16 +1040,10 @@ EOF
     objectstore_tool $dir 1 $objname set-attr _ $dir/oi
     rm $dir/oi
 
-    set_config osd 0 filestore_debug_inject_read_err true || return 1
-    set_config osd 1 filestore_debug_inject_read_err true || return 1
-    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \
-             injectdataerr $poolname ROBJ11 || return 1
-    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \
-             injectmdataerr $poolname ROBJ12 || return 1
-    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \
-             injectmdataerr $poolname ROBJ13 || return 1
-    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \
-             injectdataerr $poolname ROBJ13 || return 1
+    inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
+    inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
+    inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
+    inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
     pg_deep_scrub $pg
 
     rados list-inconsistent-pg $poolname > $dir/json || return 1
@@ -1003,7 +1066,8 @@ EOF
           "omap_digest": "0xf5fba2c6",
           "size": 7,
           "errors": [],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "data_digest": "0x2d4a11c2",
@@ -1011,15 +1075,18 @@ EOF
           "size": 9,
           "errors": [
             "data_digest_mismatch_oi",
-            "size_mismatch_oi"
+            "size_mismatch_oi",
+            "obj_size_oi_mismatch"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])",
       "union_shard_errors": [
         "data_digest_mismatch_oi",
-        "size_mismatch_oi"
+        "size_mismatch_oi",
+        "obj_size_oi_mismatch"
       ],
       "errors": [
         "data_digest_mismatch",
@@ -1042,7 +1109,8 @@ EOF
           "errors": [
             "omap_digest_mismatch_oi"
           ],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "data_digest": "0x2ddbf8f5",
@@ -1051,7 +1119,8 @@ EOF
           "errors": [
             "omap_digest_mismatch_oi"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:b1f19cbd:::ROBJ10:head(47'51 osd.0.0:50 dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [0 0 0])",
@@ -1074,14 +1143,16 @@ EOF
           "omap_digest": "0xa03cef03",
           "size": 7,
           "errors": [],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "size": 7,
           "errors": [
             "read_error"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:87abbf36:::ROBJ11:head(47'48 osd.0.0:47 dirty|omap|data_digest|omap_digest s 7 uv 33 dd 2ddbf8f5 od a03cef03 alloc_hint [0 0 0])",
@@ -1103,14 +1174,16 @@ EOF
           "errors": [
             "stat_error"
           ],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "data_digest": "0x2ddbf8f5",
           "omap_digest": "0x067f306a",
           "size": 7,
           "errors": [],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])",
@@ -1132,14 +1205,16 @@ EOF
           "errors": [
             "stat_error"
           ],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "size": 7,
           "errors": [
             "read_error"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "union_shard_errors": [
@@ -1158,22 +1233,43 @@ EOF
     {
       "shards": [
         {
+          "attrs": [
+            {
+              "Base64": false,
+              "value": "",
+              "name": "_"
+            },
+            {
+              "Base64": true,
+              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "name": "snapset"
+            }
+          ],
           "data_digest": "0x2ddbf8f5",
           "omap_digest": "0x4f14f849",
           "size": 7,
           "errors": [
             "oi_attr_corrupted"
           ],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
+          "attrs": [
+            {
+              "Base64": true,
+              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "name": "snapset"
+            }
+          ],
           "data_digest": "0x2ddbf8f5",
           "omap_digest": "0x4f14f849",
           "size": 7,
           "errors": [
             "oi_attr_missing"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "union_shard_errors": [
@@ -1208,7 +1304,8 @@ EOF
           "omap_digest": "0x2d2a4d6e",
           "size": 7,
           "errors": [],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "attrs": [
@@ -1224,16 +1321,15 @@ EOF
           "errors": [
             "oi_attr_missing"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])",
       "union_shard_errors": [
         "oi_attr_missing"
       ],
-      "errors": [
-        "attr_name_mismatch"
-      ],
+      "errors": [],
       "object": {
         "version": 45,
         "snap": "head",
@@ -1242,6 +1338,61 @@ EOF
         "name": "ROBJ15"
       }
     },
+    {
+      "errors": [],
+      "object": {
+      "locator": "",
+      "name": "ROBJ16",
+      "nspace": "",
+      "snap": "head",
+      "version": 0
+       },
+        "shards": [
+      {
+        "attrs": [
+          {
+            "Base64": true,
+            "name": "_",
+            "value": ""
+          }
+        ],
+        "data_digest": "0x2ddbf8f5",
+        "errors": [
+          "ss_attr_missing"
+        ],
+        "omap_digest": "0x8b699207",
+        "osd": 0,
+        "primary": false,
+        "size": 7
+      },
+      {
+        "attrs": [
+          {
+            "Base64": true,
+            "name": "_",
+            "value": ""
+          },
+          {
+            "Base64": false,
+            "name": "snapset",
+            "value": "bad-val"
+          }
+        ],
+        "data_digest": "0x2ddbf8f5",
+        "errors": [
+          "ss_attr_corrupted"
+        ],
+        "omap_digest": "0x8b699207",
+        "osd": 1,
+        "primary": true,
+        "size": 7
+      }
+      ],
+      "union_shard_errors": [
+        "ss_attr_missing",
+        "ss_attr_corrupted"
+      ]
+    },
     {
       "shards": [
         {
@@ -1251,14 +1402,16 @@ EOF
           "errors": [
             "data_digest_mismatch_oi"
           ],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "data_digest": "0x2ddbf8f5",
           "omap_digest": "0xf8e11918",
           "size": 7,
           "errors": [],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:e97ce31e:::ROBJ2:head(47'56 osd.0.0:55 dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [0 0 0])",
@@ -1283,13 +1436,15 @@ EOF
           "omap_digest": "0x00b35dfd",
           "size": 7,
           "errors": [],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "errors": [
             "missing"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])",
@@ -1314,14 +1469,16 @@ EOF
           "errors": [
             "omap_digest_mismatch_oi"
           ],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "data_digest": "0x2ddbf8f5",
           "omap_digest": "0xe2d46ea4",
           "size": 7,
           "errors": [],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:f4981d31:::ROBJ4:head(47'58 osd.0.0:57 dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [0 0 0])",
@@ -1346,7 +1503,8 @@ EOF
           "omap_digest": "0x1a862a41",
           "size": 7,
           "errors": [],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "data_digest": "0x2ddbf8f5",
@@ -1355,7 +1513,8 @@ EOF
           "errors": [
             "omap_digest_mismatch_oi"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:f4bfd4d1:::ROBJ5:head(47'59 osd.0.0:58 dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [0 0 0])",
@@ -1382,14 +1541,16 @@ EOF
           "errors": [
             "omap_digest_mismatch_oi"
           ],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "data_digest": "0x2ddbf8f5",
           "omap_digest": "0x179c919f",
           "size": 7,
           "errors": [],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:a53c12e8:::ROBJ6:head(47'50 osd.0.0:49 dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [0 0 0])",
@@ -1414,7 +1575,8 @@ EOF
           "omap_digest": "0xefced57a",
           "size": 7,
           "errors": [],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "data_digest": "0x2ddbf8f5",
@@ -1423,7 +1585,8 @@ EOF
           "errors": [
             "omap_digest_mismatch_oi"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
       "selected_object_info": "3:8b55fa4b:::ROBJ7:head(47'49 osd.0.0:48 dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [0 0 0])",
@@ -1470,7 +1633,8 @@ EOF
           "omap_digest": "0xd6be81dc",
           "size": 7,
           "errors": [],
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "attrs": [
@@ -1499,17 +1663,18 @@ EOF
           "omap_digest": "0xd6be81dc",
           "size": 7,
           "errors": [],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
-      "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 62 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
+      "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 66 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
       "union_shard_errors": [],
       "errors": [
         "attr_value_mismatch",
         "attr_name_mismatch"
       ],
       "object": {
-        "version": 62,
+        "version": 66,
         "snap": "head",
         "locator": "",
         "nspace": "",
@@ -1519,54 +1684,35 @@ EOF
     {
       "shards": [
         {
-          "attrs": [
-            {
-              "Base64": true,
-              "value": "",
-              "name": "_"
-            },
-            {
-              "Base64": true,
-              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
-              "name": "snapset"
-            }
-          ],
           "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])",
           "data_digest": "0x1f26fb26",
           "omap_digest": "0x2eecc539",
           "size": 3,
-          "errors": [],
-          "osd": 0
+          "errors": [
+            "obj_size_oi_mismatch"
+          ],
+          "osd": 0,
+          "primary": false
         },
         {
-          "attrs": [
-            {
-              "Base64": true,
-              "value": "",
-              "name": "_"
-            },
-            {
-              "Base64": true,
-              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
-              "name": "snapset"
-            }
-          ],
-          "object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 64 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
+          "object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 68 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
           "data_digest": "0x1f26fb26",
           "omap_digest": "0x2eecc539",
           "size": 3,
           "errors": [],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         }
       ],
-      "selected_object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 64 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
-      "union_shard_errors": [],
+      "selected_object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 68 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
+      "union_shard_errors": [
+        "obj_size_oi_mismatch"
+      ],
       "errors": [
-        "object_info_inconsistency",
-        "attr_value_mismatch"
+        "object_info_inconsistency"
       ],
       "object": {
-        "version": 64,
+        "version": 68,
         "snap": "head",
         "locator": "",
         "nspace": "",
@@ -1615,7 +1761,7 @@ function corrupt_scrub_erasure() {
        fi
     done
     create_rbd_pool || return 1
-    ceph osd pool create foo 1
+    create_pool foo 1
 
     create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1
     wait_for_clean || return 1
@@ -1690,26 +1836,31 @@ function corrupt_scrub_erasure() {
           "size": 2048,
           "errors": [],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "size": 9,
           "shard": 0,
           "errors": [
-            "size_mismatch_oi"
+            "size_mismatch_oi",
+            "obj_size_oi_mismatch"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         },
         {
           "size": 2048,
           "shard": 1,
           "errors": [],
-          "osd": 2
+          "osd": 2,
+          "primary": false
         }
       ],
       "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
       "union_shard_errors": [
-        "size_mismatch_oi"
+        "size_mismatch_oi",
+        "obj_size_oi_mismatch"
       ],
       "errors": [
         "size_mismatch"
@@ -1728,20 +1879,23 @@ function corrupt_scrub_erasure() {
           "size": 2048,
           "errors": [],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "shard": 0,
           "errors": [
             "missing"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         },
         {
           "size": 2048,
           "shard": 1,
           "errors": [],
-          "osd": 2
+          "osd": 2,
+          "primary": false
         }
       ],
       "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
@@ -1790,10 +1944,12 @@ function corrupt_scrub_erasure() {
           "size": 2048,
           "errors": [],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "osd": 1,
+          "primary": true,
           "shard": 0,
           "errors": [],
           "size": 2048,
@@ -1827,6 +1983,7 @@ function corrupt_scrub_erasure() {
         },
         {
           "osd": 2,
+          "primary": false,
           "shard": 1,
           "errors": [],
           "size": 2048,
@@ -1879,26 +2036,31 @@ function corrupt_scrub_erasure() {
           "size": 2048,
           "errors": [],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "size": 4096,
           "shard": 0,
           "errors": [
-            "size_mismatch_oi"
+            "size_mismatch_oi",
+            "obj_size_oi_mismatch"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         },
         {
           "size": 2048,
           "shard": 1,
           "errors": [],
-          "osd": 2
+          "osd": 2,
+          "primary": false
         }
       ],
       "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
       "union_shard_errors": [
-        "size_mismatch_oi"
+        "size_mismatch_oi",
+        "obj_size_oi_mismatch"
       ],
       "errors": [
         "size_mismatch"
@@ -1953,16 +2115,19 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "size": 9,
           "shard": 0,
           "errors": [
             "read_error",
-            "size_mismatch_oi"
+            "size_mismatch_oi",
+            "obj_size_oi_mismatch"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         },
         {
           "data_digest": "0x00000000",
@@ -1970,13 +2135,15 @@ EOF
           "size": 2048,
           "shard": 1,
           "errors": [],
-          "osd": 2
+          "osd": 2,
+          "primary": false
         }
       ],
       "selected_object_info": "3:9175b684:::EOBJ1:head(27'1 client.4155.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
       "union_shard_errors": [
         "read_error",
-        "size_mismatch_oi"
+        "size_mismatch_oi",
+        "obj_size_oi_mismatch"
       ],
       "errors": [
         "size_mismatch"
@@ -1997,14 +2164,16 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "shard": 0,
           "errors": [
             "missing"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         },
         {
           "data_digest": "0x00000000",
@@ -2012,7 +2181,8 @@ EOF
           "size": 2048,
           "shard": 1,
           "errors": [],
-          "osd": 2
+          "osd": 2,
+          "primary": false
         }
       ],
       "selected_object_info": "3:b197b25d:::EOBJ3:head(41'3 client.4199.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
@@ -2063,7 +2233,8 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "attrs": [
@@ -2098,7 +2269,8 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 0,
-          "osd": 1
+          "osd": 1,
+          "primary": true
         },
         {
           "attrs": [
@@ -2133,7 +2305,8 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 1,
-          "osd": 2
+          "osd": 2,
+          "primary": false
         }
       ],
       "selected_object_info": "3:5e723e06:::EOBJ4:head(48'6 client.4223.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
@@ -2158,17 +2331,20 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "data_digest": "0x00000000",
           "omap_digest": "0xffffffff",
           "size": 4096,
           "errors": [
-            "size_mismatch_oi"
+            "size_mismatch_oi",
+            "obj_size_oi_mismatch"
           ],
           "shard": 0,
-          "osd": 1
+          "osd": 1,
+          "primary": true
         },
         {
           "data_digest": "0x00000000",
@@ -2176,12 +2352,14 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 1,
-          "osd": 2
+          "osd": 2,
+          "primary": false
         }
       ],
       "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4288.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
       "union_shard_errors": [
-        "size_mismatch_oi"
+        "size_mismatch_oi",
+        "obj_size_oi_mismatch"
       ],
       "errors": [
         "size_mismatch"
@@ -2212,16 +2390,19 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "size": 9,
           "shard": 0,
           "errors": [
             "read_error",
-            "size_mismatch_oi"
+            "size_mismatch_oi",
+            "obj_size_oi_mismatch"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         },
         {
           "data_digest": "0x04cfa72f",
@@ -2229,13 +2410,15 @@ EOF
           "size": 2048,
           "shard": 1,
           "errors": [],
-          "osd": 2
+          "osd": 2,
+          "primary": false
         }
       ],
       "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
       "union_shard_errors": [
         "read_error",
-        "size_mismatch_oi"
+        "size_mismatch_oi",
+        "obj_size_oi_mismatch"
       ],
       "errors": [
         "size_mismatch"
@@ -2256,7 +2439,8 @@ EOF
             "ec_hash_error"
           ],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "data_digest": "0x04cfa72f",
@@ -2264,7 +2448,8 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 0,
-          "osd": 1
+          "osd": 1,
+          "primary": true
         },
         {
           "data_digest": "0x04cfa72f",
@@ -2272,7 +2457,8 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 1,
-          "osd": 2
+          "osd": 2,
+          "primary": false
         }
       ],
       "selected_object_info": "3:9babd184:::EOBJ2:head(29'2 client.4217.0:1 dirty|data_digest|omap_digest s 7 uv 2 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
@@ -2296,10 +2482,12 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "osd": 1,
+          "primary": true,
           "shard": 0,
           "errors": [
             "missing"
@@ -2311,7 +2499,8 @@ EOF
           "size": 2048,
           "shard": 1,
           "errors": [],
-          "osd": 2
+          "osd": 2,
+          "primary": false
         }
       ],
       "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
@@ -2362,10 +2551,12 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "osd": 1,
+          "primary": true,
           "shard": 0,
           "errors": [],
           "size": 2048,
@@ -2401,6 +2592,7 @@ EOF
         },
         {
           "osd": 2,
+          "primary": false,
           "shard": 1,
           "errors": [],
           "size": 2048,
@@ -2457,16 +2649,19 @@ EOF
           "size": 2048,
           "errors": [],
           "shard": 2,
-          "osd": 0
+          "osd": 0,
+          "primary": false
         },
         {
           "size": 4096,
           "shard": 0,
           "errors": [
             "size_mismatch_oi",
-            "ec_size_error"
+            "ec_size_error",
+            "obj_size_oi_mismatch"
           ],
-          "osd": 1
+          "osd": 1,
+          "primary": true
         },
         {
           "data_digest": "0x04cfa72f",
@@ -2474,13 +2669,15 @@ EOF
           "size": 2048,
           "shard": 1,
           "errors": [],
-          "osd": 2
+          "osd": 2,
+          "primary": false
         }
       ],
       "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
       "union_shard_errors": [
         "size_mismatch_oi",
-        "ec_size_error"
+        "ec_size_error",
+        "obj_size_oi_mismatch"
       ],
       "errors": [
         "size_mismatch"
@@ -2543,13 +2740,14 @@ function TEST_periodic_scrub_replicated() {
     setup $dir || return 1
     run_mon $dir a --osd_pool_default_size=2 || return 1
     run_mgr $dir x || return 1
-    local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0"
+    local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
+    ceph_osd_args+="--osd_scrub_backoff_ratio=0"
     run_osd $dir 0 $ceph_osd_args || return 1
     run_osd $dir 1 $ceph_osd_args || return 1
     create_rbd_pool || return 1
     wait_for_clean || return 1
 
-    ceph osd pool create $poolname 1 1 || return 1
+    create_pool $poolname 1 1 || return 1
     wait_for_clean || return 1
 
     local osd=0
@@ -2573,6 +2771,7 @@ function TEST_periodic_scrub_replicated() {
     # Make sure bad object found
     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
 
+    flush_pg_stats
     local last_scrub=$(get_last_scrub_stamp $pg)
     # Fake a schedule scrub
     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
@@ -2589,10 +2788,10 @@ function TEST_periodic_scrub_replicated() {
     # Can't upgrade with this set
     ceph osd set nodeep-scrub
     # Let map change propagate to OSDs
-    sleep 2
+    flush pg_stats
+    sleep 5
 
     # Fake a schedule scrub
-    local last_scrub=$(get_last_scrub_stamp $pg)
     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \
              trigger_scrub $pg || return 1
     # Wait for schedule regular scrub
@@ -2609,12 +2808,9 @@ function TEST_periodic_scrub_replicated() {
     # Bad object still known
     rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
 
+    flush_pg_stats
     # Request a regular scrub and it will be done
-    local scrub_backoff_ratio=$(get_config osd ${primary} osd_scrub_backoff_ratio)
-    set_config osd ${primary} osd_scrub_backoff_ratio 0
     pg_scrub $pg
-    sleep 1
-    set_config osd ${primary} osd_scrub_backoff_ratio $scrub_backoff_ratio
     grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1
 
     # deep-scrub error is no longer present