ceph/qa/standalone/scrub/osd-scrub-distrust.sh

   1 #!/bin/bash -x
   2 #
   3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
   4 #
   5 # Author: Loic Dachary <loic@dachary.org>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  18
  19 # Test development and debugging
  20 # Set to "yes" in order to ignore diff errors and save results to update test
  21 getjson="yes"
  22
  23 # Filter out mtime and local_mtime dates, version, prior_version and last_reqid (client) from any object_info.
  24 jqfilter='def walk(f):
  25   . as $in
  26   | if type == "object" then
  27       reduce keys[] as $key
  28         ( {}; . + { ($key):  ($in[$key] | walk(f)) } ) | f
  29     elif type == "array" then map( walk(f) ) | f
  30     else f
  31     end;
  32 walk(if type == "object" then del(.mtime) else . end)
  33 | walk(if type == "object" then del(.local_mtime) else . end)
  34 | walk(if type == "object" then del(.last_reqid) else . end)
  35 | walk(if type == "object" then del(.version) else . end)
  36 | walk(if type == "object" then del(.prior_version) else . end)
  37 | walk(if type == "object" then del(.redirect_target) else . end)
  38 | walk(if type == "object" then del(.legacy_snaps) else . end)'
  39
  40 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
  41
  42 function run() {
  43     local dir=$1
  44     shift
  45
  46     export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
  47     export CEPH_ARGS
  48     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  49     CEPH_ARGS+="--mon-host=$CEPH_MON "
  50     CEPH_ARGS+="--osd-distrust-data-digest=true "
  51
  52     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  53     for func in $funcs ; do
  54         $func $dir || return 1
  55     done
  56 }
  57
  58 function add_something() {
  59     local dir=$1
  60     local poolname=$2
  61     local obj=${3:-SOMETHING}
  62     local scrub=${4:-noscrub}
  63
  64     if [ "$scrub" = "noscrub" ];
  65     then
  66         ceph osd set noscrub || return 1
  67         ceph osd set nodeep-scrub || return 1
  68     else
  69         ceph osd unset noscrub || return 1
  70         ceph osd unset nodeep-scrub || return 1
  71     fi
  72
  73     local payload=ABCDEF
  74     echo $payload > $dir/ORIGINAL
  75     rados --pool $poolname put $obj $dir/ORIGINAL || return 1
  76 }
  77
  78 #
  79 # Test automatic repair with distrust set
  80 #
  81 function TEST_distrust_scrub_replicated() {
  82     local dir=$1
  83     local poolname=dsr_pool
  84     local total_objs=2
  85
  86     setup $dir || return 1
  87     run_mon $dir a --osd_pool_default_size=2 || return 1
  88     run_mgr $dir x || return 1
  89     run_osd $dir 0 || return 1
  90     run_osd $dir 1 || return 1
  91     create_rbd_pool || return 1
  92     wait_for_clean || return 1
  93
  94     create_pool foo 1 || return 1
  95     create_pool $poolname 1 1 || return 1
  96     wait_for_clean || return 1
  97
  98     for i in $(seq 1 $total_objs) ; do
  99         objname=ROBJ${i}
 100         add_something $dir $poolname $objname || return 1
 101     done
 102
 103     local pg=$(get_pg $poolname ROBJ0)
 104
 105     for i in $(seq 1 $total_objs) ; do
 106         objname=ROBJ${i}
 107
 108         case $i in
 109         1)
 110             # Deep-scrub only (all replicas are diffent than the object info
 111            local payload=XROBJ1
 112            echo $payload > $dir/new.ROBJ1
 113            objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ1 || return 1
 114            objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ1 || return 1
 115            ;;
 116
 117         2)
 118             # Deep-scrub only (all replicas are diffent than the object info
 119            local payload=XROBJ2
 120            echo $payload > $dir/new.ROBJ2
 121            objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ2 || return 1
 122            objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ2 || return 1
 123            # Make one replica have a different object info, so a full repair must happen too
 124            objectstore_tool $dir 0 $objname corrupt-info || return 1
 125            ;;
 126         esac
 127     done
 128
 129     # This should fix the data_digest because osd-distrust-data-digest is true
 130     pg_deep_scrub $pg
 131
 132     # This hangs if the scrub didn't repair the data_digest
 133     timeout 30 rados -p $poolname get ROBJ1 $dir/robj1.out || return 1
 134     diff -q $dir/new.ROBJ1 $dir/robj1.out || return 1
 135     rm -f $dir/new.ROBJ1 $dir/robj1.out || return 1
 136
 137     rados list-inconsistent-pg $poolname > $dir/json || return 1
 138     # Check pg count
 139     test $(jq '. | length' $dir/json) = "1" || return 1
 140     # Check pgid
 141     test $(jq -r '.[0]' $dir/json) = $pg || return 1
 142
 143     rados list-inconsistent-obj $pg > $dir/json || return 1
 144     # Get epoch for repair-get requests
 145     epoch=$(jq .epoch $dir/json)
 146
 147     jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson
 148 {
 149   "inconsistents": [
 150     {
 151       "shards": [
 152         {
 153           "object_info": {
 154             "watchers": {},
 155             "manifest": {
 156               "redirect_target": {
 157                 "namespace": "",
 158                 "pool": -9223372036854776000,
 159                 "max": 0,
 160                 "hash": 0,
 161                 "snapid": 0,
 162                 "key": "",
 163                 "oid": ""
 164               },
 165               "type": 0
 166             },
 167             "alloc_hint_flags": 255,
 168             "expected_write_size": 0,
 169             "local_mtime": "2018-07-24 15:05:56.027234",
 170             "mtime": "2018-07-24 15:05:56.021775",
 171             "size": 7,
 172             "user_version": 2,
 173             "last_reqid": "client.4137.0:1",
 174             "prior_version": "0'0",
 175             "version": "23'2",
 176             "oid": {
 177               "namespace": "",
 178               "pool": 3,
 179               "max": 0,
 180               "hash": 2026323607,
 181               "snapid": -2,
 182               "key": "",
 183               "oid": "ROBJ2"
 184             },
 185             "lost": 0,
 186             "flags": [
 187               "dirty",
 188               "data_digest"
 189             ],
 190             "legacy_snaps": [],
 191             "truncate_seq": 0,
 192             "truncate_size": 0,
 193             "data_digest": "0x2ddbf8f5",
 194             "omap_digest": "0xffffffff",
 195             "expected_object_size": 0
 196           },
 197           "data_digest": "0x0bb7ab52",
 198           "omap_digest": "0xffffffff",
 199           "size": 7,
 200           "errors": [],
 201           "primary": false,
 202           "osd": 0
 203         },
 204         {
 205           "object_info": {
 206             "watchers": {},
 207             "manifest": {
 208               "redirect_target": {
 209                 "namespace": "",
 210                 "pool": -9223372036854776000,
 211                 "max": 0,
 212                 "hash": 0,
 213                 "snapid": 0,
 214                 "key": "",
 215                 "oid": ""
 216               },
 217               "type": 0
 218             },
 219             "alloc_hint_flags": 0,
 220             "expected_write_size": 0,
 221             "local_mtime": "2018-07-24 15:05:56.027234",
 222             "mtime": "2018-07-24 15:05:56.021775",
 223             "size": 7,
 224             "user_version": 2,
 225             "last_reqid": "client.4137.0:1",
 226             "prior_version": "0'0",
 227             "version": "23'2",
 228             "oid": {
 229               "namespace": "",
 230               "pool": 3,
 231               "max": 0,
 232               "hash": 2026323607,
 233               "snapid": -2,
 234               "key": "",
 235               "oid": "ROBJ2"
 236             },
 237             "lost": 0,
 238             "flags": [
 239               "dirty",
 240               "data_digest"
 241             ],
 242             "legacy_snaps": [],
 243             "truncate_seq": 0,
 244             "truncate_size": 0,
 245             "data_digest": "0x2ddbf8f5",
 246             "omap_digest": "0xffffffff",
 247             "expected_object_size": 0
 248           },
 249           "data_digest": "0x0bb7ab52",
 250           "omap_digest": "0xffffffff",
 251           "size": 7,
 252           "errors": [],
 253           "primary": true,
 254           "osd": 1
 255         }
 256       ],
 257       "selected_object_info": {
 258         "watchers": {},
 259         "manifest": {
 260           "redirect_target": {
 261             "namespace": "",
 262             "pool": -9223372036854776000,
 263             "max": 0,
 264             "hash": 0,
 265             "snapid": 0,
 266             "key": "",
 267             "oid": ""
 268           },
 269           "type": 0
 270         },
 271         "alloc_hint_flags": 0,
 272         "expected_write_size": 0,
 273         "local_mtime": "2018-07-24 15:05:56.027234",
 274         "mtime": "2018-07-24 15:05:56.021775",
 275         "size": 7,
 276         "user_version": 2,
 277         "last_reqid": "client.4137.0:1",
 278         "prior_version": "0'0",
 279         "version": "23'2",
 280         "oid": {
 281           "namespace": "",
 282           "pool": 3,
 283           "max": 0,
 284           "hash": 2026323607,
 285           "snapid": -2,
 286           "key": "",
 287           "oid": "ROBJ2"
 288         },
 289         "lost": 0,
 290         "flags": [
 291           "dirty",
 292           "data_digest"
 293         ],
 294         "legacy_snaps": [],
 295         "truncate_seq": 0,
 296         "truncate_size": 0,
 297         "data_digest": "0x2ddbf8f5",
 298         "omap_digest": "0xffffffff",
 299         "expected_object_size": 0
 300       },
 301       "union_shard_errors": [],
 302       "errors": [
 303         "object_info_inconsistency"
 304       ],
 305       "object": {
 306         "version": 2,
 307         "snap": "head",
 308         "locator": "",
 309         "nspace": "",
 310         "name": "ROBJ2"
 311       }
 312     }
 313   ],
 314   "epoch": 42
 315 }
 316 EOF
 317
 318     jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson
 319     diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
 320     if test $getjson = "yes"
 321     then
 322         jq '.' $dir/json > save1.json
 323     fi
 324
 325     if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
 326     then
 327       jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
 328     fi
 329
 330     repair $pg
 331     wait_for_clean
 332
 333     timeout 30 rados -p $poolname get ROBJ2 $dir/robj2.out || return 1
 334     diff -q $dir/new.ROBJ2 $dir/robj2.out || return 1
 335     rm -f $dir/new.ROBJ2 $dir/robj2.out || return 1
 336
 337     rados rmpool $poolname $poolname --yes-i-really-really-mean-it
 338     teardown $dir || return 1
 339 }
 340
 341 main osd-scrub-distrust "$@"
 342
 343 # Local Variables:
 344 # compile-command: "cd build ; make -j4 && \
 345 #    ../qa/run-standalone.sh osd-scrub-distrust"
 346 # End: