ceph/qa/standalone/osd/osd-recovery-space.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Copyright (C) 2018 Red Hat <contact@redhat.com>
   4 #
   5 # Author: David Zafman <dzafman@redhat.com>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17
  18 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  19
  20 function run() {
  21     local dir=$1
  22     shift
  23
  24     export CEPH_MON="127.0.0.1:7221" # git grep '\<7221\>' : there must be only one
  25     export CEPH_ARGS
  26     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  27     CEPH_ARGS+="--mon-host=$CEPH_MON "
  28     CEPH_ARGS+="--osd_max_backfills=10 "
  29     export objects=600
  30     export poolprefix=test
  31
  32     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  33     for func in $funcs ; do
  34         setup $dir || return 1
  35         $func $dir || return 1
  36         teardown $dir || return 1
  37     done
  38 }
  39
  40
  41 function get_num_in_state() {
  42     local state=$1
  43     local expression
  44     expression+="select(contains(\"${state}\"))"
  45     ceph --format json pg dump pgs 2>/dev/null | \
  46         jq ".pg_stats | [.[] | .state | $expression] | length"
  47 }
  48
  49
  50 function wait_for_state() {
  51     local state=$1
  52     local num_in_state=-1
  53     local cur_in_state
  54     local -a delays=($(get_timeout_delays $2 5))
  55     local -i loop=0
  56
  57     flush_pg_stats || return 1
  58     while test $(get_num_pgs) == 0 ; do
  59         sleep 1
  60     done
  61
  62     while true ; do
  63         cur_in_state=$(get_num_in_state ${state})
  64         test $cur_in_state = "0" && break
  65         if test $cur_in_state != $num_in_state ; then
  66             loop=0
  67             num_in_state=$cur_in_state
  68         elif (( $loop >= ${#delays[*]} )) ; then
  69             ceph pg dump pgs
  70             return 1
  71         fi
  72         sleep ${delays[$loop]}
  73         loop+=1
  74     done
  75     return 0
  76 }
  77
  78
  79 function wait_for_recovery_toofull() {
  80     local timeout=$1
  81     wait_for_state recovery_toofull $timeout
  82 }
  83
  84
  85 # Create 1 pools with size 1
  86 # set ful-ratio to 50%
  87 # Write data 600 5K (3000K)
  88 # Inject fake_statfs_for_testing to 3600K (83% full)
  89 # Incresase the pool size to 2
  90 # The pool shouldn't have room to recovery
  91 function TEST_recovery_test_simple() {
  92     local dir=$1
  93     local pools=1
  94     local OSDS=2
  95
  96     run_mon $dir a || return 1
  97     run_mgr $dir x || return 1
  98     export CEPH_ARGS
  99
 100     for osd in $(seq 0 $(expr $OSDS - 1))
 101     do
 102       run_osd $dir $osd || return 1
 103     done
 104
 105     ceph osd set-nearfull-ratio .40
 106     ceph osd set-backfillfull-ratio .45
 107     ceph osd set-full-ratio .50
 108
 109     for p in $(seq 1 $pools)
 110     do
 111       create_pool "${poolprefix}$p" 1 1
 112       ceph osd pool set "${poolprefix}$p" size 1
 113     done
 114
 115     wait_for_clean || return 1
 116
 117     dd if=/dev/urandom of=$dir/datafile bs=1024 count=5
 118     for o in $(seq 1 $objects)
 119     do
 120       rados -p "${poolprefix}$p" put obj$o $dir/datafile
 121     done
 122
 123     for o in $(seq 0 $(expr $OSDS - 1))
 124     do
 125       ceph tell osd.$o injectargs '--fake_statfs_for_testing 3686400' || return 1
 126     done
 127     sleep 5
 128
 129     ceph pg dump pgs
 130
 131     for p in $(seq 1 $pools)
 132     do
 133       ceph osd pool set "${poolprefix}$p" size 2
 134     done
 135
 136     # If this times out, we'll detected errors below
 137     wait_for_recovery_toofull 30
 138
 139     ERRORS=0
 140     if [ "$(ceph pg dump pgs | grep +recovery_toofull | wc -l)" != "1" ];
 141     then
 142       echo "One pool should have been in recovery_toofull"
 143       ERRORS="$(expr $ERRORS + 1)"
 144     fi
 145
 146     ceph pg dump pgs
 147     ceph status
 148     ceph status --format=json-pretty > $dir/stat.json
 149
 150     eval SEV=$(jq '.health.checks.PG_RECOVERY_FULL.severity' $dir/stat.json)
 151     if [ "$SEV" != "HEALTH_ERR" ]; then
 152       echo "PG_RECOVERY_FULL severity $SEV not HEALTH_ERR"
 153       ERRORS="$(expr $ERRORS + 1)"
 154     fi
 155     eval MSG=$(jq '.health.checks.PG_RECOVERY_FULL.summary.message' $dir/stat.json)
 156     if [ "$MSG" != "Full OSDs blocking recovery: 1 pg recovery_toofull" ]; then
 157       echo "PG_RECOVERY_FULL message '$MSG' mismatched"
 158       ERRORS="$(expr $ERRORS + 1)"
 159     fi
 160     rm -f $dir/stat.json
 161
 162     if [ $ERRORS != "0" ];
 163     then
 164       return 1
 165     fi
 166
 167     for i in $(seq 1 $pools)
 168     do
 169       delete_pool "${poolprefix}$i"
 170     done
 171     kill_daemons $dir || return 1
 172 }
 173
 174
 175 main osd-recovery-space "$@"
 176
 177 # Local Variables:
 178 # compile-command: "make -j4 && ../qa/run-standalone.sh osd-recovery-space.sh"
 179 # End: