ceph/qa/standalone/mon/health-mute.sh

   1 #!/bin/bash
   2
   3 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
   4
   5 function run() {
   6     local dir=$1
   7     shift
   8
   9     export CEPH_MON="127.0.0.1:7143" # git grep '\<714\>' : there must be only one
  10     export CEPH_ARGS
  11     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none --mon-pg-warn-min-per-osd 0 --mon-max-pg-per-osd 1000 "
  12     CEPH_ARGS+="--mon-host=$CEPH_MON "
  13
  14     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  15     for func in $funcs ; do
  16         setup $dir || return 1
  17         $func $dir || return 1
  18         teardown $dir || return 1
  19     done
  20 }
  21
  22 function TEST_mute() {
  23     local dir=$1
  24     setup $dir || return 1
  25
  26     set -o pipefail
  27
  28     run_mon $dir a || return 1
  29     run_mgr $dir x || return 1
  30     run_osd $dir 0 || return 1
  31     run_osd $dir 1 || return 1
  32     run_osd $dir 2 || return 1
  33
  34     ceph osd pool create foo 8
  35     ceph osd pool application enable foo rbd --yes-i-really-mean-it
  36     wait_for_clean || return 1
  37
  38     ceph -s
  39     ceph health | grep HEALTH_OK || return 1
  40     # test warning on setting pool size=1
  41     ceph osd pool set foo size 1
  42     ceph -s
  43     ceph health | grep HEALTH_WARN || return 1
  44     ceph health detail | grep POOL_NO_REDUNDANCY || return 1
  45     ceph health mute POOL_NO_REDUNDANCY
  46     ceph -s
  47     ceph health | grep HEALTH_OK | grep POOL_NO_REDUNDANCY || return 1
  48     ceph health unmute POOL_NO_REDUNDANCY
  49     ceph -s
  50     ceph health | grep HEALTH_WARN || return 1
  51     # restore pool size to default
  52     ceph osd pool set foo size 3
  53     ceph -s
  54     ceph health | grep HEALTH_OK || return 1
  55     ceph osd set noup
  56     ceph -s
  57     ceph health detail | grep OSDMAP_FLAGS || return 1
  58     ceph osd down 0
  59     ceph -s
  60     ceph health detail | grep OSD_DOWN || return 1
  61     ceph health detail | grep HEALTH_WARN || return 1
  62
  63     ceph health mute OSD_DOWN
  64     ceph health mute OSDMAP_FLAGS
  65     ceph -s
  66     ceph health | grep HEALTH_OK | grep OSD_DOWN | grep OSDMAP_FLAGS || return 1
  67     ceph health unmute OSD_DOWN
  68     ceph -s
  69     ceph health | grep HEALTH_WARN || return 1
  70
  71     # ttl
  72     ceph health mute OSD_DOWN 10s
  73     ceph -s
  74     ceph health | grep HEALTH_OK || return 1
  75     sleep 15
  76     ceph -s
  77     ceph health | grep HEALTH_WARN || return 1
  78
  79     # sticky
  80     ceph health mute OSDMAP_FLAGS --sticky
  81     ceph osd unset noup
  82     sleep 5
  83     ceph -s
  84     ceph health | grep OSDMAP_FLAGS || return 1
  85     ceph osd set noup
  86     ceph -s
  87     ceph health | grep HEALTH_OK || return 1
  88
  89     # rachet down on OSD_DOWN count
  90     ceph osd down 0 1
  91     ceph -s
  92     ceph health detail | grep OSD_DOWN || return 1
  93
  94     ceph health mute OSD_DOWN
  95     kill_daemons $dir TERM osd.0
  96     ceph osd unset noup
  97     sleep 10
  98     ceph -s
  99     ceph health detail | grep OSD_DOWN || return 1
 100     ceph health detail | grep '1 osds down' || return 1
 101     ceph health | grep HEALTH_OK || return 1
 102
 103     sleep 10 # give time for mon tick to rachet the mute
 104     ceph osd set noup
 105     ceph health mute OSDMAP_FLAGS
 106     ceph -s
 107     ceph health detail
 108     ceph health | grep HEALTH_OK || return 1
 109
 110     ceph osd down 1
 111     ceph -s
 112     ceph health detail
 113     ceph health detail | grep '2 osds down' || return 1
 114
 115     sleep 10 # give time for mute to clear
 116     ceph -s
 117     ceph health detail
 118     ceph health | grep HEALTH_WARN || return 1
 119     ceph health detail | grep '2 osds down' || return 1
 120
 121     teardown $dir || return 1
 122 }
 123
 124 main health-mute "$@"