ceph/qa/standalone/osd/osd-markdown.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Copyright (C) 2015 Intel <contact@intel.com.com>
   4 # Copyright (C) 2014, 2015 Red Hat <contact@redhat.com>
   5 #
   6 # Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU Library Public License as published by
  10 # the Free Software Foundation; either version 2, or (at your option)
  11 # any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Library Public License for more details.
  17 #
  18
  19 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  20
  21 function run() {
  22     local dir=$1
  23     shift
  24
  25     export CEPH_MON="127.0.0.1:7108" # git grep '\<7108\>' : there must be only one
  26     export CEPH_ARGS
  27     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  28     CEPH_ARGS+="--mon-host=$CEPH_MON "
  29
  30     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  31     for func in $funcs ; do
  32         setup $dir || return 1
  33         $func $dir || return 1
  34         teardown $dir || return 1
  35     done
  36 }
  37
  38 function markdown_N_impl() {
  39   markdown_times=$1
  40   total_time=$2
  41   sleeptime=$3
  42   for i in `seq 1 $markdown_times`
  43   do
  44     # check the OSD is UP
  45     ceph tell osd.0 get_latest_osdmap || return 1
  46     ceph osd tree
  47     ceph osd tree | grep osd.0 |grep up || return 1
  48     # mark the OSD down.
  49     # override any dup setting in the environment to ensure we do this
  50     # exactly once (modulo messenger failures, at least; we can't *actually*
  51     # provide exactly-once semantics for mon commands).
  52     ( unset CEPH_CLI_TEST_DUP_COMMAND ; ceph osd down 0 )
  53     sleep $sleeptime
  54   done
  55 }
  56
  57
  58 function TEST_markdown_exceed_maxdown_count() {
  59     local dir=$1
  60
  61     run_mon $dir a || return 1
  62     run_mgr $dir x || return 1
  63     run_osd $dir 0 || return 1
  64     run_osd $dir 1 || return 1
  65     run_osd $dir 2 || return 1
  66
  67     create_rbd_pool || return 1
  68
  69     # 3+1 times within 300s, osd should stay dead on the 4th time
  70     local count=3
  71     local sleeptime=10
  72     local period=300
  73     ceph tell osd.0 injectargs '--osd_max_markdown_count '$count'' || return 1
  74     ceph tell osd.0 injectargs '--osd_max_markdown_period '$period'' || return 1
  75
  76     markdown_N_impl $(($count+1)) $period $sleeptime
  77     # down N+1 times ,the osd.0 should die
  78     ceph osd tree | grep down | grep osd.0 || return 1
  79 }
  80
  81 function TEST_markdown_boot() {
  82     local dir=$1
  83
  84     run_mon $dir a || return 1
  85     run_mgr $dir x || return 1
  86     run_osd $dir 0 || return 1
  87     run_osd $dir 1 || return 1
  88     run_osd $dir 2 || return 1
  89
  90     create_rbd_pool || return 1
  91
  92     # 3 times within 120s, should stay up
  93     local count=3
  94     local sleeptime=10
  95     local period=120
  96     ceph tell osd.0 injectargs '--osd_max_markdown_count '$count'' || return 1
  97     ceph tell osd.0 injectargs '--osd_max_markdown_period '$period'' || return 1
  98
  99     markdown_N_impl $count $period $sleeptime
 100     #down N times, osd.0 should be up
 101     sleep 15  # give osd plenty of time to notice and come back up
 102     ceph tell osd.0 get_latest_osdmap || return 1
 103     ceph osd tree | grep up | grep osd.0 || return 1
 104 }
 105
 106 function TEST_markdown_boot_exceed_time() {
 107     local dir=$1
 108
 109     run_mon $dir a || return 1
 110     run_mgr $dir x || return 1
 111     run_osd $dir 0 || return 1
 112     run_osd $dir 1 || return 1
 113     run_osd $dir 2 || return 1
 114
 115     create_rbd_pool || return 1
 116
 117     # 3+1 times, but over 40s, > 20s, so should stay up
 118     local count=3
 119     local period=20
 120     local sleeptime=10
 121     ceph tell osd.0 injectargs '--osd_max_markdown_count '$count'' || return 1
 122     ceph tell osd.0 injectargs '--osd_max_markdown_period '$period'' || return 1
 123
 124     markdown_N_impl $(($count+1)) $period $sleeptime
 125     sleep 15  # give osd plenty of time to notice and come back up
 126     ceph tell osd.0 get_latest_osdmap || return 1
 127     ceph osd tree | grep up | grep osd.0 || return 1
 128 }
 129
 130 function TEST_osd_stop() {
 131
 132     local dir=$1
 133
 134     run_mon $dir a || return 1
 135     run_mgr $dir x || return 1
 136     run_osd $dir 0 || return 1
 137     run_osd $dir 1 || return 1
 138     run_osd $dir 2 || return 1
 139     osd_0_pid=$(cat $dir/osd.0.pid)
 140     ps -p $osd_0_pid || return 1
 141
 142     ceph osd tree | grep osd.0 | grep up || return 1
 143     ceph osd stop osd.0
 144     sleep 15 # give osd plenty of time to notice and exit
 145     ceph osd tree | grep down | grep osd.0 || return 1
 146     ! ps -p $osd_0_pid || return 1
 147 }
 148
 149 main osd-markdown "$@"