ceph/qa/workunits/rbd/rbd_mirror_ha.sh

   1 #!/usr/bin/env bash
   2 #
   3 # rbd_mirror_ha.sh - test rbd-mirror daemons in HA mode
   4 #
   5
   6 set -ex
   7
   8 RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-7}
   9
  10 . $(dirname $0)/rbd_mirror_helpers.sh
  11
  12 setup
  13
  14 is_leader()
  15 {
  16     local instance=$1
  17     local pool=$2
  18
  19     test -n "${pool}" || pool=${POOL}
  20
  21     admin_daemon "${CLUSTER1}:${instance}" \
  22                  rbd mirror status ${pool} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} |
  23         grep '"leader": true'
  24 }
  25
  26 wait_for_leader()
  27 {
  28     local s instance
  29
  30     for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64; do
  31         sleep $s
  32         for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
  33             is_leader ${instance} || continue
  34             LEADER=${instance}
  35             return 0
  36         done
  37     done
  38
  39     LEADER=
  40     return 1
  41 }
  42
  43 release_leader()
  44 {
  45     local pool=$1
  46     local cmd="rbd mirror leader release"
  47
  48     test -n "${pool}" && cmd="${cmd} ${pool} ${CLUSTER2}"
  49
  50     admin_daemon "${CLUSTER1}:${LEADER}" ${cmd}
  51 }
  52
  53 wait_for_leader_released()
  54 {
  55     local i
  56
  57     test -n "${LEADER}"
  58     for i in `seq 10`; do
  59         is_leader ${LEADER} || return 0
  60         sleep 1
  61     done
  62
  63     return 1
  64 }
  65
  66 test_replay()
  67 {
  68     local image
  69
  70     for image; do
  71         wait_for_image_replay_started ${CLUSTER1}:${LEADER} ${POOL} ${image}
  72         write_image ${CLUSTER2} ${POOL} ${image} 100
  73         wait_for_replay_complete ${CLUSTER1}:${LEADER} ${CLUSTER2} ${POOL} \
  74                                  ${image}
  75         wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' \
  76                                     'primary_position' \
  77                                     "${MIRROR_USER_ID_PREFIX}${LEADER} on $(hostname -s)"
  78         if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  79             wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} \
  80                                         'down+unknown'
  81         fi
  82         compare_images ${POOL} ${image}
  83     done
  84 }
  85
  86 testlog "TEST: start first daemon instance and test replay"
  87 start_mirror ${CLUSTER1}:0
  88 image1=test1
  89 create_image ${CLUSTER2} ${POOL} ${image1}
  90 LEADER=0
  91 test_replay ${image1}
  92
  93 testlog "TEST: release leader and wait it is reacquired"
  94 is_leader 0 ${POOL}
  95 is_leader 0 ${PARENT_POOL}
  96 release_leader ${POOL}
  97 wait_for_leader_released
  98 is_leader 0 ${PARENT_POOL}
  99 wait_for_leader
 100 release_leader
 101 wait_for_leader_released
 102 expect_failure "" is_leader 0 ${PARENT_POOL}
 103 wait_for_leader
 104
 105 testlog "TEST: start second daemon instance and test replay"
 106 start_mirror ${CLUSTER1}:1
 107 image2=test2
 108 create_image ${CLUSTER2} ${POOL} ${image2}
 109 test_replay ${image1} ${image2}
 110
 111 testlog "TEST: release leader and test it is acquired by secondary"
 112 is_leader 0 ${POOL}
 113 is_leader 0 ${PARENT_POOL}
 114 release_leader ${POOL}
 115 wait_for_leader_released
 116 wait_for_leader
 117 test_replay ${image1} ${image2}
 118 release_leader
 119 wait_for_leader_released
 120 wait_for_leader
 121 test "${LEADER}" = 0
 122
 123 testlog "TEST: stop first daemon instance and test replay"
 124 stop_mirror ${CLUSTER1}:0
 125 image3=test3
 126 create_image ${CLUSTER2} ${POOL} ${image3}
 127 LEADER=1
 128 test_replay ${image1} ${image2} ${image3}
 129
 130 testlog "TEST: start first daemon instance and test replay"
 131 start_mirror ${CLUSTER1}:0
 132 image4=test4
 133 create_image ${CLUSTER2} ${POOL} ${image4}
 134 test_replay ${image3} ${image4}
 135
 136 testlog "TEST: crash leader and test replay"
 137 stop_mirror ${CLUSTER1}:1 -KILL
 138 image5=test5
 139 create_image ${CLUSTER2} ${POOL} ${image5}
 140 LEADER=0
 141 test_replay ${image1} ${image4} ${image5}
 142
 143 testlog "TEST: start crashed leader and test replay"
 144 start_mirror ${CLUSTER1}:1
 145 image6=test6
 146 create_image ${CLUSTER2} ${POOL} ${image6}
 147 test_replay ${image1} ${image6}
 148
 149 testlog "TEST: start yet another daemon instance and test replay"
 150 start_mirror ${CLUSTER1}:2
 151 image7=test7
 152 create_image ${CLUSTER2} ${POOL} ${image7}
 153 test_replay ${image1} ${image7}
 154
 155 testlog "TEST: release leader and test it is acquired by secondary"
 156 is_leader 0
 157 release_leader
 158 wait_for_leader_released
 159 wait_for_leader
 160 test_replay ${image1} ${image2}
 161
 162 testlog "TEST: stop leader and test replay"
 163 stop_mirror ${CLUSTER1}:${LEADER}
 164 image8=test8
 165 create_image ${CLUSTER2} ${POOL} ${image8}
 166 prev_leader=${LEADER}
 167 wait_for_leader
 168 test_replay ${image1} ${image8}
 169
 170 testlog "TEST: start previous leader and test replay"
 171 start_mirror ${CLUSTER1}:${prev_leader}
 172 image9=test9
 173 create_image ${CLUSTER2} ${POOL} ${image9}
 174 test_replay ${image1} ${image9}
 175
 176 testlog "TEST: crash leader and test replay"
 177 stop_mirror ${CLUSTER1}:${LEADER} -KILL
 178 image10=test10
 179 create_image ${CLUSTER2} ${POOL} ${image10}
 180 prev_leader=${LEADER}
 181 wait_for_leader
 182 test_replay ${image1} ${image10}
 183
 184 testlog "TEST: start previous leader and test replay"
 185 start_mirror ${CLUSTER1}:${prev_leader}
 186 image11=test11
 187 create_image ${CLUSTER2} ${POOL} ${image11}
 188 test_replay ${image1} ${image11}
 189
 190 testlog "TEST: start some more daemon instances and test replay"
 191 start_mirror ${CLUSTER1}:3
 192 start_mirror ${CLUSTER1}:4
 193 start_mirror ${CLUSTER1}:5
 194 start_mirror ${CLUSTER1}:6
 195 image13=test13
 196 create_image ${CLUSTER2} ${POOL} ${image13}
 197 test_replay ${leader} ${image1} ${image13}
 198
 199 testlog "TEST: release leader and test it is acquired by secondary"
 200 release_leader
 201 wait_for_leader_released
 202 wait_for_leader
 203 test_replay ${image1} ${image2}
 204
 205 testlog "TEST: in loop: stop leader and test replay"
 206 for i in 0 1 2 3 4 5; do
 207     stop_mirror ${CLUSTER1}:${LEADER}
 208     wait_for_leader
 209     test_replay ${image1}
 210 done
 211
 212 stop_mirror ${CLUSTER1}:${LEADER}