]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | #!/bin/sh |
2 | # | |
3 | # rbd_mirror_ha.sh - test rbd-mirror daemons in HA mode | |
4 | # | |
5 | ||
6 | . $(dirname $0)/rbd_mirror_helpers.sh | |
7 | ||
8 | is_leader() | |
9 | { | |
10 | local instance=$1 | |
11 | local pool=$2 | |
12 | ||
13 | test -n "${pool}" || pool=${POOL} | |
14 | ||
15 | admin_daemon "${CLUSTER1}:${instance}" \ | |
16 | rbd mirror status ${pool} ${CLUSTER2} | | |
17 | grep '"leader": true' | |
18 | } | |
19 | ||
20 | wait_for_leader() | |
21 | { | |
22 | local s instance | |
23 | ||
24 | for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64; do | |
25 | sleep $s | |
26 | for instance in `seq 0 9`; do | |
27 | is_leader ${instance} || continue | |
28 | LEADER=${instance} | |
29 | return 0 | |
30 | done | |
31 | done | |
32 | ||
33 | LEADER= | |
34 | return 1 | |
35 | } | |
36 | ||
37 | release_leader() | |
38 | { | |
39 | local pool=$1 | |
40 | local cmd="rbd mirror leader release" | |
41 | ||
42 | test -n "${pool}" && cmd="${cmd} ${pool} ${CLUSTER2}" | |
43 | ||
44 | admin_daemon "${CLUSTER1}:${LEADER}" ${cmd} | |
45 | } | |
46 | ||
47 | wait_for_leader_released() | |
48 | { | |
49 | local i | |
50 | ||
51 | test -n "${LEADER}" | |
52 | for i in `seq 10`; do | |
53 | is_leader ${LEADER} || return 0 | |
54 | sleep 1 | |
55 | done | |
56 | ||
57 | return 1 | |
58 | } | |
59 | ||
60 | test_replay() | |
61 | { | |
62 | local image | |
63 | ||
64 | for image; do | |
65 | wait_for_image_replay_started ${CLUSTER1}:${LEADER} ${POOL} ${image} | |
66 | write_image ${CLUSTER2} ${POOL} ${image} 100 | |
67 | wait_for_replay_complete ${CLUSTER1}:${LEADER} ${CLUSTER2} ${POOL} \ | |
68 | ${image} | |
69 | wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' \ | |
70 | 'master_position' | |
71 | if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then | |
72 | wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} \ | |
73 | 'down+unknown' | |
74 | fi | |
75 | compare_images ${POOL} ${image} | |
76 | done | |
77 | } | |
78 | ||
79 | testlog "TEST: start first daemon instance and test replay" | |
80 | start_mirror ${CLUSTER1}:0 | |
81 | image1=test1 | |
82 | create_image ${CLUSTER2} ${POOL} ${image1} | |
83 | LEADER=0 | |
84 | test_replay ${image1} | |
85 | ||
86 | testlog "TEST: release leader and wait it is reacquired" | |
87 | is_leader 0 ${POOL} | |
88 | is_leader 0 ${PARENT_POOL} | |
89 | release_leader ${POOL} | |
90 | wait_for_leader_released | |
91 | is_leader 0 ${PARENT_POOL} | |
92 | wait_for_leader | |
93 | release_leader | |
94 | wait_for_leader_released | |
95 | expect_failure "" is_leader 0 ${PARENT_POOL} | |
96 | wait_for_leader | |
97 | ||
98 | testlog "TEST: start second daemon instance and test replay" | |
99 | start_mirror ${CLUSTER1}:1 | |
100 | image2=test2 | |
101 | create_image ${CLUSTER2} ${POOL} ${image2} | |
102 | test_replay ${image1} ${image2} | |
103 | ||
104 | testlog "TEST: release leader and test it is acquired by secondary" | |
105 | is_leader 0 ${POOL} | |
106 | is_leader 0 ${PARENT_POOL} | |
107 | release_leader ${POOL} | |
108 | wait_for_leader_released | |
109 | wait_for_leader | |
110 | test_replay ${image1} ${image2} | |
111 | release_leader | |
112 | wait_for_leader_released | |
113 | wait_for_leader | |
114 | test "${LEADER}" = 0 | |
115 | ||
116 | testlog "TEST: stop first daemon instance and test replay" | |
117 | stop_mirror ${CLUSTER1}:0 | |
118 | image3=test3 | |
119 | create_image ${CLUSTER2} ${POOL} ${image3} | |
120 | LEADER=1 | |
121 | test_replay ${image1} ${image2} ${image3} | |
122 | ||
123 | testlog "TEST: start first daemon instance and test replay" | |
124 | start_mirror ${CLUSTER1}:0 | |
125 | image4=test4 | |
126 | create_image ${CLUSTER2} ${POOL} ${image4} | |
127 | test_replay ${image3} ${image4} | |
128 | ||
129 | testlog "TEST: crash leader and test replay" | |
130 | stop_mirror ${CLUSTER1}:1 -KILL | |
131 | image5=test5 | |
132 | create_image ${CLUSTER2} ${POOL} ${image5} | |
133 | LEADER=0 | |
134 | test_replay ${image1} ${image4} ${image5} | |
135 | ||
136 | testlog "TEST: start crashed leader and test replay" | |
137 | start_mirror ${CLUSTER1}:1 | |
138 | image6=test6 | |
139 | create_image ${CLUSTER2} ${POOL} ${image6} | |
140 | test_replay ${image1} ${image6} | |
141 | ||
142 | testlog "TEST: start yet another daemon instance and test replay" | |
143 | start_mirror ${CLUSTER1}:2 | |
144 | image7=test7 | |
145 | create_image ${CLUSTER2} ${POOL} ${image7} | |
146 | test_replay ${image1} ${image7} | |
147 | ||
148 | testlog "TEST: release leader and test it is acquired by secondary" | |
149 | is_leader 0 | |
150 | release_leader | |
151 | wait_for_leader_released | |
152 | wait_for_leader | |
153 | test_replay ${image1} ${image2} | |
154 | ||
155 | testlog "TEST: stop leader and test replay" | |
156 | stop_mirror ${CLUSTER1}:${LEADER} | |
157 | image8=test8 | |
158 | create_image ${CLUSTER2} ${POOL} ${image8} | |
159 | prev_leader=${LEADER} | |
160 | wait_for_leader | |
161 | test_replay ${image1} ${image8} | |
162 | ||
163 | testlog "TEST: start previous leader and test replay" | |
164 | start_mirror ${CLUSTER1}:${prev_leader} | |
165 | image9=test9 | |
166 | create_image ${CLUSTER2} ${POOL} ${image9} | |
167 | test_replay ${image1} ${image9} | |
168 | ||
169 | testlog "TEST: crash leader and test replay" | |
170 | stop_mirror ${CLUSTER1}:${LEADER} -KILL | |
171 | image10=test10 | |
172 | create_image ${CLUSTER2} ${POOL} ${image10} | |
173 | prev_leader=${LEADER} | |
174 | wait_for_leader | |
175 | test_replay ${image1} ${image10} | |
176 | ||
177 | testlog "TEST: start previous leader and test replay" | |
178 | start_mirror ${CLUSTER1}:${prev_leader} | |
179 | image11=test11 | |
180 | create_image ${CLUSTER2} ${POOL} ${image11} | |
181 | test_replay ${image1} ${image11} | |
182 | ||
183 | testlog "TEST: start some more daemon instances and test replay" | |
184 | start_mirror ${CLUSTER1}:3 | |
185 | start_mirror ${CLUSTER1}:4 | |
186 | start_mirror ${CLUSTER1}:5 | |
187 | start_mirror ${CLUSTER1}:6 | |
188 | image13=test13 | |
189 | create_image ${CLUSTER2} ${POOL} ${image13} | |
190 | test_replay ${leader} ${image1} ${image13} | |
191 | ||
192 | testlog "TEST: release leader and test it is acquired by secondary" | |
193 | release_leader | |
194 | wait_for_leader_released | |
195 | wait_for_leader | |
196 | test_replay ${image1} ${image2} | |
197 | ||
198 | testlog "TEST: in loop: stop leader and test replay" | |
199 | for i in 0 1 2 3 4 5; do | |
200 | stop_mirror ${CLUSTER1}:${LEADER} | |
201 | wait_for_leader | |
202 | test_replay ${image1} | |
203 | done | |
204 | ||
205 | stop_mirror ${CLUSTER1}:${LEADER} | |
206 | ||
207 | echo OK |