]>
Commit | Line | Data |
---|---|---|
1b1d2e6d BP |
1 | OVS_START_SHELL_HELPERS |
2 | # ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT TRANSACTION... | |
3 | ovsdb_check_cluster () { | |
4 | local n=$1 schema_func=$2 output=$3 | |
5 | shift; shift; shift | |
6 | ||
7 | $schema_func > schema | |
8 | schema=`ovsdb-tool schema-name schema` | |
9 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr]) | |
10 | AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1]) | |
11 | cid=`ovsdb-tool db-cid s1.db` | |
12 | for i in `seq 2 $n`; do | |
13 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft]) | |
14 | done | |
15 | ||
16 | on_exit 'kill `cat *.pid`' | |
17 | for i in `seq $n`; do | |
5a0e4aec | 18 | AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) |
1b1d2e6d BP |
19 | done |
20 | for i in `seq $n`; do | |
d97af428 | 21 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema connected]) |
1b1d2e6d BP |
22 | done |
23 | ||
24 | for txn | |
25 | do | |
15394e0f | 26 | AT_CHECK([ovsdb-client -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout]) |
1b1d2e6d BP |
27 | cat stdout >> output |
28 | done | |
29 | AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output]) | |
30 | for i in `seq $n`; do | |
5a0e4aec | 31 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) |
1b1d2e6d BP |
32 | done |
33 | ||
34 | AT_CHECK([ovsdb-tool check-cluster s*.db]) | |
35 | } | |
36 | OVS_END_SHELL_HELPERS | |
37 | ||
38 | # Test a 1-server cluster. | |
39 | AT_BANNER([OVSDB - clustered transactions (1 server)]) | |
40 | m4_define([OVSDB_CHECK_EXECUTION], | |
41 | [AT_SETUP([$1 - cluster of 1]) | |
42 | AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5]) | |
43 | ovsdb_check_cluster 1 "$2" '$4' m4_foreach([txn], [$3], ['txn' ]) | |
44 | AT_CLEANUP]) | |
45 | EXECUTION_EXAMPLES | |
46 | ||
47 | # Test a 3-server cluster. | |
48 | AT_BANNER([OVSDB - clustered transactions (3 servers)]) | |
49 | m4_define([OVSDB_CHECK_EXECUTION], | |
50 | [AT_SETUP([$1 - cluster of 3]) | |
51 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5]) | |
52 | ovsdb_check_cluster 3 "$2" '$4' m4_foreach([txn], [$3], ['txn' ]) | |
53 | AT_CLEANUP]) | |
54 | EXECUTION_EXAMPLES | |
55 | ||
56 | # Test a 5-server cluster. | |
57 | AT_BANNER([OVSDB - clustered transactions (5 servers)]) | |
58 | m4_define([OVSDB_CHECK_EXECUTION], | |
59 | [AT_SETUP([$1 - cluster of 5]) | |
60 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5]) | |
61 | ovsdb_check_cluster 5 "$2" '$4' m4_foreach([txn], [$3], ['txn' ]) | |
62 | AT_CLEANUP]) | |
63 | EXECUTION_EXAMPLES | |
eb692258 HZ |
64 | \f |
65 | ||
ca367fa5 HZ |
66 | AT_BANNER([OVSDB - disconnect from cluster]) |
67 | ||
89771c1e | 68 | OVS_START_SHELL_HELPERS |
923f01ca HZ |
69 | # ovsdb_test_cluster_disconnect N_SERVERS LEADER_OR_FOLLOWER [CHECK_FLAPPING] |
70 | # Test server disconnected from the cluster. | |
71 | # N_SERVERS: Number of servers in the cluster. | |
72 | # LEADER_OR_FOLLOWER: The role of the server that is disconnected from the | |
73 | # cluster: "leader" or "follower". | |
74 | # CHECK_FLAPPING: Whether to check if is_disconnected flapped. "yes", "no". | |
89771c1e | 75 | ovsdb_test_cluster_disconnect () { |
923f01ca HZ |
76 | n=$1 |
77 | leader_or_follower=$2 | |
78 | check_flapping=$3 | |
89771c1e HZ |
79 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` |
80 | ordinal_schema > schema | |
81 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr]) | |
82 | cid=`ovsdb-tool db-cid s1.db` | |
83 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
923f01ca | 84 | for i in `seq 2 $n`; do |
89771c1e HZ |
85 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft]) |
86 | done | |
87 | ||
88 | on_exit 'kill `cat *.pid`' | |
923f01ca | 89 | for i in `seq $n`; do |
89771c1e HZ |
90 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) |
91 | done | |
923f01ca | 92 | for i in `seq $n`; do |
89771c1e HZ |
93 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected]) |
94 | done | |
95 | ||
96 | AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest", | |
97 | {"op": "insert", | |
98 | "table": "simple", | |
99 | "row": {"i": 1}}]]'], [0], [ignore], [ignore]) | |
100 | ||
101 | # When a node is disconnected from the cluster, the IDL should disconnect | |
102 | # and retry even if it uses a single remote, because the remote IP can be | |
103 | # a VIP on a load-balance. So we use single remote to test here. | |
104 | if test $leader_or_follower == "leader"; then | |
105 | target=1 | |
923f01ca HZ |
106 | shutdown=`seq $(($n/2 + 1)) $n` |
107 | cleanup=`seq $(($n/2))` | |
89771c1e | 108 | else |
923f01ca | 109 | target=$n |
89771c1e | 110 | |
923f01ca HZ |
111 | # shutdown followers before the leader (s1) so that there is no chance for |
112 | # s$n to become leader during the process. | |
113 | shutdown="`seq 2 $(($n/2 + 1))` 1" | |
114 | cleanup=`seq $(($n/2 + 2)) $n` | |
89771c1e | 115 | fi |
923f01ca HZ |
116 | echo shutdown=$shutdown |
117 | echo cleanup=$cleanup | |
89771c1e HZ |
118 | |
119 | # Connect to $target. Use "wait" to trigger a non-op transaction so | |
120 | # that test-ovsdb will not quit. | |
121 | ||
122 | test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -v -t10 idl unix:s$target.ovsdb '[["idltest", | |
123 | {"op": "wait", | |
124 | "table": "simple", | |
125 | "where": [["i", "==", 1]], | |
126 | "columns": ["i"], | |
127 | "until": "==", | |
128 | "rows": [{"i": 1}]}]]' > test-ovsdb.log 2>&1 & | |
129 | echo $! > test-ovsdb.pid | |
ca367fa5 | 130 | |
89771c1e | 131 | OVS_WAIT_UNTIL([grep "000: i=1" test-ovsdb.log]) |
ca367fa5 | 132 | |
923f01ca HZ |
133 | # Start collecting raft_is_connected logs for $target before shutting down |
134 | # any servers. | |
135 | tail -f s$target.log > raft_is_connected.log & | |
136 | echo $! > tail.pid | |
137 | ||
89771c1e HZ |
138 | # Shutdown the other servers so that $target is disconnected from the cluster. |
139 | for i in $shutdown; do | |
140 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
141 | done | |
142 | ||
143 | # The test-ovsdb should detect the disconnect and retry. | |
144 | OVS_WAIT_UNTIL([grep disconnect test-ovsdb.log]) | |
145 | ||
923f01ca HZ |
146 | # The $target debug log should show raft_is_connected: false. |
147 | OVS_WAIT_UNTIL([grep "raft_is_connected: false" raft_is_connected.log]) | |
148 | ||
149 | # Save the current count of "raft_is_connected: true" | |
150 | count_old=`grep "raft_is_connected: true" raft_is_connected.log | wc -l` | |
151 | echo count_old $count_old | |
152 | ||
153 | if test X$check_flapping == X"yes"; then | |
154 | sleep 10 | |
155 | fi | |
156 | # Make sure raft_is_connected didn't flap from false to true. | |
157 | count_new=`grep "raft_is_connected: true" raft_is_connected.log | wc -l` | |
158 | echo count_new $count_new | |
159 | AT_CHECK([test $count_new == $count_old]) | |
160 | ||
161 | for i in $cleanup; do | |
162 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
163 | done | |
89771c1e HZ |
164 | } |
165 | OVS_END_SHELL_HELPERS | |
166 | ||
167 | AT_SETUP([OVSDB cluster - follower disconnect from cluster, single remote]) | |
168 | AT_KEYWORDS([ovsdb server negative unix cluster disconnect]) | |
923f01ca | 169 | ovsdb_test_cluster_disconnect 3 follower |
89771c1e HZ |
170 | AT_CLEANUP |
171 | ||
172 | AT_SETUP([OVSDB cluster - leader disconnect from cluster, single remote]) | |
173 | AT_KEYWORDS([ovsdb server negative unix cluster disconnect]) | |
923f01ca HZ |
174 | ovsdb_test_cluster_disconnect 3 leader |
175 | AT_CLEANUP | |
176 | ||
177 | AT_SETUP([OVSDB cluster - leader disconnect from cluster, check flapping]) | |
178 | AT_KEYWORDS([ovsdb server negative unix cluster disconnect]) | |
179 | ovsdb_test_cluster_disconnect 5 leader yes | |
ca367fa5 | 180 | AT_CLEANUP |
89771c1e | 181 | |
2833885f HZ |
182 | AT_SETUP([OVSDB cluster - initial status should be disconnected]) |
183 | AT_KEYWORDS([ovsdb server negative unix cluster disconnect]) | |
184 | ||
185 | n=3 | |
186 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
187 | ordinal_schema > schema | |
188 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr]) | |
189 | cid=`ovsdb-tool db-cid s1.db` | |
190 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
191 | for i in `seq 2 $n`; do | |
192 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft]) | |
193 | done | |
194 | ||
195 | on_exit 'kill `cat *.pid`' | |
196 | for i in `seq $n`; do | |
197 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
198 | done | |
199 | for i in `seq $n`; do | |
200 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected]) | |
201 | done | |
202 | ||
203 | # Stop all servers, and start the s1 only, to test initial connection status | |
204 | # when there is no leader yet. | |
205 | for i in `seq 1 $n`; do | |
206 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
207 | done | |
208 | i=1 | |
209 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
210 | ||
211 | # The initial status should be disconnected. So wait should fail. | |
212 | AT_CHECK([ovsdb_client_wait --timeout=1 unix:s$i.ovsdb $schema_name connected], [142], [ignore], [ignore]) | |
213 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
214 | ||
215 | AT_CLEANUP | |
216 | ||
ca367fa5 HZ |
217 | \f |
218 | ||
8e354614 HZ |
219 | AT_BANNER([OVSDB cluster election timer change]) |
220 | ||
221 | AT_SETUP([OVSDB cluster - election timer change]) | |
222 | AT_KEYWORDS([ovsdb server positive unix cluster timer]) | |
223 | ||
224 | n=3 | |
225 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
226 | ordinal_schema > schema | |
227 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr]) | |
228 | cid=`ovsdb-tool db-cid s1.db` | |
229 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
230 | for i in `seq 2 $n`; do | |
231 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft]) | |
232 | done | |
233 | ||
234 | on_exit 'kill `cat *.pid`' | |
235 | for i in `seq $n`; do | |
236 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
237 | done | |
238 | for i in `seq $n`; do | |
239 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected]) | |
240 | done | |
241 | ||
242 | # Change not allowed through follower. | |
243 | AT_CHECK([ovs-appctl -t "`pwd`"/s2 cluster/change-election-timer $schema_name 2000], [2], [], [ignore]) | |
244 | ||
245 | # Timer cannot be changed to bigger than 2x the original value. | |
246 | AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [2], [], [ignore]) | |
247 | ||
248 | AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 2000], [0], [dnl | |
249 | change of election timer initiated. | |
250 | ], []) | |
251 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 2000"]) | |
252 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 2000"]) | |
253 | ||
254 | AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [0], [dnl | |
255 | change of election timer initiated. | |
256 | ], []) | |
257 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"]) | |
258 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"]) | |
259 | ||
260 | # Latest timer should be used after restart | |
261 | for i in `seq $n`; do | |
262 | printf "\ns$i: stopping\n" | |
263 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
264 | done | |
265 | for i in `seq $n`; do | |
266 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
267 | done | |
268 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"]) | |
269 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"]) | |
270 | ||
6d034053 HZ |
271 | # Wait until cluster is ready |
272 | for i in `seq $n`; do | |
273 | OVS_WAIT_WHILE([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "Leader: unknown"]) | |
274 | done | |
275 | ||
9bfb280a HZ |
276 | # Latest timer should be restored after DB compact and restart. |
277 | # This is to test the install_snapshot RPC. | |
278 | ||
9bfb280a HZ |
279 | # Compact online |
280 | for i in `seq $n`; do | |
281 | AT_CHECK([ovs-appctl -t "`pwd`"/s$i ovsdb-server/compact]) | |
282 | done | |
283 | ||
9bfb280a HZ |
284 | for i in `seq $n`; do |
285 | printf "\ns$i: stopping\n" | |
286 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
287 | done | |
288 | for i in `seq $n`; do | |
289 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
290 | done | |
291 | for i in `seq $n`; do | |
292 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "Election timer: 4000"]) | |
293 | done | |
294 | ||
295 | # Wait until cluster is ready | |
296 | for i in `seq $n`; do | |
297 | OVS_WAIT_WHILE([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "Leader: unknown"]) | |
298 | done | |
299 | ||
300 | # Newly joined member should use latest timer value | |
301 | AT_CHECK([ovsdb-tool join-cluster s4.db $schema_name unix:s4.raft unix:s1.raft]) | |
302 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s4.log --pidfile=s4.pid --unixctl=s4 --remote=punix:s4.ovsdb s4.db]) | |
303 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s4 cluster/status $schema_name | grep "Election timer: 4000"]) | |
304 | # for i in `seq 10`; do | |
305 | # ovs-appctl -t "`pwd`"/s4 cluster/status $schema_name | |
306 | # sleep 1 | |
307 | # done | |
308 | ||
8e354614 HZ |
309 | AT_CLEANUP |
310 | ||
bda1f6b6 HZ |
311 | \f |
312 | AT_BANNER([OVSDB cluster install snapshot RPC]) | |
313 | ||
314 | AT_SETUP([OVSDB cluster - install snapshot RPC]) | |
315 | AT_KEYWORDS([ovsdb server positive unix cluster snapshot]) | |
316 | ||
317 | n=3 | |
318 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
319 | ordinal_schema > schema | |
320 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr]) | |
321 | cid=`ovsdb-tool db-cid s1.db` | |
322 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
323 | for i in `seq 2 $n`; do | |
324 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft]) | |
325 | done | |
326 | ||
327 | on_exit 'kill `cat *.pid`' | |
328 | for i in `seq $n`; do | |
329 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
330 | done | |
331 | for i in `seq $n`; do | |
332 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected]) | |
333 | done | |
334 | ||
335 | # Kill one follower (s2) and write some data to cluster, so that the follower is falling behind | |
336 | printf "\ns2: stopping\n" | |
337 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s2], [s2.pid]) | |
338 | ||
339 | AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest", | |
340 | {"op": "insert", | |
341 | "table": "simple", | |
342 | "row": {"i": 1}}]]'], [0], [ignore], [ignore]) | |
343 | ||
344 | # Compact leader online to generate snapshot | |
345 | AT_CHECK([ovs-appctl -t "`pwd`"/s1 ovsdb-server/compact]) | |
346 | ||
347 | # Start the follower s2 again. | |
348 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s2.log --pidfile=s2.pid --unixctl=s2 --remote=punix:s2.ovsdb s2.db]) | |
349 | AT_CHECK([ovsdb_client_wait unix:s2.ovsdb $schema_name connected]) | |
350 | ||
351 | # A client transaction through s2. During this transaction, there will be a | |
352 | # install_snapshot RPC because s2 detects it is behind and s1 doesn't have the | |
353 | # pre_log_index requested by s2 because it is already compacted. | |
354 | # After the install_snapshot RPC process, the transaction through s2 should | |
355 | # succeed. | |
356 | AT_CHECK([ovsdb-client transact unix:s2.ovsdb '[["idltest", | |
357 | {"op": "insert", | |
358 | "table": "simple", | |
359 | "row": {"i": 1}}]]'], [0], [ignore], [ignore]) | |
360 | ||
361 | for i in `seq $n`; do | |
362 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
363 | done | |
364 | ||
365 | AT_CLEANUP | |
366 | ||
8e354614 HZ |
367 | \f |
368 | ||
eb692258 HZ |
369 | OVS_START_SHELL_HELPERS |
370 | # ovsdb_cluster_failure_test SCHEMA_FUNC OUTPUT TRANSACTION... | |
371 | ovsdb_cluster_failure_test () { | |
372 | # Initial state: s1 is leader, s2 and s3 are followers | |
373 | remote_1=$1 | |
374 | remote_2=$2 | |
375 | crash_node=$3 | |
376 | crash_command=$4 | |
377 | if test "$crash_node" == "1"; then | |
378 | new_leader=$5 | |
379 | fi | |
380 | ||
817db730 | 381 | cp $top_srcdir/vswitchd/vswitch.ovsschema schema |
eb692258 HZ |
382 | schema=`ovsdb-tool schema-name schema` |
383 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl | |
817db730 | 384 | ovsdb|WARN|schema: changed 30 columns in 'Open_vSwitch' database from ephemeral to persistent, including 'status' column in 'Manager' table, because clusters do not support ephemeral columns |
eb692258 HZ |
385 | ]) |
386 | ||
387 | n=3 | |
388 | join_cluster() { | |
389 | local i=$1 | |
390 | others= | |
391 | for j in `seq 1 $n`; do | |
392 | if test $i != $j; then | |
393 | others="$others unix:s$j.raft" | |
394 | fi | |
395 | done | |
396 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others]) | |
397 | } | |
398 | start_server() { | |
399 | local i=$1 | |
400 | printf "\ns$i: starting\n" | |
401 | AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
402 | } | |
403 | connect_server() { | |
404 | local i=$1 | |
405 | printf "\ns$i: waiting to connect to storage\n" | |
406 | AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected]) | |
407 | } | |
408 | cid=`ovsdb-tool db-cid s1.db` | |
409 | for i in `seq 2 $n`; do join_cluster $i; done | |
410 | ||
411 | on_exit 'kill `cat *.pid`' | |
412 | for i in `seq $n`; do start_server $i; done | |
413 | for i in `seq $n`; do connect_server $i; done | |
414 | ||
817db730 | 415 | db=unix:s$remote_1.ovsdb,unix:s$remote_2.ovsdb |
eb692258 HZ |
416 | |
417 | # To ensure $new_leader node the new leader, we delay election timer for | |
418 | # the other follower. | |
419 | if test -n "$new_leader"; then | |
420 | if test "$new_leader" == "2"; then | |
421 | delay_election_node=3 | |
422 | else | |
423 | delay_election_node=2 | |
424 | fi | |
425 | AT_CHECK([ovs-appctl -t "`pwd`"/s$delay_election_node cluster/failure-test delay-election], [0], [ignore]) | |
426 | fi | |
427 | AT_CHECK([ovs-appctl -t "`pwd`"/s$crash_node cluster/failure-test $crash_command], [0], [ignore]) | |
15394e0f | 428 | AT_CHECK([ovs-vsctl -v --db="$db" --no-leader-only --no-shuffle-remotes --no-wait create QoS type=x], [0], [ignore], [ignore]) |
eb692258 HZ |
429 | |
430 | # Make sure that the node really crashed. | |
431 | AT_CHECK([ls s$crash_node.ovsdb], [2], [ignore], [ignore]) | |
432 | # XXX: Client will fail if remotes contains unix socket that doesn't exist (killed). | |
817db730 BP |
433 | if test "$remote_1" = "$crash_node"; then |
434 | db=unix:s$remote_2.ovsdb | |
eb692258 | 435 | fi |
817db730 | 436 | AT_CHECK([ovs-vsctl --db="$db" --no-leader-only --no-wait --columns=type --bare list QoS], [0], [x |
eb692258 HZ |
437 | ]) |
438 | } | |
439 | OVS_END_SHELL_HELPERS | |
440 | AT_BANNER([OVSDB - cluster failure with pending transaction]) | |
441 | ||
442 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-2 becomes leader]) | |
443 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
444 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 2 |
445 | AT_CLEANUP | |
446 | ||
447 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-3 becomes leader]) | |
448 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
449 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 3 | |
450 | AT_CLEANUP | |
451 | ||
452 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-2 becomes leader]) | |
453 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
454 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 2 |
455 | AT_CLEANUP | |
456 | ||
457 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-3 becomes leader]) | |
458 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
459 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 3 |
460 | AT_CLEANUP | |
461 | ||
462 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-2 becomes leader]) | |
463 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
464 | ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 2 |
465 | AT_CLEANUP | |
466 | ||
467 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-3 becomes leader]) | |
468 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
469 | ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 3 | |
470 | AT_CLEANUP | |
471 | ||
472 | AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-2 becomes leader]) | |
473 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
474 | ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 2 |
475 | AT_CLEANUP | |
476 | ||
477 | AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-3 becomes leader]) | |
478 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
479 | ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 3 | |
480 | AT_CLEANUP | |
481 | ||
482 | AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-2 becomes leader]) | |
483 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
484 | # XXX: Detect and skip repeated transaction before enabling this test | |
485 | AT_CHECK([exit 77]) | |
486 | ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 2 | |
487 | AT_CLEANUP | |
488 | ||
489 | AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-3 becomes leader]) | |
490 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
491 | # XXX: Detect and skip repeated transaction before enabling this test | |
492 | AT_CHECK([exit 77]) | |
493 | ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 3 | |
494 | AT_CLEANUP | |
495 | ||
496 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to follower-3]) | |
497 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
498 | ovsdb_cluster_failure_test 2 3 2 crash-before-sending-execute-command-request | |
499 | AT_CLEANUP | |
500 | ||
501 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to leader]) | |
502 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
503 | ovsdb_cluster_failure_test 2 1 2 crash-before-sending-execute-command-request | |
504 | AT_CLEANUP | |
505 | ||
506 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to follower-3]) | |
507 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
508 | # XXX: Detect and skip repeated transaction before enabling this test | |
509 | AT_CHECK([exit 77]) | |
510 | ovsdb_cluster_failure_test 2 3 2 crash-after-sending-execute-command-request | |
511 | AT_CLEANUP | |
512 | ||
513 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to leader]) | |
514 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
515 | # XXX: Detect and skip repeated transaction before enabling this test | |
516 | AT_CHECK([exit 77]) | |
517 | ovsdb_cluster_failure_test 2 1 2 crash-after-sending-execute-command-request | |
518 | AT_CLEANUP | |
519 | ||
520 | AT_SETUP([OVSDB cluster - txn on leader, follower-2 crash after receiving appendReq for the update]) | |
521 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
522 | ovsdb_cluster_failure_test 1 1 2 crash-after-receiving-append-request-update | |
523 | AT_CLEANUP | |
524 | ||
525 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-3 crash after receiving appendReq for the update]) | |
526 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
527 | ovsdb_cluster_failure_test 2 2 3 crash-after-receiving-append-request-update | |
528 | AT_CLEANUP | |
529 | ||
93ee4209 HZ |
530 | \f |
531 | AT_SETUP([OVSDB cluster - competing candidates]) | |
532 | AT_KEYWORDS([ovsdb server negative unix cluster competing-candidates]) | |
533 | ||
534 | n=3 | |
535 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
536 | ordinal_schema > schema | |
537 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr]) | |
538 | cid=`ovsdb-tool db-cid s1.db` | |
539 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
540 | for i in `seq 2 $n`; do | |
541 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft]) | |
542 | done | |
543 | ||
544 | on_exit 'kill `cat *.pid`' | |
545 | for i in `seq $n`; do | |
546 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
547 | done | |
548 | for i in `seq $n`; do | |
549 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected]) | |
550 | done | |
551 | ||
552 | # We need to simulate the situation when 2 candidates starts election with same | |
553 | # term. | |
554 | # | |
555 | # Before triggering leader election, tell follower s2 don't send vote request (simulating | |
556 | # vote-request lost or not handled in time), and tell follower s3 to delay | |
557 | # election timer to make sure s3 doesn't send vote-request before s2 enters | |
558 | # term 2. | |
559 | AT_CHECK([ovs-appctl -t "`pwd`"/s2 cluster/failure-test dont-send-vote-request], [0], [ignore]) | |
560 | AT_CHECK([ovs-appctl -t "`pwd`"/s3 cluster/failure-test delay-election], [0], [ignore]) | |
561 | ||
562 | # Restart leader, which will become follower, and both old followers will start | |
563 | # election as candidate. The new follower (old leader) will vote one of them, | |
564 | # and the other candidate should step back as follower as again. | |
565 | kill -9 `cat s1.pid` | |
566 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s1.log --pidfile=s1.pid --unixctl=s1 --remote=punix:s1.ovsdb s1.db]) | |
567 | ||
568 | # Tell s1 to delay election timer so that it won't start election before s3 | |
569 | # becomes candidate. | |
570 | AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/failure-test delay-election], [0], [ignore]) | |
571 | ||
572 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Term: 2"]) | |
573 | ||
574 | for i in `seq $n`; do | |
575 | OVS_WAIT_WHILE([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "candidate"]) | |
576 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected]) | |
577 | done | |
578 | ||
579 | for i in `seq $n`; do | |
580 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
581 | done | |
582 | ||
583 | AT_CLEANUP | |
584 | ||
1b1d2e6d BP |
585 | \f |
586 | AT_BANNER([OVSDB - cluster tests]) | |
587 | ||
588 | # Torture test. | |
589 | OVS_START_SHELL_HELPERS | |
590 | ovsdb_torture_test () { | |
5a0e4aec BP |
591 | local n=$1 # Number of cluster members |
592 | local victim=$2 # Cluster member to kill or remove | |
1b1d2e6d | 593 | local variant=$3 # 'kill' and restart or 'remove' and add |
817db730 | 594 | cp $top_srcdir/vswitchd/vswitch.ovsschema schema |
1b1d2e6d BP |
595 | schema=`ovsdb-tool schema-name schema` |
596 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl | |
817db730 | 597 | ovsdb|WARN|schema: changed 30 columns in 'Open_vSwitch' database from ephemeral to persistent, including 'status' column in 'Manager' table, because clusters do not support ephemeral columns |
1b1d2e6d BP |
598 | ]) |
599 | ||
600 | join_cluster() { | |
601 | local i=$1 | |
5a0e4aec BP |
602 | others= |
603 | for j in `seq 1 $n`; do | |
604 | if test $i != $j; then | |
605 | others="$others unix:s$j.raft" | |
1b1d2e6d | 606 | fi |
5a0e4aec BP |
607 | done |
608 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others]) | |
1b1d2e6d BP |
609 | } |
610 | ||
611 | start_server() { | |
612 | local i=$1 | |
5a0e4aec BP |
613 | printf "\ns$i: starting\n" |
614 | AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
1b1d2e6d BP |
615 | } |
616 | stop_server() { | |
617 | local i=$1 | |
5a0e4aec | 618 | printf "\ns$i: stopping\n" |
1b1d2e6d BP |
619 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) |
620 | } | |
621 | connect_server() { | |
622 | local i=$1 | |
5a0e4aec | 623 | printf "\ns$i: waiting to connect to storage\n" |
d97af428 | 624 | AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected]) |
1b1d2e6d BP |
625 | } |
626 | remove_server() { | |
627 | local i=$1 | |
5a0e4aec | 628 | printf "\ns$i: removing from cluster\n" |
15394e0f | 629 | AT_CHECK([ovs-appctl -t "`pwd`"/s$i cluster/leave Open_vSwitch]) |
5a0e4aec | 630 | printf "\ns$i: waiting for removal to complete\n" |
d97af428 | 631 | AT_CHECK([ovsdb_client_wait --log-file=remove$i.log unix:s$i.ovsdb $schema removed]) |
c7b5c534 | 632 | stop_server $i |
1b1d2e6d BP |
633 | } |
634 | add_server() { | |
635 | local i=$1 | |
5a0e4aec | 636 | rm s$i.db |
1b1d2e6d | 637 | join_cluster $i |
5a0e4aec BP |
638 | start_server $i |
639 | connect_server $i | |
1b1d2e6d BP |
640 | } |
641 | ||
642 | cid=`ovsdb-tool db-cid s1.db` | |
643 | for i in `seq 2 $n`; do join_cluster $i; done | |
644 | ||
645 | on_exit 'kill `cat *.pid`' | |
646 | for i in `seq $n`; do start_server $i; done | |
647 | for i in `seq $n`; do connect_server $i; done | |
648 | ||
817db730 | 649 | db=unix:s1.ovsdb |
1b1d2e6d | 650 | for i in `seq 2 $n`; do |
817db730 | 651 | db=$db,unix:s$i.ovsdb |
1b1d2e6d | 652 | done |
1b1d2e6d | 653 | |
0f954f32 | 654 | n1=10 n2=5 n3=50 |
817db730 | 655 | echo "starting $n1*$n2 ovs-vsctl processes..." |
1b1d2e6d | 656 | for i in $(seq 0 $(expr $n1 - 1) ); do |
5a0e4aec | 657 | (for j in $(seq $n2); do |
1b1d2e6d | 658 | : > $i-$j.running |
817db730 | 659 | txn="add Open_vSwitch . external_ids $i-$j=$i-$j" |
0f954f32 | 660 | for k in $(seq $n3); do |
817db730 | 661 | txn="$txn -- add Open_vSwitch . external_ids $i-$j-$k=$i-$j-$k" |
0f954f32 | 662 | done |
817db730 | 663 | run_as "ovs-vsctl($i-$j)" ovs-vsctl "-vPATTERN:console:ovs-vsctl($i-$j)|%D{%H:%M:%S}|%05N|%c|%p|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --db="$db" --no-leader-only --no-wait $txn |
5a0e4aec BP |
664 | status=$? |
665 | if test $status != 0; then | |
666 | echo "$i-$j exited with status $status" > $i-$j:$status | |
1b1d2e6d BP |
667 | fi |
668 | rm $i-$j.running | |
5a0e4aec BP |
669 | done |
670 | : > $i.done)& | |
1b1d2e6d BP |
671 | done |
672 | echo "...done" | |
1b1d2e6d | 673 | |
817db730 | 674 | echo "waiting for ovs-vsctl processes to exit..." |
6c8dd8ca | 675 | # Use file instead of var because code inside "while" runs in a subshell. |
7ee9c6e0 | 676 | echo 0 > phase |
0f03ae37 | 677 | i=0 |
0f954f32 | 678 | (while :; do echo; sleep 0.1; done) | while read REPLY; do |
1b1d2e6d | 679 | printf "t=%2d s:" $i |
5a0e4aec | 680 | done=0 |
1b1d2e6d | 681 | for j in $(seq 0 $(expr $n1 - 1)); do |
5a0e4aec BP |
682 | if test -f $j.done; then |
683 | printf " $j" | |
684 | done=$(expr $done + 1) | |
1b1d2e6d | 685 | fi |
5a0e4aec BP |
686 | done |
687 | printf '\n' | |
688 | if test $done = $n1; then | |
689 | break | |
1b1d2e6d BP |
690 | fi |
691 | ||
7ee9c6e0 | 692 | case $(cat phase) in # ( |
5a0e4aec | 693 | 0) |
0f954f32 | 694 | if test $done -ge $(expr $n1 / 10); then |
5a0e4aec BP |
695 | if test $variant = kill; then |
696 | stop_server $victim | |
697 | else | |
698 | remove_server $victim | |
699 | fi | |
7ee9c6e0 | 700 | echo 1 > phase |
5a0e4aec | 701 | next=$(expr $i + 2) |
1b1d2e6d | 702 | fi |
5a0e4aec | 703 | ;; # ( |
1b1d2e6d | 704 | 1) |
5a0e4aec BP |
705 | if test $i -ge $next; then |
706 | if test $variant = kill; then | |
707 | start_server $victim | |
708 | connect_server $victim | |
709 | else | |
710 | add_server $victim | |
711 | fi | |
7ee9c6e0 | 712 | echo 2 > phase |
5a0e4aec BP |
713 | fi |
714 | ;; | |
715 | esac | |
716 | ||
717 | i=$(expr $i + 1) | |
1b1d2e6d BP |
718 | done |
719 | echo "...done" | |
7ee9c6e0 | 720 | AT_CHECK([if test $(cat phase) != 2; then exit 77; fi]) |
1b1d2e6d | 721 | |
dff60a1e BP |
722 | for i in $(seq 0 $(expr $n1 - 1) ); do |
723 | for j in `seq $n2`; do | |
724 | echo "$i-$j=$i-$j" | |
0f954f32 HZ |
725 | for k in `seq $n3`; do |
726 | echo "$i-$j-$k=$i-$j-$k" | |
727 | done | |
dff60a1e BP |
728 | done |
729 | done | sort > expout | |
15394e0f | 730 | AT_CHECK([ovs-vsctl --db="$db" --no-wait --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get Open_vSwitch . external-ids | tr ',' '\n' | sed 's/[[{}"" ]]//g' | sort], [0], [expout]) |
1b1d2e6d BP |
731 | |
732 | for i in `seq $n`; do | |
7ee9c6e0 | 733 | if test $i != $victim || test $(cat phase) != 1; then |
5a0e4aec BP |
734 | stop_server $i |
735 | fi | |
1b1d2e6d BP |
736 | done |
737 | ||
738 | # We ignore stdout because non-fatal warnings get printed there. | |
739 | AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore]) | |
740 | } | |
741 | OVS_END_SHELL_HELPERS | |
742 | ||
743 | AT_SETUP([OVSDB 3-server torture test - kill/restart leader]) | |
744 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
745 | ovsdb_torture_test 3 1 kill | |
746 | AT_CLEANUP | |
747 | AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1]) | |
748 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
749 | ovsdb_torture_test 3 2 kill | |
750 | AT_CLEANUP | |
751 | AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2]) | |
752 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
753 | ovsdb_torture_test 3 3 kill | |
754 | AT_CLEANUP | |
755 | AT_SETUP([OVSDB 5-server torture test - kill/restart leader]) | |
756 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
757 | ovsdb_torture_test 5 1 kill | |
758 | AT_CLEANUP | |
759 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1]) | |
760 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
761 | ovsdb_torture_test 5 2 kill | |
762 | AT_CLEANUP | |
763 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2]) | |
764 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
765 | ovsdb_torture_test 5 3 kill | |
766 | AT_CLEANUP | |
767 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3]) | |
768 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
769 | ovsdb_torture_test 5 4 kill | |
770 | AT_CLEANUP | |
771 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4]) | |
772 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
773 | ovsdb_torture_test 5 5 kill | |
774 | AT_CLEANUP | |
775 | ||
776 | AT_SETUP([OVSDB 3-server torture test - remove/re-add leader]) | |
777 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
778 | ovsdb_torture_test 3 1 remove | |
779 | AT_CLEANUP | |
780 | AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1]) | |
781 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
782 | ovsdb_torture_test 3 2 remove | |
783 | AT_CLEANUP | |
784 | AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2]) | |
785 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
786 | ovsdb_torture_test 3 3 remove | |
787 | AT_CLEANUP | |
788 | AT_SETUP([OVSDB 5-server torture test - remove/re-add leader]) | |
789 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
790 | ovsdb_torture_test 5 1 remove | |
791 | AT_CLEANUP | |
792 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1]) | |
793 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
794 | ovsdb_torture_test 5 2 remove | |
795 | AT_CLEANUP | |
796 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2]) | |
797 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
798 | ovsdb_torture_test 5 3 remove | |
799 | AT_CLEANUP | |
800 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3]) | |
801 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
802 | ovsdb_torture_test 5 4 remove | |
803 | AT_CLEANUP | |
804 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4]) | |
805 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
806 | ovsdb_torture_test 5 5 remove | |
807 | AT_CLEANUP |