1 OVS_START_SHELL_HELPERS
2 # ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT TRANSACTION...
3 ovsdb_check_cluster () {
4 local n=$1 schema_func=$2 output=$3
8 schema=`ovsdb-tool schema-name schema`
9 AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
10 AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1])
11 cid=`ovsdb-tool db-cid s1.db`
12 for i in `seq 2 $n`; do
13 AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft])
16 on_exit 'kill `cat *.pid`'
18 AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
21 AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema connected])
26 AT_CHECK([ovsdb-client -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout])
29 AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output])
31 OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
34 AT_CHECK([ovsdb-tool check-cluster s*.db])
38 # Test a 1-server cluster.
39 AT_BANNER([OVSDB - clustered transactions (1 server)])
40 m4_define([OVSDB_CHECK_EXECUTION],
41 [AT_SETUP([$1 - cluster of 1])
42 AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5])
43 ovsdb_check_cluster 1 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
47 # Test a 3-server cluster.
48 AT_BANNER([OVSDB - clustered transactions (3 servers)])
49 m4_define([OVSDB_CHECK_EXECUTION],
50 [AT_SETUP([$1 - cluster of 3])
51 AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5])
52 ovsdb_check_cluster 3 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
56 # Test a 5-server cluster.
57 AT_BANNER([OVSDB - clustered transactions (5 servers)])
58 m4_define([OVSDB_CHECK_EXECUTION],
59 [AT_SETUP([$1 - cluster of 5])
60 AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5])
61 ovsdb_check_cluster 5 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
66 AT_BANNER([OVSDB - disconnect from cluster])
68 OVS_START_SHELL_HELPERS
69 # ovsdb_test_cluster_disconnect N_SERVERS LEADER_OR_FOLLOWER [CHECK_FLAPPING]
70 # Test server disconnected from the cluster.
71 # N_SERVERS: Number of servers in the cluster.
72 # LEADER_OR_FOLLOWER: The role of the server that is disconnected from the
73 # cluster: "leader" or "follower".
74 # CHECK_FLAPPING: Whether to check if is_disconnected flapped. "yes", "no".
75 ovsdb_test_cluster_disconnect () {
79 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
80 ordinal_schema > schema
81 AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
82 cid=`ovsdb-tool db-cid s1.db`
83 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
84 for i in `seq 2 $n`; do
85 AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
88 on_exit 'kill `cat *.pid`'
90 AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
93 AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
96 AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
99 "row": {"i": 1}}]]'], [0], [ignore], [ignore])
101 # When a node is disconnected from the cluster, the IDL should disconnect
102 # and retry even if it uses a single remote, because the remote IP can be
103 # a VIP on a load-balance. So we use single remote to test here.
104 if test $leader_or_follower == "leader"; then
106 shutdown=`seq $(($n/2 + 1)) $n`
107 cleanup=`seq $(($n/2))`
111 # shutdown followers before the leader (s1) so that there is no chance for
112 # s$n to become leader during the process.
113 shutdown="`seq 2 $(($n/2 + 1))` 1"
114 cleanup=`seq $(($n/2 + 2)) $n`
116 echo shutdown=$shutdown
117 echo cleanup=$cleanup
119 # Connect to $target. Use "wait" to trigger a non-op transaction so
120 # that test-ovsdb will not quit.
122 test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -v -t10 idl unix:s$target.ovsdb '[["idltest",
125 "where": [["i", "==", 1]],
128 "rows": [{"i": 1}]}]]' > test-ovsdb.log 2>&1 &
129 echo $! > test-ovsdb.pid
131 OVS_WAIT_UNTIL([grep "000: i=1" test-ovsdb.log])
133 # Start collecting raft_is_connected logs for $target before shutting down
135 tail -f s$target.log > raft_is_connected.log &
138 # Shutdown the other servers so that $target is disconnected from the cluster.
139 for i in $shutdown; do
140 OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
143 # The test-ovsdb should detect the disconnect and retry.
144 OVS_WAIT_UNTIL([grep disconnect test-ovsdb.log])
146 # The $target debug log should show raft_is_connected: false.
147 OVS_WAIT_UNTIL([grep "raft_is_connected: false" raft_is_connected.log])
149 # Save the current count of "raft_is_connected: true"
150 count_old=`grep "raft_is_connected: true" raft_is_connected.log | wc -l`
151 echo count_old $count_old
153 if test X$check_flapping == X"yes"; then
156 # Make sure raft_is_connected didn't flap from false to true.
157 count_new=`grep "raft_is_connected: true" raft_is_connected.log | wc -l`
158 echo count_new $count_new
159 AT_CHECK([test $count_new == $count_old])
161 for i in $cleanup; do
162 OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
165 OVS_END_SHELL_HELPERS
167 AT_SETUP([OVSDB cluster - follower disconnect from cluster, single remote])
168 AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
169 ovsdb_test_cluster_disconnect 3 follower
172 AT_SETUP([OVSDB cluster - leader disconnect from cluster, single remote])
173 AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
174 ovsdb_test_cluster_disconnect 3 leader
177 AT_SETUP([OVSDB cluster - leader disconnect from cluster, check flapping])
178 AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
179 ovsdb_test_cluster_disconnect 5 leader yes
184 AT_BANNER([OVSDB cluster election timer change])
186 AT_SETUP([OVSDB cluster - election timer change])
187 AT_KEYWORDS([ovsdb server positive unix cluster timer])
190 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
191 ordinal_schema > schema
192 AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
193 cid=`ovsdb-tool db-cid s1.db`
194 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
195 for i in `seq 2 $n`; do
196 AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
199 on_exit 'kill `cat *.pid`'
200 for i in `seq $n`; do
201 AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
203 for i in `seq $n`; do
204 AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
207 # Change not allowed through follower.
208 AT_CHECK([ovs-appctl -t "`pwd`"/s2 cluster/change-election-timer $schema_name 2000], [2], [], [ignore])
210 # Timer cannot be changed to bigger than 2x the original value.
211 AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [2], [], [ignore])
213 AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 2000], [0], [dnl
214 change of election timer initiated.
216 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 2000"])
217 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 2000"])
219 AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [0], [dnl
220 change of election timer initiated.
222 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"])
223 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"])
225 # Latest timer should be used after restart
226 for i in `seq $n`; do
227 printf "\ns$i: stopping\n"
228 OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
230 for i in `seq $n`; do
231 AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
233 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"])
234 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"])
240 OVS_START_SHELL_HELPERS
241 # ovsdb_cluster_failure_test SCHEMA_FUNC OUTPUT TRANSACTION...
242 ovsdb_cluster_failure_test () {
243 # Initial state: s1 is leader, s2 and s3 are followers
248 if test "$crash_node" == "1"; then
252 cp $top_srcdir/vswitchd/vswitch.ovsschema schema
253 schema=`ovsdb-tool schema-name schema`
254 AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl
255 ovsdb|WARN|schema: changed 30 columns in 'Open_vSwitch' database from ephemeral to persistent, including 'status' column in 'Manager' table, because clusters do not support ephemeral columns
262 for j in `seq 1 $n`; do
263 if test $i != $j; then
264 others="$others unix:s$j.raft"
267 AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
271 printf "\ns$i: starting\n"
272 AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
276 printf "\ns$i: waiting to connect to storage\n"
277 AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
279 cid=`ovsdb-tool db-cid s1.db`
280 for i in `seq 2 $n`; do join_cluster $i; done
282 on_exit 'kill `cat *.pid`'
283 for i in `seq $n`; do start_server $i; done
284 for i in `seq $n`; do connect_server $i; done
286 db=unix:s$remote_1.ovsdb,unix:s$remote_2.ovsdb
288 # To ensure $new_leader node the new leader, we delay election timer for
289 # the other follower.
290 if test -n "$new_leader"; then
291 if test "$new_leader" == "2"; then
292 delay_election_node=3
294 delay_election_node=2
296 AT_CHECK([ovs-appctl -t "`pwd`"/s$delay_election_node cluster/failure-test delay-election], [0], [ignore])
298 AT_CHECK([ovs-appctl -t "`pwd`"/s$crash_node cluster/failure-test $crash_command], [0], [ignore])
299 AT_CHECK([ovs-vsctl -v --db="$db" --no-leader-only --no-shuffle-remotes --no-wait create QoS type=x], [0], [ignore], [ignore])
301 # Make sure that the node really crashed.
302 AT_CHECK([ls s$crash_node.ovsdb], [2], [ignore], [ignore])
303 # XXX: Client will fail if remotes contains unix socket that doesn't exist (killed).
304 if test "$remote_1" = "$crash_node"; then
305 db=unix:s$remote_2.ovsdb
307 AT_CHECK([ovs-vsctl --db="$db" --no-leader-only --no-wait --columns=type --bare list QoS], [0], [x
310 OVS_END_SHELL_HELPERS
311 AT_BANNER([OVSDB - cluster failure with pending transaction])
313 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-2 becomes leader])
314 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
315 ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 2
318 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-3 becomes leader])
319 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
320 ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 3
323 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-2 becomes leader])
324 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
325 ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 2
328 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-3 becomes leader])
329 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
330 ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 3
333 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-2 becomes leader])
334 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
335 ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 2
338 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-3 becomes leader])
339 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
340 ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 3
343 AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-2 becomes leader])
344 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
345 ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 2
348 AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-3 becomes leader])
349 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
350 ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 3
353 AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-2 becomes leader])
354 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
355 # XXX: Detect and skip repeated transaction before enabling this test
357 ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 2
360 AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-3 becomes leader])
361 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
362 # XXX: Detect and skip repeated transaction before enabling this test
364 ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 3
367 AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to follower-3])
368 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
369 ovsdb_cluster_failure_test 2 3 2 crash-before-sending-execute-command-request
372 AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to leader])
373 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
374 ovsdb_cluster_failure_test 2 1 2 crash-before-sending-execute-command-request
377 AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to follower-3])
378 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
379 # XXX: Detect and skip repeated transaction before enabling this test
381 ovsdb_cluster_failure_test 2 3 2 crash-after-sending-execute-command-request
384 AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to leader])
385 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
386 # XXX: Detect and skip repeated transaction before enabling this test
388 ovsdb_cluster_failure_test 2 1 2 crash-after-sending-execute-command-request
391 AT_SETUP([OVSDB cluster - txn on leader, follower-2 crash after receiving appendReq for the update])
392 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
393 ovsdb_cluster_failure_test 1 1 2 crash-after-receiving-append-request-update
396 AT_SETUP([OVSDB cluster - txn on follower-2, follower-3 crash after receiving appendReq for the update])
397 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
398 ovsdb_cluster_failure_test 2 2 3 crash-after-receiving-append-request-update
402 AT_BANNER([OVSDB - cluster tests])
405 OVS_START_SHELL_HELPERS
406 ovsdb_torture_test () {
407 local n=$1 # Number of cluster members
408 local victim=$2 # Cluster member to kill or remove
409 local variant=$3 # 'kill' and restart or 'remove' and add
410 cp $top_srcdir/vswitchd/vswitch.ovsschema schema
411 schema=`ovsdb-tool schema-name schema`
412 AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl
413 ovsdb|WARN|schema: changed 30 columns in 'Open_vSwitch' database from ephemeral to persistent, including 'status' column in 'Manager' table, because clusters do not support ephemeral columns
419 for j in `seq 1 $n`; do
420 if test $i != $j; then
421 others="$others unix:s$j.raft"
424 AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
429 printf "\ns$i: starting\n"
430 AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
434 printf "\ns$i: stopping\n"
435 OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
439 printf "\ns$i: waiting to connect to storage\n"
440 AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
444 printf "\ns$i: removing from cluster\n"
445 AT_CHECK([ovs-appctl -t "`pwd`"/s$i cluster/leave Open_vSwitch])
446 printf "\ns$i: waiting for removal to complete\n"
447 AT_CHECK([ovsdb_client_wait --log-file=remove$i.log unix:s$i.ovsdb $schema removed])
458 cid=`ovsdb-tool db-cid s1.db`
459 for i in `seq 2 $n`; do join_cluster $i; done
461 on_exit 'kill `cat *.pid`'
462 for i in `seq $n`; do start_server $i; done
463 for i in `seq $n`; do connect_server $i; done
466 for i in `seq 2 $n`; do
467 db=$db,unix:s$i.ovsdb
471 echo "starting $n1*$n2 ovs-vsctl processes..."
472 for i in $(seq 0 $(expr $n1 - 1) ); do
473 (for j in $(seq $n2); do
475 txn="add Open_vSwitch . external_ids $i-$j=$i-$j"
476 for k in $(seq $n3); do
477 txn="$txn -- add Open_vSwitch . external_ids $i-$j-$k=$i-$j-$k"
479 run_as "ovs-vsctl($i-$j)" ovs-vsctl "-vPATTERN:console:ovs-vsctl($i-$j)|%D{%H:%M:%S}|%05N|%c|%p|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --db="$db" --no-leader-only --no-wait $txn
481 if test $status != 0; then
482 echo "$i-$j exited with status $status" > $i-$j:$status
490 echo "waiting for ovs-vsctl processes to exit..."
491 # Use file instead of var because code inside "while" runs in a subshell.
494 (while :; do echo; sleep 0.1; done) | while read REPLY; do
497 for j in $(seq 0 $(expr $n1 - 1)); do
498 if test -f $j.done; then
500 done=$(expr $done + 1)
504 if test $done = $n1; then
508 case $(cat phase) in # (
510 if test $done -ge $(expr $n1 / 10); then
511 if test $variant = kill; then
514 remove_server $victim
521 if test $i -ge $next; then
522 if test $variant = kill; then
524 connect_server $victim
536 AT_CHECK([if test $(cat phase) != 2; then exit 77; fi])
538 for i in $(seq 0 $(expr $n1 - 1) ); do
539 for j in `seq $n2`; do
541 for k in `seq $n3`; do
542 echo "$i-$j-$k=$i-$j-$k"
546 AT_CHECK([ovs-vsctl --db="$db" --no-wait --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get Open_vSwitch . external-ids | tr ',' '\n' | sed 's/[[{}"" ]]//g' | sort], [0], [expout])
548 for i in `seq $n`; do
549 if test $i != $victim || test $(cat phase) != 1; then
554 # We ignore stdout because non-fatal warnings get printed there.
555 AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore])
557 OVS_END_SHELL_HELPERS
559 AT_SETUP([OVSDB 3-server torture test - kill/restart leader])
560 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
561 ovsdb_torture_test 3 1 kill
563 AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1])
564 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
565 ovsdb_torture_test 3 2 kill
567 AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2])
568 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
569 ovsdb_torture_test 3 3 kill
571 AT_SETUP([OVSDB 5-server torture test - kill/restart leader])
572 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
573 ovsdb_torture_test 5 1 kill
575 AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1])
576 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
577 ovsdb_torture_test 5 2 kill
579 AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2])
580 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
581 ovsdb_torture_test 5 3 kill
583 AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3])
584 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
585 ovsdb_torture_test 5 4 kill
587 AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4])
588 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
589 ovsdb_torture_test 5 5 kill
592 AT_SETUP([OVSDB 3-server torture test - remove/re-add leader])
593 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
594 ovsdb_torture_test 3 1 remove
596 AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1])
597 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
598 ovsdb_torture_test 3 2 remove
600 AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2])
601 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
602 ovsdb_torture_test 3 3 remove
604 AT_SETUP([OVSDB 5-server torture test - remove/re-add leader])
605 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
606 ovsdb_torture_test 5 1 remove
608 AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1])
609 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
610 ovsdb_torture_test 5 2 remove
612 AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2])
613 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
614 ovsdb_torture_test 5 3 remove
616 AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3])
617 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
618 ovsdb_torture_test 5 4 remove
620 AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4])
621 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
622 ovsdb_torture_test 5 5 remove