]>
Commit | Line | Data |
---|---|---|
1 | OVS_START_SHELL_HELPERS | |
2 | # ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT TRANSACTION... | |
3 | ovsdb_check_cluster () { | |
4 | local n=$1 schema_func=$2 output=$3 | |
5 | shift; shift; shift | |
6 | ||
7 | $schema_func > schema | |
8 | schema=`ovsdb-tool schema-name schema` | |
9 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr]) | |
10 | AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1]) | |
11 | cid=`ovsdb-tool db-cid s1.db` | |
12 | for i in `seq 2 $n`; do | |
13 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft]) | |
14 | done | |
15 | ||
16 | on_exit 'kill `cat *.pid`' | |
17 | for i in `seq $n`; do | |
18 | AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
19 | done | |
20 | for i in `seq $n`; do | |
21 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema connected]) | |
22 | done | |
23 | ||
24 | for txn | |
25 | do | |
26 | AT_CHECK([ovsdb-client --timeout=30 -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout]) | |
27 | cat stdout >> output | |
28 | done | |
29 | AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output]) | |
30 | for i in `seq $n`; do | |
31 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
32 | done | |
33 | ||
34 | AT_CHECK([ovsdb-tool check-cluster s*.db]) | |
35 | } | |
36 | OVS_END_SHELL_HELPERS | |
37 | ||
38 | # Test a 1-server cluster. | |
39 | AT_BANNER([OVSDB - clustered transactions (1 server)]) | |
40 | m4_define([OVSDB_CHECK_EXECUTION], | |
41 | [AT_SETUP([$1 - cluster of 1]) | |
42 | AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5]) | |
43 | ovsdb_check_cluster 1 "$2" '$4' m4_foreach([txn], [$3], ['txn' ]) | |
44 | AT_CLEANUP]) | |
45 | EXECUTION_EXAMPLES | |
46 | ||
47 | # Test a 3-server cluster. | |
48 | AT_BANNER([OVSDB - clustered transactions (3 servers)]) | |
49 | m4_define([OVSDB_CHECK_EXECUTION], | |
50 | [AT_SETUP([$1 - cluster of 3]) | |
51 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5]) | |
52 | ovsdb_check_cluster 3 "$2" '$4' m4_foreach([txn], [$3], ['txn' ]) | |
53 | AT_CLEANUP]) | |
54 | EXECUTION_EXAMPLES | |
55 | ||
56 | # Test a 5-server cluster. | |
57 | AT_BANNER([OVSDB - clustered transactions (5 servers)]) | |
58 | m4_define([OVSDB_CHECK_EXECUTION], | |
59 | [AT_SETUP([$1 - cluster of 5]) | |
60 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5]) | |
61 | ovsdb_check_cluster 5 "$2" '$4' m4_foreach([txn], [$3], ['txn' ]) | |
62 | AT_CLEANUP]) | |
63 | EXECUTION_EXAMPLES | |
64 | \f | |
65 | ||
66 | OVS_START_SHELL_HELPERS | |
67 | # ovsdb_cluster_failure_test SCHEMA_FUNC OUTPUT TRANSACTION... | |
68 | ovsdb_cluster_failure_test () { | |
69 | # Initial state: s1 is leader, s2 and s3 are followers | |
70 | remote_1=$1 | |
71 | remote_2=$2 | |
72 | crash_node=$3 | |
73 | crash_command=$4 | |
74 | if test "$crash_node" == "1"; then | |
75 | new_leader=$5 | |
76 | fi | |
77 | ||
78 | cp $top_srcdir/ovn/ovn-nb.ovsschema schema | |
79 | schema=`ovsdb-tool schema-name schema` | |
80 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl | |
81 | ovsdb|WARN|schema: changed 2 columns in 'OVN_Northbound' database from ephemeral to persistent, including 'status' column in 'Connection' table, because clusters do not support ephemeral columns | |
82 | ]) | |
83 | ||
84 | n=3 | |
85 | join_cluster() { | |
86 | local i=$1 | |
87 | others= | |
88 | for j in `seq 1 $n`; do | |
89 | if test $i != $j; then | |
90 | others="$others unix:s$j.raft" | |
91 | fi | |
92 | done | |
93 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others]) | |
94 | } | |
95 | start_server() { | |
96 | local i=$1 | |
97 | printf "\ns$i: starting\n" | |
98 | AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
99 | } | |
100 | connect_server() { | |
101 | local i=$1 | |
102 | printf "\ns$i: waiting to connect to storage\n" | |
103 | AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected]) | |
104 | } | |
105 | cid=`ovsdb-tool db-cid s1.db` | |
106 | for i in `seq 2 $n`; do join_cluster $i; done | |
107 | ||
108 | on_exit 'kill `cat *.pid`' | |
109 | for i in `seq $n`; do start_server $i; done | |
110 | for i in `seq $n`; do connect_server $i; done | |
111 | ||
112 | export OVN_NB_DB=unix:s$remote_1.ovsdb,unix:s$remote_2.ovsdb | |
113 | ||
114 | # To ensure $new_leader node the new leader, we delay election timer for | |
115 | # the other follower. | |
116 | if test -n "$new_leader"; then | |
117 | if test "$new_leader" == "2"; then | |
118 | delay_election_node=3 | |
119 | else | |
120 | delay_election_node=2 | |
121 | fi | |
122 | AT_CHECK([ovs-appctl -t "`pwd`"/s$delay_election_node cluster/failure-test delay-election], [0], [ignore]) | |
123 | fi | |
124 | AT_CHECK([ovs-appctl -t "`pwd`"/s$crash_node cluster/failure-test $crash_command], [0], [ignore]) | |
125 | AT_CHECK([ovn-nbctl -v --timeout=10 --no-leader-only --no-shuffle-remotes create logical_switch name=ls1], [0], [ignore], [ignore]) | |
126 | ||
127 | # Make sure that the node really crashed. | |
128 | AT_CHECK([ls s$crash_node.ovsdb], [2], [ignore], [ignore]) | |
129 | # XXX: Client will fail if remotes contains unix socket that doesn't exist (killed). | |
130 | if test "$remote_1" == "$crash_node"; then | |
131 | export OVN_NB_DB=unix:s$remote_2.ovsdb | |
132 | fi | |
133 | AT_CHECK([ovn-nbctl --no-leader-only ls-list | awk '{ print $2 }'], [0], [(ls1) | |
134 | ]) | |
135 | } | |
136 | OVS_END_SHELL_HELPERS | |
137 | AT_BANNER([OVSDB - cluster failure with pending transaction]) | |
138 | ||
139 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-2 becomes leader]) | |
140 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
141 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 2 | |
142 | AT_CLEANUP | |
143 | ||
144 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-3 becomes leader]) | |
145 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
146 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 3 | |
147 | AT_CLEANUP | |
148 | ||
149 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-2 becomes leader]) | |
150 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
151 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 2 | |
152 | AT_CLEANUP | |
153 | ||
154 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-3 becomes leader]) | |
155 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
156 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 3 | |
157 | AT_CLEANUP | |
158 | ||
159 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-2 becomes leader]) | |
160 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
161 | ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 2 | |
162 | AT_CLEANUP | |
163 | ||
164 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-3 becomes leader]) | |
165 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
166 | ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 3 | |
167 | AT_CLEANUP | |
168 | ||
169 | AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-2 becomes leader]) | |
170 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
171 | ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 2 | |
172 | AT_CLEANUP | |
173 | ||
174 | AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-3 becomes leader]) | |
175 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
176 | ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 3 | |
177 | AT_CLEANUP | |
178 | ||
179 | AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-2 becomes leader]) | |
180 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
181 | # XXX: Detect and skip repeated transaction before enabling this test | |
182 | AT_CHECK([exit 77]) | |
183 | ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 2 | |
184 | AT_CLEANUP | |
185 | ||
186 | AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-3 becomes leader]) | |
187 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
188 | # XXX: Detect and skip repeated transaction before enabling this test | |
189 | AT_CHECK([exit 77]) | |
190 | ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 3 | |
191 | AT_CLEANUP | |
192 | ||
193 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to follower-3]) | |
194 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
195 | ovsdb_cluster_failure_test 2 3 2 crash-before-sending-execute-command-request | |
196 | AT_CLEANUP | |
197 | ||
198 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to leader]) | |
199 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
200 | ovsdb_cluster_failure_test 2 1 2 crash-before-sending-execute-command-request | |
201 | AT_CLEANUP | |
202 | ||
203 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to follower-3]) | |
204 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
205 | # XXX: Detect and skip repeated transaction before enabling this test | |
206 | AT_CHECK([exit 77]) | |
207 | ovsdb_cluster_failure_test 2 3 2 crash-after-sending-execute-command-request | |
208 | AT_CLEANUP | |
209 | ||
210 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to leader]) | |
211 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
212 | # XXX: Detect and skip repeated transaction before enabling this test | |
213 | AT_CHECK([exit 77]) | |
214 | ovsdb_cluster_failure_test 2 1 2 crash-after-sending-execute-command-request | |
215 | AT_CLEANUP | |
216 | ||
217 | AT_SETUP([OVSDB cluster - txn on leader, follower-2 crash after receiving appendReq for the update]) | |
218 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
219 | ovsdb_cluster_failure_test 1 1 2 crash-after-receiving-append-request-update | |
220 | AT_CLEANUP | |
221 | ||
222 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-3 crash after receiving appendReq for the update]) | |
223 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
224 | ovsdb_cluster_failure_test 2 2 3 crash-after-receiving-append-request-update | |
225 | AT_CLEANUP | |
226 | ||
227 | \f | |
228 | AT_BANNER([OVSDB - cluster tests]) | |
229 | ||
230 | # Torture test. | |
231 | OVS_START_SHELL_HELPERS | |
232 | ovsdb_torture_test () { | |
233 | local n=$1 # Number of cluster members | |
234 | local victim=$2 # Cluster member to kill or remove | |
235 | local variant=$3 # 'kill' and restart or 'remove' and add | |
236 | cp $top_srcdir/ovn/ovn-sb.ovsschema schema | |
237 | schema=`ovsdb-tool schema-name schema` | |
238 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl | |
239 | ovsdb|WARN|schema: changed 2 columns in 'OVN_Southbound' database from ephemeral to persistent, including 'status' column in 'Connection' table, because clusters do not support ephemeral columns | |
240 | ]) | |
241 | ||
242 | join_cluster() { | |
243 | local i=$1 | |
244 | others= | |
245 | for j in `seq 1 $n`; do | |
246 | if test $i != $j; then | |
247 | others="$others unix:s$j.raft" | |
248 | fi | |
249 | done | |
250 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others]) | |
251 | } | |
252 | ||
253 | start_server() { | |
254 | local i=$1 | |
255 | printf "\ns$i: starting\n" | |
256 | AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
257 | } | |
258 | stop_server() { | |
259 | local i=$1 | |
260 | printf "\ns$i: stopping\n" | |
261 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
262 | } | |
263 | connect_server() { | |
264 | local i=$1 | |
265 | printf "\ns$i: waiting to connect to storage\n" | |
266 | AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected]) | |
267 | } | |
268 | remove_server() { | |
269 | local i=$1 | |
270 | printf "\ns$i: removing from cluster\n" | |
271 | AT_CHECK([ovs-appctl --timeout=30 -t "`pwd`"/s$i cluster/leave OVN_Southbound]) | |
272 | printf "\ns$i: waiting for removal to complete\n" | |
273 | AT_CHECK([ovsdb_client_wait --log-file=remove$i.log unix:s$i.ovsdb $schema removed]) | |
274 | stop_server $i | |
275 | } | |
276 | add_server() { | |
277 | local i=$1 | |
278 | rm s$i.db | |
279 | join_cluster $i | |
280 | start_server $i | |
281 | connect_server $i | |
282 | } | |
283 | ||
284 | cid=`ovsdb-tool db-cid s1.db` | |
285 | for i in `seq 2 $n`; do join_cluster $i; done | |
286 | ||
287 | on_exit 'kill `cat *.pid`' | |
288 | for i in `seq $n`; do start_server $i; done | |
289 | for i in `seq $n`; do connect_server $i; done | |
290 | ||
291 | OVN_SB_DB=unix:s1.ovsdb | |
292 | for i in `seq 2 $n`; do | |
293 | OVN_SB_DB=$OVN_SB_DB,unix:s$i.ovsdb | |
294 | done | |
295 | export OVN_SB_DB | |
296 | ||
297 | n1=10 n2=5 n3=50 | |
298 | echo "starting $n1*$n2 ovn-sbctl processes..." | |
299 | for i in $(seq 0 $(expr $n1 - 1) ); do | |
300 | (for j in $(seq $n2); do | |
301 | : > $i-$j.running | |
302 | txn="add SB_Global . external_ids $i-$j=$i-$j" | |
303 | for k in $(seq $n3); do | |
304 | txn="$txn -- add SB_Global . external_ids $i-$j-$k=$i-$j-$k" | |
305 | done | |
306 | run_as "ovn-sbctl($i-$j)" ovn-sbctl "-vPATTERN:console:ovn-sbctl($i-$j)|%D{%H:%M:%S}|%05N|%c|%p|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --no-leader-only $txn | |
307 | status=$? | |
308 | if test $status != 0; then | |
309 | echo "$i-$j exited with status $status" > $i-$j:$status | |
310 | fi | |
311 | rm $i-$j.running | |
312 | done | |
313 | : > $i.done)& | |
314 | done | |
315 | echo "...done" | |
316 | ||
317 | echo "waiting for ovn-sbctl processes to exit..." | |
318 | # Use file instead of var because code inside "while" runs in a subshell. | |
319 | echo 0 > phase | |
320 | i=0 | |
321 | (while :; do echo; sleep 0.1; done) | while read REPLY; do | |
322 | printf "t=%2d s:" $i | |
323 | done=0 | |
324 | for j in $(seq 0 $(expr $n1 - 1)); do | |
325 | if test -f $j.done; then | |
326 | printf " $j" | |
327 | done=$(expr $done + 1) | |
328 | fi | |
329 | done | |
330 | printf '\n' | |
331 | if test $done = $n1; then | |
332 | break | |
333 | fi | |
334 | ||
335 | case $(cat phase) in # ( | |
336 | 0) | |
337 | if test $done -ge $(expr $n1 / 10); then | |
338 | if test $variant = kill; then | |
339 | stop_server $victim | |
340 | else | |
341 | remove_server $victim | |
342 | fi | |
343 | echo 1 > phase | |
344 | next=$(expr $i + 2) | |
345 | fi | |
346 | ;; # ( | |
347 | 1) | |
348 | if test $i -ge $next; then | |
349 | if test $variant = kill; then | |
350 | start_server $victim | |
351 | connect_server $victim | |
352 | else | |
353 | add_server $victim | |
354 | fi | |
355 | echo 2 > phase | |
356 | fi | |
357 | ;; | |
358 | esac | |
359 | ||
360 | i=$(expr $i + 1) | |
361 | done | |
362 | echo "...done" | |
363 | AT_CHECK([if test $(cat phase) != 2; then exit 77; fi]) | |
364 | ||
365 | for i in $(seq 0 $(expr $n1 - 1) ); do | |
366 | for j in `seq $n2`; do | |
367 | echo "$i-$j=$i-$j" | |
368 | for k in `seq $n3`; do | |
369 | echo "$i-$j-$k=$i-$j-$k" | |
370 | done | |
371 | done | |
372 | done | sort > expout | |
373 | AT_CHECK([ovn-sbctl --timeout=30 --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get SB_Global . external-ids | tr ',' '\n' | sed 's/[[{}"" ]]//g' | sort], [0], [expout]) | |
374 | ||
375 | for i in `seq $n`; do | |
376 | if test $i != $victim || test $(cat phase) != 1; then | |
377 | stop_server $i | |
378 | fi | |
379 | done | |
380 | ||
381 | # We ignore stdout because non-fatal warnings get printed there. | |
382 | AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore]) | |
383 | } | |
384 | OVS_END_SHELL_HELPERS | |
385 | ||
386 | AT_SETUP([OVSDB 3-server torture test - kill/restart leader]) | |
387 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
388 | ovsdb_torture_test 3 1 kill | |
389 | AT_CLEANUP | |
390 | AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1]) | |
391 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
392 | ovsdb_torture_test 3 2 kill | |
393 | AT_CLEANUP | |
394 | AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2]) | |
395 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
396 | ovsdb_torture_test 3 3 kill | |
397 | AT_CLEANUP | |
398 | AT_SETUP([OVSDB 5-server torture test - kill/restart leader]) | |
399 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
400 | ovsdb_torture_test 5 1 kill | |
401 | AT_CLEANUP | |
402 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1]) | |
403 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
404 | ovsdb_torture_test 5 2 kill | |
405 | AT_CLEANUP | |
406 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2]) | |
407 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
408 | ovsdb_torture_test 5 3 kill | |
409 | AT_CLEANUP | |
410 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3]) | |
411 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
412 | ovsdb_torture_test 5 4 kill | |
413 | AT_CLEANUP | |
414 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4]) | |
415 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
416 | ovsdb_torture_test 5 5 kill | |
417 | AT_CLEANUP | |
418 | ||
419 | AT_SETUP([OVSDB 3-server torture test - remove/re-add leader]) | |
420 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
421 | ovsdb_torture_test 3 1 remove | |
422 | AT_CLEANUP | |
423 | AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1]) | |
424 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
425 | ovsdb_torture_test 3 2 remove | |
426 | AT_CLEANUP | |
427 | AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2]) | |
428 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
429 | ovsdb_torture_test 3 3 remove | |
430 | AT_CLEANUP | |
431 | AT_SETUP([OVSDB 5-server torture test - remove/re-add leader]) | |
432 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
433 | ovsdb_torture_test 5 1 remove | |
434 | AT_CLEANUP | |
435 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1]) | |
436 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
437 | ovsdb_torture_test 5 2 remove | |
438 | AT_CLEANUP | |
439 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2]) | |
440 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
441 | ovsdb_torture_test 5 3 remove | |
442 | AT_CLEANUP | |
443 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3]) | |
444 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
445 | ovsdb_torture_test 5 4 remove | |
446 | AT_CLEANUP | |
447 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4]) | |
448 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
449 | ovsdb_torture_test 5 5 remove | |
450 | AT_CLEANUP |