]>
Commit | Line | Data |
---|---|---|
1b1d2e6d BP |
1 | OVS_START_SHELL_HELPERS |
2 | # ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT TRANSACTION... | |
3 | ovsdb_check_cluster () { | |
4 | local n=$1 schema_func=$2 output=$3 | |
5 | shift; shift; shift | |
6 | ||
7 | $schema_func > schema | |
8 | schema=`ovsdb-tool schema-name schema` | |
9 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr]) | |
10 | AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1]) | |
11 | cid=`ovsdb-tool db-cid s1.db` | |
12 | for i in `seq 2 $n`; do | |
13 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft]) | |
14 | done | |
15 | ||
16 | on_exit 'kill `cat *.pid`' | |
17 | for i in `seq $n`; do | |
5a0e4aec | 18 | AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) |
1b1d2e6d BP |
19 | done |
20 | for i in `seq $n`; do | |
d97af428 | 21 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema connected]) |
1b1d2e6d BP |
22 | done |
23 | ||
24 | for txn | |
25 | do | |
15394e0f | 26 | AT_CHECK([ovsdb-client -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout]) |
1b1d2e6d BP |
27 | cat stdout >> output |
28 | done | |
29 | AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output]) | |
30 | for i in `seq $n`; do | |
5a0e4aec | 31 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) |
1b1d2e6d BP |
32 | done |
33 | ||
34 | AT_CHECK([ovsdb-tool check-cluster s*.db]) | |
35 | } | |
36 | OVS_END_SHELL_HELPERS | |
37 | ||
38 | # Test a 1-server cluster. | |
39 | AT_BANNER([OVSDB - clustered transactions (1 server)]) | |
40 | m4_define([OVSDB_CHECK_EXECUTION], | |
41 | [AT_SETUP([$1 - cluster of 1]) | |
42 | AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5]) | |
43 | ovsdb_check_cluster 1 "$2" '$4' m4_foreach([txn], [$3], ['txn' ]) | |
44 | AT_CLEANUP]) | |
45 | EXECUTION_EXAMPLES | |
46 | ||
47 | # Test a 3-server cluster. | |
48 | AT_BANNER([OVSDB - clustered transactions (3 servers)]) | |
49 | m4_define([OVSDB_CHECK_EXECUTION], | |
50 | [AT_SETUP([$1 - cluster of 3]) | |
51 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5]) | |
52 | ovsdb_check_cluster 3 "$2" '$4' m4_foreach([txn], [$3], ['txn' ]) | |
53 | AT_CLEANUP]) | |
54 | EXECUTION_EXAMPLES | |
55 | ||
56 | # Test a 5-server cluster. | |
57 | AT_BANNER([OVSDB - clustered transactions (5 servers)]) | |
58 | m4_define([OVSDB_CHECK_EXECUTION], | |
59 | [AT_SETUP([$1 - cluster of 5]) | |
60 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5]) | |
61 | ovsdb_check_cluster 5 "$2" '$4' m4_foreach([txn], [$3], ['txn' ]) | |
62 | AT_CLEANUP]) | |
63 | EXECUTION_EXAMPLES | |
eb692258 HZ |
64 | \f |
65 | ||
ca367fa5 HZ |
66 | AT_BANNER([OVSDB - disconnect from cluster]) |
67 | ||
89771c1e | 68 | OVS_START_SHELL_HELPERS |
923f01ca HZ |
69 | # ovsdb_test_cluster_disconnect N_SERVERS LEADER_OR_FOLLOWER [CHECK_FLAPPING] |
70 | # Test server disconnected from the cluster. | |
71 | # N_SERVERS: Number of servers in the cluster. | |
72 | # LEADER_OR_FOLLOWER: The role of the server that is disconnected from the | |
73 | # cluster: "leader" or "follower". | |
74 | # CHECK_FLAPPING: Whether to check if is_disconnected flapped. "yes", "no". | |
89771c1e | 75 | ovsdb_test_cluster_disconnect () { |
923f01ca HZ |
76 | n=$1 |
77 | leader_or_follower=$2 | |
78 | check_flapping=$3 | |
89771c1e HZ |
79 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` |
80 | ordinal_schema > schema | |
81 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr]) | |
82 | cid=`ovsdb-tool db-cid s1.db` | |
83 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
923f01ca | 84 | for i in `seq 2 $n`; do |
89771c1e HZ |
85 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft]) |
86 | done | |
87 | ||
88 | on_exit 'kill `cat *.pid`' | |
923f01ca | 89 | for i in `seq $n`; do |
89771c1e HZ |
90 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) |
91 | done | |
923f01ca | 92 | for i in `seq $n`; do |
89771c1e HZ |
93 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected]) |
94 | done | |
95 | ||
96 | AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest", | |
97 | {"op": "insert", | |
98 | "table": "simple", | |
99 | "row": {"i": 1}}]]'], [0], [ignore], [ignore]) | |
100 | ||
101 | # When a node is disconnected from the cluster, the IDL should disconnect | |
102 | # and retry even if it uses a single remote, because the remote IP can be | |
103 | # a VIP on a load-balance. So we use single remote to test here. | |
104 | if test $leader_or_follower == "leader"; then | |
105 | target=1 | |
923f01ca HZ |
106 | shutdown=`seq $(($n/2 + 1)) $n` |
107 | cleanup=`seq $(($n/2))` | |
89771c1e | 108 | else |
923f01ca | 109 | target=$n |
89771c1e | 110 | |
923f01ca HZ |
111 | # shutdown followers before the leader (s1) so that there is no chance for |
112 | # s$n to become leader during the process. | |
113 | shutdown="`seq 2 $(($n/2 + 1))` 1" | |
114 | cleanup=`seq $(($n/2 + 2)) $n` | |
89771c1e | 115 | fi |
923f01ca HZ |
116 | echo shutdown=$shutdown |
117 | echo cleanup=$cleanup | |
89771c1e HZ |
118 | |
119 | # Connect to $target. Use "wait" to trigger a non-op transaction so | |
120 | # that test-ovsdb will not quit. | |
121 | ||
122 | test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -v -t10 idl unix:s$target.ovsdb '[["idltest", | |
123 | {"op": "wait", | |
124 | "table": "simple", | |
125 | "where": [["i", "==", 1]], | |
126 | "columns": ["i"], | |
127 | "until": "==", | |
128 | "rows": [{"i": 1}]}]]' > test-ovsdb.log 2>&1 & | |
129 | echo $! > test-ovsdb.pid | |
ca367fa5 | 130 | |
89771c1e | 131 | OVS_WAIT_UNTIL([grep "000: i=1" test-ovsdb.log]) |
ca367fa5 | 132 | |
923f01ca HZ |
133 | # Start collecting raft_is_connected logs for $target before shutting down |
134 | # any servers. | |
135 | tail -f s$target.log > raft_is_connected.log & | |
136 | echo $! > tail.pid | |
137 | ||
89771c1e HZ |
138 | # Shutdown the other servers so that $target is disconnected from the cluster. |
139 | for i in $shutdown; do | |
140 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
141 | done | |
142 | ||
143 | # The test-ovsdb should detect the disconnect and retry. | |
144 | OVS_WAIT_UNTIL([grep disconnect test-ovsdb.log]) | |
145 | ||
923f01ca HZ |
146 | # The $target debug log should show raft_is_connected: false. |
147 | OVS_WAIT_UNTIL([grep "raft_is_connected: false" raft_is_connected.log]) | |
148 | ||
149 | # Save the current count of "raft_is_connected: true" | |
150 | count_old=`grep "raft_is_connected: true" raft_is_connected.log | wc -l` | |
151 | echo count_old $count_old | |
152 | ||
153 | if test X$check_flapping == X"yes"; then | |
154 | sleep 10 | |
155 | fi | |
156 | # Make sure raft_is_connected didn't flap from false to true. | |
157 | count_new=`grep "raft_is_connected: true" raft_is_connected.log | wc -l` | |
158 | echo count_new $count_new | |
159 | AT_CHECK([test $count_new == $count_old]) | |
160 | ||
161 | for i in $cleanup; do | |
162 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
163 | done | |
89771c1e HZ |
164 | } |
165 | OVS_END_SHELL_HELPERS | |
166 | ||
167 | AT_SETUP([OVSDB cluster - follower disconnect from cluster, single remote]) | |
168 | AT_KEYWORDS([ovsdb server negative unix cluster disconnect]) | |
923f01ca | 169 | ovsdb_test_cluster_disconnect 3 follower |
89771c1e HZ |
170 | AT_CLEANUP |
171 | ||
172 | AT_SETUP([OVSDB cluster - leader disconnect from cluster, single remote]) | |
173 | AT_KEYWORDS([ovsdb server negative unix cluster disconnect]) | |
923f01ca HZ |
174 | ovsdb_test_cluster_disconnect 3 leader |
175 | AT_CLEANUP | |
176 | ||
177 | AT_SETUP([OVSDB cluster - leader disconnect from cluster, check flapping]) | |
178 | AT_KEYWORDS([ovsdb server negative unix cluster disconnect]) | |
179 | ovsdb_test_cluster_disconnect 5 leader yes | |
ca367fa5 | 180 | AT_CLEANUP |
89771c1e | 181 | |
ca367fa5 HZ |
182 | \f |
183 | ||
8e354614 HZ |
184 | AT_BANNER([OVSDB cluster election timer change]) |
185 | ||
186 | AT_SETUP([OVSDB cluster - election timer change]) | |
187 | AT_KEYWORDS([ovsdb server positive unix cluster timer]) | |
188 | ||
189 | n=3 | |
190 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
191 | ordinal_schema > schema | |
192 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr]) | |
193 | cid=`ovsdb-tool db-cid s1.db` | |
194 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
195 | for i in `seq 2 $n`; do | |
196 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft]) | |
197 | done | |
198 | ||
199 | on_exit 'kill `cat *.pid`' | |
200 | for i in `seq $n`; do | |
201 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
202 | done | |
203 | for i in `seq $n`; do | |
204 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected]) | |
205 | done | |
206 | ||
207 | # Change not allowed through follower. | |
208 | AT_CHECK([ovs-appctl -t "`pwd`"/s2 cluster/change-election-timer $schema_name 2000], [2], [], [ignore]) | |
209 | ||
210 | # Timer cannot be changed to bigger than 2x the original value. | |
211 | AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [2], [], [ignore]) | |
212 | ||
213 | AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 2000], [0], [dnl | |
214 | change of election timer initiated. | |
215 | ], []) | |
216 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 2000"]) | |
217 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 2000"]) | |
218 | ||
219 | AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [0], [dnl | |
220 | change of election timer initiated. | |
221 | ], []) | |
222 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"]) | |
223 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"]) | |
224 | ||
225 | # Latest timer should be used after restart | |
226 | for i in `seq $n`; do | |
227 | printf "\ns$i: stopping\n" | |
228 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
229 | done | |
230 | for i in `seq $n`; do | |
231 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
232 | done | |
233 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"]) | |
234 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"]) | |
235 | ||
236 | AT_CLEANUP | |
237 | ||
238 | \f | |
239 | ||
eb692258 HZ |
240 | OVS_START_SHELL_HELPERS |
241 | # ovsdb_cluster_failure_test SCHEMA_FUNC OUTPUT TRANSACTION... | |
242 | ovsdb_cluster_failure_test () { | |
243 | # Initial state: s1 is leader, s2 and s3 are followers | |
244 | remote_1=$1 | |
245 | remote_2=$2 | |
246 | crash_node=$3 | |
247 | crash_command=$4 | |
248 | if test "$crash_node" == "1"; then | |
249 | new_leader=$5 | |
250 | fi | |
251 | ||
817db730 | 252 | cp $top_srcdir/vswitchd/vswitch.ovsschema schema |
eb692258 HZ |
253 | schema=`ovsdb-tool schema-name schema` |
254 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl | |
817db730 | 255 | ovsdb|WARN|schema: changed 30 columns in 'Open_vSwitch' database from ephemeral to persistent, including 'status' column in 'Manager' table, because clusters do not support ephemeral columns |
eb692258 HZ |
256 | ]) |
257 | ||
258 | n=3 | |
259 | join_cluster() { | |
260 | local i=$1 | |
261 | others= | |
262 | for j in `seq 1 $n`; do | |
263 | if test $i != $j; then | |
264 | others="$others unix:s$j.raft" | |
265 | fi | |
266 | done | |
267 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others]) | |
268 | } | |
269 | start_server() { | |
270 | local i=$1 | |
271 | printf "\ns$i: starting\n" | |
272 | AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
273 | } | |
274 | connect_server() { | |
275 | local i=$1 | |
276 | printf "\ns$i: waiting to connect to storage\n" | |
277 | AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected]) | |
278 | } | |
279 | cid=`ovsdb-tool db-cid s1.db` | |
280 | for i in `seq 2 $n`; do join_cluster $i; done | |
281 | ||
282 | on_exit 'kill `cat *.pid`' | |
283 | for i in `seq $n`; do start_server $i; done | |
284 | for i in `seq $n`; do connect_server $i; done | |
285 | ||
817db730 | 286 | db=unix:s$remote_1.ovsdb,unix:s$remote_2.ovsdb |
eb692258 HZ |
287 | |
288 | # To ensure $new_leader node the new leader, we delay election timer for | |
289 | # the other follower. | |
290 | if test -n "$new_leader"; then | |
291 | if test "$new_leader" == "2"; then | |
292 | delay_election_node=3 | |
293 | else | |
294 | delay_election_node=2 | |
295 | fi | |
296 | AT_CHECK([ovs-appctl -t "`pwd`"/s$delay_election_node cluster/failure-test delay-election], [0], [ignore]) | |
297 | fi | |
298 | AT_CHECK([ovs-appctl -t "`pwd`"/s$crash_node cluster/failure-test $crash_command], [0], [ignore]) | |
15394e0f | 299 | AT_CHECK([ovs-vsctl -v --db="$db" --no-leader-only --no-shuffle-remotes --no-wait create QoS type=x], [0], [ignore], [ignore]) |
eb692258 HZ |
300 | |
301 | # Make sure that the node really crashed. | |
302 | AT_CHECK([ls s$crash_node.ovsdb], [2], [ignore], [ignore]) | |
303 | # XXX: Client will fail if remotes contains unix socket that doesn't exist (killed). | |
817db730 BP |
304 | if test "$remote_1" = "$crash_node"; then |
305 | db=unix:s$remote_2.ovsdb | |
eb692258 | 306 | fi |
817db730 | 307 | AT_CHECK([ovs-vsctl --db="$db" --no-leader-only --no-wait --columns=type --bare list QoS], [0], [x |
eb692258 HZ |
308 | ]) |
309 | } | |
310 | OVS_END_SHELL_HELPERS | |
311 | AT_BANNER([OVSDB - cluster failure with pending transaction]) | |
312 | ||
313 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-2 becomes leader]) | |
314 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
315 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 2 |
316 | AT_CLEANUP | |
317 | ||
318 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-3 becomes leader]) | |
319 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
320 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 3 | |
321 | AT_CLEANUP | |
322 | ||
323 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-2 becomes leader]) | |
324 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
325 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 2 |
326 | AT_CLEANUP | |
327 | ||
328 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-3 becomes leader]) | |
329 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
330 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 3 |
331 | AT_CLEANUP | |
332 | ||
333 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-2 becomes leader]) | |
334 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
335 | ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 2 |
336 | AT_CLEANUP | |
337 | ||
338 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-3 becomes leader]) | |
339 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
340 | ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 3 | |
341 | AT_CLEANUP | |
342 | ||
343 | AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-2 becomes leader]) | |
344 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
345 | ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 2 |
346 | AT_CLEANUP | |
347 | ||
348 | AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-3 becomes leader]) | |
349 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
350 | ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 3 | |
351 | AT_CLEANUP | |
352 | ||
353 | AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-2 becomes leader]) | |
354 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
355 | # XXX: Detect and skip repeated transaction before enabling this test | |
356 | AT_CHECK([exit 77]) | |
357 | ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 2 | |
358 | AT_CLEANUP | |
359 | ||
360 | AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-3 becomes leader]) | |
361 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
362 | # XXX: Detect and skip repeated transaction before enabling this test | |
363 | AT_CHECK([exit 77]) | |
364 | ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 3 | |
365 | AT_CLEANUP | |
366 | ||
367 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to follower-3]) | |
368 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
369 | ovsdb_cluster_failure_test 2 3 2 crash-before-sending-execute-command-request | |
370 | AT_CLEANUP | |
371 | ||
372 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to leader]) | |
373 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
374 | ovsdb_cluster_failure_test 2 1 2 crash-before-sending-execute-command-request | |
375 | AT_CLEANUP | |
376 | ||
377 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to follower-3]) | |
378 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
379 | # XXX: Detect and skip repeated transaction before enabling this test | |
380 | AT_CHECK([exit 77]) | |
381 | ovsdb_cluster_failure_test 2 3 2 crash-after-sending-execute-command-request | |
382 | AT_CLEANUP | |
383 | ||
384 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to leader]) | |
385 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
386 | # XXX: Detect and skip repeated transaction before enabling this test | |
387 | AT_CHECK([exit 77]) | |
388 | ovsdb_cluster_failure_test 2 1 2 crash-after-sending-execute-command-request | |
389 | AT_CLEANUP | |
390 | ||
391 | AT_SETUP([OVSDB cluster - txn on leader, follower-2 crash after receiving appendReq for the update]) | |
392 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
393 | ovsdb_cluster_failure_test 1 1 2 crash-after-receiving-append-request-update | |
394 | AT_CLEANUP | |
395 | ||
396 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-3 crash after receiving appendReq for the update]) | |
397 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
398 | ovsdb_cluster_failure_test 2 2 3 crash-after-receiving-append-request-update | |
399 | AT_CLEANUP | |
400 | ||
1b1d2e6d BP |
401 | \f |
402 | AT_BANNER([OVSDB - cluster tests]) | |
403 | ||
404 | # Torture test. | |
405 | OVS_START_SHELL_HELPERS | |
406 | ovsdb_torture_test () { | |
5a0e4aec BP |
407 | local n=$1 # Number of cluster members |
408 | local victim=$2 # Cluster member to kill or remove | |
1b1d2e6d | 409 | local variant=$3 # 'kill' and restart or 'remove' and add |
817db730 | 410 | cp $top_srcdir/vswitchd/vswitch.ovsschema schema |
1b1d2e6d BP |
411 | schema=`ovsdb-tool schema-name schema` |
412 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl | |
817db730 | 413 | ovsdb|WARN|schema: changed 30 columns in 'Open_vSwitch' database from ephemeral to persistent, including 'status' column in 'Manager' table, because clusters do not support ephemeral columns |
1b1d2e6d BP |
414 | ]) |
415 | ||
416 | join_cluster() { | |
417 | local i=$1 | |
5a0e4aec BP |
418 | others= |
419 | for j in `seq 1 $n`; do | |
420 | if test $i != $j; then | |
421 | others="$others unix:s$j.raft" | |
1b1d2e6d | 422 | fi |
5a0e4aec BP |
423 | done |
424 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others]) | |
1b1d2e6d BP |
425 | } |
426 | ||
427 | start_server() { | |
428 | local i=$1 | |
5a0e4aec BP |
429 | printf "\ns$i: starting\n" |
430 | AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
1b1d2e6d BP |
431 | } |
432 | stop_server() { | |
433 | local i=$1 | |
5a0e4aec | 434 | printf "\ns$i: stopping\n" |
1b1d2e6d BP |
435 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) |
436 | } | |
437 | connect_server() { | |
438 | local i=$1 | |
5a0e4aec | 439 | printf "\ns$i: waiting to connect to storage\n" |
d97af428 | 440 | AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected]) |
1b1d2e6d BP |
441 | } |
442 | remove_server() { | |
443 | local i=$1 | |
5a0e4aec | 444 | printf "\ns$i: removing from cluster\n" |
15394e0f | 445 | AT_CHECK([ovs-appctl -t "`pwd`"/s$i cluster/leave Open_vSwitch]) |
5a0e4aec | 446 | printf "\ns$i: waiting for removal to complete\n" |
d97af428 | 447 | AT_CHECK([ovsdb_client_wait --log-file=remove$i.log unix:s$i.ovsdb $schema removed]) |
c7b5c534 | 448 | stop_server $i |
1b1d2e6d BP |
449 | } |
450 | add_server() { | |
451 | local i=$1 | |
5a0e4aec | 452 | rm s$i.db |
1b1d2e6d | 453 | join_cluster $i |
5a0e4aec BP |
454 | start_server $i |
455 | connect_server $i | |
1b1d2e6d BP |
456 | } |
457 | ||
458 | cid=`ovsdb-tool db-cid s1.db` | |
459 | for i in `seq 2 $n`; do join_cluster $i; done | |
460 | ||
461 | on_exit 'kill `cat *.pid`' | |
462 | for i in `seq $n`; do start_server $i; done | |
463 | for i in `seq $n`; do connect_server $i; done | |
464 | ||
817db730 | 465 | db=unix:s1.ovsdb |
1b1d2e6d | 466 | for i in `seq 2 $n`; do |
817db730 | 467 | db=$db,unix:s$i.ovsdb |
1b1d2e6d | 468 | done |
1b1d2e6d | 469 | |
0f954f32 | 470 | n1=10 n2=5 n3=50 |
817db730 | 471 | echo "starting $n1*$n2 ovs-vsctl processes..." |
1b1d2e6d | 472 | for i in $(seq 0 $(expr $n1 - 1) ); do |
5a0e4aec | 473 | (for j in $(seq $n2); do |
1b1d2e6d | 474 | : > $i-$j.running |
817db730 | 475 | txn="add Open_vSwitch . external_ids $i-$j=$i-$j" |
0f954f32 | 476 | for k in $(seq $n3); do |
817db730 | 477 | txn="$txn -- add Open_vSwitch . external_ids $i-$j-$k=$i-$j-$k" |
0f954f32 | 478 | done |
817db730 | 479 | run_as "ovs-vsctl($i-$j)" ovs-vsctl "-vPATTERN:console:ovs-vsctl($i-$j)|%D{%H:%M:%S}|%05N|%c|%p|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --db="$db" --no-leader-only --no-wait $txn |
5a0e4aec BP |
480 | status=$? |
481 | if test $status != 0; then | |
482 | echo "$i-$j exited with status $status" > $i-$j:$status | |
1b1d2e6d BP |
483 | fi |
484 | rm $i-$j.running | |
5a0e4aec BP |
485 | done |
486 | : > $i.done)& | |
1b1d2e6d BP |
487 | done |
488 | echo "...done" | |
1b1d2e6d | 489 | |
817db730 | 490 | echo "waiting for ovs-vsctl processes to exit..." |
6c8dd8ca | 491 | # Use file instead of var because code inside "while" runs in a subshell. |
7ee9c6e0 | 492 | echo 0 > phase |
0f03ae37 | 493 | i=0 |
0f954f32 | 494 | (while :; do echo; sleep 0.1; done) | while read REPLY; do |
1b1d2e6d | 495 | printf "t=%2d s:" $i |
5a0e4aec | 496 | done=0 |
1b1d2e6d | 497 | for j in $(seq 0 $(expr $n1 - 1)); do |
5a0e4aec BP |
498 | if test -f $j.done; then |
499 | printf " $j" | |
500 | done=$(expr $done + 1) | |
1b1d2e6d | 501 | fi |
5a0e4aec BP |
502 | done |
503 | printf '\n' | |
504 | if test $done = $n1; then | |
505 | break | |
1b1d2e6d BP |
506 | fi |
507 | ||
7ee9c6e0 | 508 | case $(cat phase) in # ( |
5a0e4aec | 509 | 0) |
0f954f32 | 510 | if test $done -ge $(expr $n1 / 10); then |
5a0e4aec BP |
511 | if test $variant = kill; then |
512 | stop_server $victim | |
513 | else | |
514 | remove_server $victim | |
515 | fi | |
7ee9c6e0 | 516 | echo 1 > phase |
5a0e4aec | 517 | next=$(expr $i + 2) |
1b1d2e6d | 518 | fi |
5a0e4aec | 519 | ;; # ( |
1b1d2e6d | 520 | 1) |
5a0e4aec BP |
521 | if test $i -ge $next; then |
522 | if test $variant = kill; then | |
523 | start_server $victim | |
524 | connect_server $victim | |
525 | else | |
526 | add_server $victim | |
527 | fi | |
7ee9c6e0 | 528 | echo 2 > phase |
5a0e4aec BP |
529 | fi |
530 | ;; | |
531 | esac | |
532 | ||
533 | i=$(expr $i + 1) | |
1b1d2e6d BP |
534 | done |
535 | echo "...done" | |
7ee9c6e0 | 536 | AT_CHECK([if test $(cat phase) != 2; then exit 77; fi]) |
1b1d2e6d | 537 | |
dff60a1e BP |
538 | for i in $(seq 0 $(expr $n1 - 1) ); do |
539 | for j in `seq $n2`; do | |
540 | echo "$i-$j=$i-$j" | |
0f954f32 HZ |
541 | for k in `seq $n3`; do |
542 | echo "$i-$j-$k=$i-$j-$k" | |
543 | done | |
dff60a1e BP |
544 | done |
545 | done | sort > expout | |
15394e0f | 546 | AT_CHECK([ovs-vsctl --db="$db" --no-wait --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get Open_vSwitch . external-ids | tr ',' '\n' | sed 's/[[{}"" ]]//g' | sort], [0], [expout]) |
1b1d2e6d BP |
547 | |
548 | for i in `seq $n`; do | |
7ee9c6e0 | 549 | if test $i != $victim || test $(cat phase) != 1; then |
5a0e4aec BP |
550 | stop_server $i |
551 | fi | |
1b1d2e6d BP |
552 | done |
553 | ||
554 | # We ignore stdout because non-fatal warnings get printed there. | |
555 | AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore]) | |
556 | } | |
557 | OVS_END_SHELL_HELPERS | |
558 | ||
559 | AT_SETUP([OVSDB 3-server torture test - kill/restart leader]) | |
560 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
561 | ovsdb_torture_test 3 1 kill | |
562 | AT_CLEANUP | |
563 | AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1]) | |
564 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
565 | ovsdb_torture_test 3 2 kill | |
566 | AT_CLEANUP | |
567 | AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2]) | |
568 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
569 | ovsdb_torture_test 3 3 kill | |
570 | AT_CLEANUP | |
571 | AT_SETUP([OVSDB 5-server torture test - kill/restart leader]) | |
572 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
573 | ovsdb_torture_test 5 1 kill | |
574 | AT_CLEANUP | |
575 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1]) | |
576 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
577 | ovsdb_torture_test 5 2 kill | |
578 | AT_CLEANUP | |
579 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2]) | |
580 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
581 | ovsdb_torture_test 5 3 kill | |
582 | AT_CLEANUP | |
583 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3]) | |
584 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
585 | ovsdb_torture_test 5 4 kill | |
586 | AT_CLEANUP | |
587 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4]) | |
588 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
589 | ovsdb_torture_test 5 5 kill | |
590 | AT_CLEANUP | |
591 | ||
592 | AT_SETUP([OVSDB 3-server torture test - remove/re-add leader]) | |
593 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
594 | ovsdb_torture_test 3 1 remove | |
595 | AT_CLEANUP | |
596 | AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1]) | |
597 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
598 | ovsdb_torture_test 3 2 remove | |
599 | AT_CLEANUP | |
600 | AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2]) | |
601 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
602 | ovsdb_torture_test 3 3 remove | |
603 | AT_CLEANUP | |
604 | AT_SETUP([OVSDB 5-server torture test - remove/re-add leader]) | |
605 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
606 | ovsdb_torture_test 5 1 remove | |
607 | AT_CLEANUP | |
608 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1]) | |
609 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
610 | ovsdb_torture_test 5 2 remove | |
611 | AT_CLEANUP | |
612 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2]) | |
613 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
614 | ovsdb_torture_test 5 3 remove | |
615 | AT_CLEANUP | |
616 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3]) | |
617 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
618 | ovsdb_torture_test 5 4 remove | |
619 | AT_CLEANUP | |
620 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4]) | |
621 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
622 | ovsdb_torture_test 5 5 remove | |
623 | AT_CLEANUP |