]>
Commit | Line | Data |
---|---|---|
1b1d2e6d BP |
1 | OVS_START_SHELL_HELPERS |
2 | # ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT TRANSACTION... | |
3 | ovsdb_check_cluster () { | |
4 | local n=$1 schema_func=$2 output=$3 | |
5 | shift; shift; shift | |
6 | ||
7 | $schema_func > schema | |
8 | schema=`ovsdb-tool schema-name schema` | |
9 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr]) | |
10 | AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1]) | |
11 | cid=`ovsdb-tool db-cid s1.db` | |
12 | for i in `seq 2 $n`; do | |
13 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft]) | |
14 | done | |
15 | ||
16 | on_exit 'kill `cat *.pid`' | |
17 | for i in `seq $n`; do | |
5a0e4aec | 18 | AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) |
1b1d2e6d BP |
19 | done |
20 | for i in `seq $n`; do | |
d97af428 | 21 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema connected]) |
1b1d2e6d BP |
22 | done |
23 | ||
24 | for txn | |
25 | do | |
26 | AT_CHECK([ovsdb-client --timeout=30 -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout]) | |
27 | cat stdout >> output | |
28 | done | |
29 | AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output]) | |
30 | for i in `seq $n`; do | |
5a0e4aec | 31 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) |
1b1d2e6d BP |
32 | done |
33 | ||
34 | AT_CHECK([ovsdb-tool check-cluster s*.db]) | |
35 | } | |
36 | OVS_END_SHELL_HELPERS | |
37 | ||
38 | # Test a 1-server cluster. | |
39 | AT_BANNER([OVSDB - clustered transactions (1 server)]) | |
40 | m4_define([OVSDB_CHECK_EXECUTION], | |
41 | [AT_SETUP([$1 - cluster of 1]) | |
42 | AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5]) | |
43 | ovsdb_check_cluster 1 "$2" '$4' m4_foreach([txn], [$3], ['txn' ]) | |
44 | AT_CLEANUP]) | |
45 | EXECUTION_EXAMPLES | |
46 | ||
47 | # Test a 3-server cluster. | |
48 | AT_BANNER([OVSDB - clustered transactions (3 servers)]) | |
49 | m4_define([OVSDB_CHECK_EXECUTION], | |
50 | [AT_SETUP([$1 - cluster of 3]) | |
51 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5]) | |
52 | ovsdb_check_cluster 3 "$2" '$4' m4_foreach([txn], [$3], ['txn' ]) | |
53 | AT_CLEANUP]) | |
54 | EXECUTION_EXAMPLES | |
55 | ||
56 | # Test a 5-server cluster. | |
57 | AT_BANNER([OVSDB - clustered transactions (5 servers)]) | |
58 | m4_define([OVSDB_CHECK_EXECUTION], | |
59 | [AT_SETUP([$1 - cluster of 5]) | |
60 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5]) | |
61 | ovsdb_check_cluster 5 "$2" '$4' m4_foreach([txn], [$3], ['txn' ]) | |
62 | AT_CLEANUP]) | |
63 | EXECUTION_EXAMPLES | |
eb692258 HZ |
64 | \f |
65 | ||
ca367fa5 HZ |
66 | AT_BANNER([OVSDB - disconnect from cluster]) |
67 | ||
89771c1e | 68 | OVS_START_SHELL_HELPERS |
923f01ca HZ |
69 | # ovsdb_test_cluster_disconnect N_SERVERS LEADER_OR_FOLLOWER [CHECK_FLAPPING] |
70 | # Test server disconnected from the cluster. | |
71 | # N_SERVERS: Number of servers in the cluster. | |
72 | # LEADER_OR_FOLLOWER: The role of the server that is disconnected from the | |
73 | # cluster: "leader" or "follower". | |
74 | # CHECK_FLAPPING: Whether to check if is_disconnected flapped. "yes", "no". | |
89771c1e | 75 | ovsdb_test_cluster_disconnect () { |
923f01ca HZ |
76 | n=$1 |
77 | leader_or_follower=$2 | |
78 | check_flapping=$3 | |
89771c1e HZ |
79 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` |
80 | ordinal_schema > schema | |
81 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr]) | |
82 | cid=`ovsdb-tool db-cid s1.db` | |
83 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
923f01ca | 84 | for i in `seq 2 $n`; do |
89771c1e HZ |
85 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft]) |
86 | done | |
87 | ||
88 | on_exit 'kill `cat *.pid`' | |
923f01ca | 89 | for i in `seq $n`; do |
89771c1e HZ |
90 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) |
91 | done | |
923f01ca | 92 | for i in `seq $n`; do |
89771c1e HZ |
93 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected]) |
94 | done | |
95 | ||
96 | AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest", | |
97 | {"op": "insert", | |
98 | "table": "simple", | |
99 | "row": {"i": 1}}]]'], [0], [ignore], [ignore]) | |
100 | ||
101 | # When a node is disconnected from the cluster, the IDL should disconnect | |
102 | # and retry even if it uses a single remote, because the remote IP can be | |
103 | # a VIP on a load-balance. So we use single remote to test here. | |
104 | if test $leader_or_follower == "leader"; then | |
105 | target=1 | |
923f01ca HZ |
106 | shutdown=`seq $(($n/2 + 1)) $n` |
107 | cleanup=`seq $(($n/2))` | |
89771c1e | 108 | else |
923f01ca | 109 | target=$n |
89771c1e | 110 | |
923f01ca HZ |
111 | # shutdown followers before the leader (s1) so that there is no chance for |
112 | # s$n to become leader during the process. | |
113 | shutdown="`seq 2 $(($n/2 + 1))` 1" | |
114 | cleanup=`seq $(($n/2 + 2)) $n` | |
89771c1e | 115 | fi |
923f01ca HZ |
116 | echo shutdown=$shutdown |
117 | echo cleanup=$cleanup | |
89771c1e HZ |
118 | |
119 | # Connect to $target. Use "wait" to trigger a non-op transaction so | |
120 | # that test-ovsdb will not quit. | |
121 | ||
122 | test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -v -t10 idl unix:s$target.ovsdb '[["idltest", | |
123 | {"op": "wait", | |
124 | "table": "simple", | |
125 | "where": [["i", "==", 1]], | |
126 | "columns": ["i"], | |
127 | "until": "==", | |
128 | "rows": [{"i": 1}]}]]' > test-ovsdb.log 2>&1 & | |
129 | echo $! > test-ovsdb.pid | |
ca367fa5 | 130 | |
89771c1e | 131 | OVS_WAIT_UNTIL([grep "000: i=1" test-ovsdb.log]) |
ca367fa5 | 132 | |
923f01ca HZ |
133 | # Start collecting raft_is_connected logs for $target before shutting down |
134 | # any servers. | |
135 | tail -f s$target.log > raft_is_connected.log & | |
136 | echo $! > tail.pid | |
137 | ||
89771c1e HZ |
138 | # Shutdown the other servers so that $target is disconnected from the cluster. |
139 | for i in $shutdown; do | |
140 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
141 | done | |
142 | ||
143 | # The test-ovsdb should detect the disconnect and retry. | |
144 | OVS_WAIT_UNTIL([grep disconnect test-ovsdb.log]) | |
145 | ||
923f01ca HZ |
146 | # The $target debug log should show raft_is_connected: false. |
147 | OVS_WAIT_UNTIL([grep "raft_is_connected: false" raft_is_connected.log]) | |
148 | ||
149 | # Save the current count of "raft_is_connected: true" | |
150 | count_old=`grep "raft_is_connected: true" raft_is_connected.log | wc -l` | |
151 | echo count_old $count_old | |
152 | ||
153 | if test X$check_flapping == X"yes"; then | |
154 | sleep 10 | |
155 | fi | |
156 | # Make sure raft_is_connected didn't flap from false to true. | |
157 | count_new=`grep "raft_is_connected: true" raft_is_connected.log | wc -l` | |
158 | echo count_new $count_new | |
159 | AT_CHECK([test $count_new == $count_old]) | |
160 | ||
161 | for i in $cleanup; do | |
162 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
163 | done | |
89771c1e HZ |
164 | } |
165 | OVS_END_SHELL_HELPERS | |
166 | ||
167 | AT_SETUP([OVSDB cluster - follower disconnect from cluster, single remote]) | |
168 | AT_KEYWORDS([ovsdb server negative unix cluster disconnect]) | |
923f01ca | 169 | ovsdb_test_cluster_disconnect 3 follower |
89771c1e HZ |
170 | AT_CLEANUP |
171 | ||
172 | AT_SETUP([OVSDB cluster - leader disconnect from cluster, single remote]) | |
173 | AT_KEYWORDS([ovsdb server negative unix cluster disconnect]) | |
923f01ca HZ |
174 | ovsdb_test_cluster_disconnect 3 leader |
175 | AT_CLEANUP | |
176 | ||
177 | AT_SETUP([OVSDB cluster - leader disconnect from cluster, check flapping]) | |
178 | AT_KEYWORDS([ovsdb server negative unix cluster disconnect]) | |
179 | ovsdb_test_cluster_disconnect 5 leader yes | |
ca367fa5 | 180 | AT_CLEANUP |
89771c1e | 181 | |
ca367fa5 HZ |
182 | \f |
183 | ||
8e354614 HZ |
184 | AT_BANNER([OVSDB cluster election timer change]) |
185 | ||
186 | AT_SETUP([OVSDB cluster - election timer change]) | |
187 | AT_KEYWORDS([ovsdb server positive unix cluster timer]) | |
188 | ||
189 | n=3 | |
190 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
191 | ordinal_schema > schema | |
192 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr]) | |
193 | cid=`ovsdb-tool db-cid s1.db` | |
194 | schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema` | |
195 | for i in `seq 2 $n`; do | |
196 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft]) | |
197 | done | |
198 | ||
199 | on_exit 'kill `cat *.pid`' | |
200 | for i in `seq $n`; do | |
201 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
202 | done | |
203 | for i in `seq $n`; do | |
204 | AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected]) | |
205 | done | |
206 | ||
207 | # Change not allowed through follower. | |
208 | AT_CHECK([ovs-appctl -t "`pwd`"/s2 cluster/change-election-timer $schema_name 2000], [2], [], [ignore]) | |
209 | ||
210 | # Timer cannot be changed to bigger than 2x the original value. | |
211 | AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [2], [], [ignore]) | |
212 | ||
213 | AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 2000], [0], [dnl | |
214 | change of election timer initiated. | |
215 | ], []) | |
216 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 2000"]) | |
217 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 2000"]) | |
218 | ||
219 | AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [0], [dnl | |
220 | change of election timer initiated. | |
221 | ], []) | |
222 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"]) | |
223 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"]) | |
224 | ||
225 | # Latest timer should be used after restart | |
226 | for i in `seq $n`; do | |
227 | printf "\ns$i: stopping\n" | |
228 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) | |
229 | done | |
230 | for i in `seq $n`; do | |
231 | AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
232 | done | |
233 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"]) | |
234 | OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"]) | |
235 | ||
236 | AT_CLEANUP | |
237 | ||
238 | \f | |
239 | ||
eb692258 HZ |
240 | OVS_START_SHELL_HELPERS |
241 | # ovsdb_cluster_failure_test SCHEMA_FUNC OUTPUT TRANSACTION... | |
242 | ovsdb_cluster_failure_test () { | |
243 | # Initial state: s1 is leader, s2 and s3 are followers | |
244 | remote_1=$1 | |
245 | remote_2=$2 | |
246 | crash_node=$3 | |
247 | crash_command=$4 | |
248 | if test "$crash_node" == "1"; then | |
249 | new_leader=$5 | |
250 | fi | |
251 | ||
252 | cp $top_srcdir/ovn/ovn-nb.ovsschema schema | |
253 | schema=`ovsdb-tool schema-name schema` | |
254 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl | |
255 | ovsdb|WARN|schema: changed 2 columns in 'OVN_Northbound' database from ephemeral to persistent, including 'status' column in 'Connection' table, because clusters do not support ephemeral columns | |
256 | ]) | |
257 | ||
258 | n=3 | |
259 | join_cluster() { | |
260 | local i=$1 | |
261 | others= | |
262 | for j in `seq 1 $n`; do | |
263 | if test $i != $j; then | |
264 | others="$others unix:s$j.raft" | |
265 | fi | |
266 | done | |
267 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others]) | |
268 | } | |
269 | start_server() { | |
270 | local i=$1 | |
271 | printf "\ns$i: starting\n" | |
272 | AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
273 | } | |
274 | connect_server() { | |
275 | local i=$1 | |
276 | printf "\ns$i: waiting to connect to storage\n" | |
277 | AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected]) | |
278 | } | |
279 | cid=`ovsdb-tool db-cid s1.db` | |
280 | for i in `seq 2 $n`; do join_cluster $i; done | |
281 | ||
282 | on_exit 'kill `cat *.pid`' | |
283 | for i in `seq $n`; do start_server $i; done | |
284 | for i in `seq $n`; do connect_server $i; done | |
285 | ||
286 | export OVN_NB_DB=unix:s$remote_1.ovsdb,unix:s$remote_2.ovsdb | |
287 | ||
288 | # To ensure $new_leader node the new leader, we delay election timer for | |
289 | # the other follower. | |
290 | if test -n "$new_leader"; then | |
291 | if test "$new_leader" == "2"; then | |
292 | delay_election_node=3 | |
293 | else | |
294 | delay_election_node=2 | |
295 | fi | |
296 | AT_CHECK([ovs-appctl -t "`pwd`"/s$delay_election_node cluster/failure-test delay-election], [0], [ignore]) | |
297 | fi | |
298 | AT_CHECK([ovs-appctl -t "`pwd`"/s$crash_node cluster/failure-test $crash_command], [0], [ignore]) | |
299 | AT_CHECK([ovn-nbctl -v --timeout=10 --no-leader-only --no-shuffle-remotes create logical_switch name=ls1], [0], [ignore], [ignore]) | |
300 | ||
301 | # Make sure that the node really crashed. | |
302 | AT_CHECK([ls s$crash_node.ovsdb], [2], [ignore], [ignore]) | |
303 | # XXX: Client will fail if remotes contains unix socket that doesn't exist (killed). | |
304 | if test "$remote_1" == "$crash_node"; then | |
305 | export OVN_NB_DB=unix:s$remote_2.ovsdb | |
306 | fi | |
307 | AT_CHECK([ovn-nbctl --no-leader-only ls-list | awk '{ print $2 }'], [0], [(ls1) | |
308 | ]) | |
309 | } | |
310 | OVS_END_SHELL_HELPERS | |
311 | AT_BANNER([OVSDB - cluster failure with pending transaction]) | |
312 | ||
313 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-2 becomes leader]) | |
314 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
315 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 2 |
316 | AT_CLEANUP | |
317 | ||
318 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-3 becomes leader]) | |
319 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
320 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 3 | |
321 | AT_CLEANUP | |
322 | ||
323 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-2 becomes leader]) | |
324 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
325 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 2 |
326 | AT_CLEANUP | |
327 | ||
328 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-3 becomes leader]) | |
329 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
330 | ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 3 |
331 | AT_CLEANUP | |
332 | ||
333 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-2 becomes leader]) | |
334 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
335 | ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 2 |
336 | AT_CLEANUP | |
337 | ||
338 | AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-3 becomes leader]) | |
339 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
340 | ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 3 | |
341 | AT_CLEANUP | |
342 | ||
343 | AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-2 becomes leader]) | |
344 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
eb692258 HZ |
345 | ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 2 |
346 | AT_CLEANUP | |
347 | ||
348 | AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-3 becomes leader]) | |
349 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
350 | ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 3 | |
351 | AT_CLEANUP | |
352 | ||
353 | AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-2 becomes leader]) | |
354 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
355 | # XXX: Detect and skip repeated transaction before enabling this test | |
356 | AT_CHECK([exit 77]) | |
357 | ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 2 | |
358 | AT_CLEANUP | |
359 | ||
360 | AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-3 becomes leader]) | |
361 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
362 | # XXX: Detect and skip repeated transaction before enabling this test | |
363 | AT_CHECK([exit 77]) | |
364 | ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 3 | |
365 | AT_CLEANUP | |
366 | ||
367 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to follower-3]) | |
368 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
369 | ovsdb_cluster_failure_test 2 3 2 crash-before-sending-execute-command-request | |
370 | AT_CLEANUP | |
371 | ||
372 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to leader]) | |
373 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
374 | ovsdb_cluster_failure_test 2 1 2 crash-before-sending-execute-command-request | |
375 | AT_CLEANUP | |
376 | ||
377 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to follower-3]) | |
378 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
379 | # XXX: Detect and skip repeated transaction before enabling this test | |
380 | AT_CHECK([exit 77]) | |
381 | ovsdb_cluster_failure_test 2 3 2 crash-after-sending-execute-command-request | |
382 | AT_CLEANUP | |
383 | ||
384 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to leader]) | |
385 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
386 | # XXX: Detect and skip repeated transaction before enabling this test | |
387 | AT_CHECK([exit 77]) | |
388 | ovsdb_cluster_failure_test 2 1 2 crash-after-sending-execute-command-request | |
389 | AT_CLEANUP | |
390 | ||
391 | AT_SETUP([OVSDB cluster - txn on leader, follower-2 crash after receiving appendReq for the update]) | |
392 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
393 | ovsdb_cluster_failure_test 1 1 2 crash-after-receiving-append-request-update | |
394 | AT_CLEANUP | |
395 | ||
396 | AT_SETUP([OVSDB cluster - txn on follower-2, follower-3 crash after receiving appendReq for the update]) | |
397 | AT_KEYWORDS([ovsdb server negative unix cluster pending-txn]) | |
398 | ovsdb_cluster_failure_test 2 2 3 crash-after-receiving-append-request-update | |
399 | AT_CLEANUP | |
400 | ||
1b1d2e6d BP |
401 | \f |
402 | AT_BANNER([OVSDB - cluster tests]) | |
403 | ||
404 | # Torture test. | |
405 | OVS_START_SHELL_HELPERS | |
406 | ovsdb_torture_test () { | |
5a0e4aec BP |
407 | local n=$1 # Number of cluster members |
408 | local victim=$2 # Cluster member to kill or remove | |
1b1d2e6d BP |
409 | local variant=$3 # 'kill' and restart or 'remove' and add |
410 | cp $top_srcdir/ovn/ovn-sb.ovsschema schema | |
411 | schema=`ovsdb-tool schema-name schema` | |
412 | AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl | |
413 | ovsdb|WARN|schema: changed 2 columns in 'OVN_Southbound' database from ephemeral to persistent, including 'status' column in 'Connection' table, because clusters do not support ephemeral columns | |
414 | ]) | |
415 | ||
416 | join_cluster() { | |
417 | local i=$1 | |
5a0e4aec BP |
418 | others= |
419 | for j in `seq 1 $n`; do | |
420 | if test $i != $j; then | |
421 | others="$others unix:s$j.raft" | |
1b1d2e6d | 422 | fi |
5a0e4aec BP |
423 | done |
424 | AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others]) | |
1b1d2e6d BP |
425 | } |
426 | ||
427 | start_server() { | |
428 | local i=$1 | |
5a0e4aec BP |
429 | printf "\ns$i: starting\n" |
430 | AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db]) | |
1b1d2e6d BP |
431 | } |
432 | stop_server() { | |
433 | local i=$1 | |
5a0e4aec | 434 | printf "\ns$i: stopping\n" |
1b1d2e6d BP |
435 | OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid]) |
436 | } | |
437 | connect_server() { | |
438 | local i=$1 | |
5a0e4aec | 439 | printf "\ns$i: waiting to connect to storage\n" |
d97af428 | 440 | AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected]) |
1b1d2e6d BP |
441 | } |
442 | remove_server() { | |
443 | local i=$1 | |
5a0e4aec BP |
444 | printf "\ns$i: removing from cluster\n" |
445 | AT_CHECK([ovs-appctl --timeout=30 -t "`pwd`"/s$i cluster/leave OVN_Southbound]) | |
446 | printf "\ns$i: waiting for removal to complete\n" | |
d97af428 | 447 | AT_CHECK([ovsdb_client_wait --log-file=remove$i.log unix:s$i.ovsdb $schema removed]) |
c7b5c534 | 448 | stop_server $i |
1b1d2e6d BP |
449 | } |
450 | add_server() { | |
451 | local i=$1 | |
5a0e4aec | 452 | rm s$i.db |
1b1d2e6d | 453 | join_cluster $i |
5a0e4aec BP |
454 | start_server $i |
455 | connect_server $i | |
1b1d2e6d BP |
456 | } |
457 | ||
458 | cid=`ovsdb-tool db-cid s1.db` | |
459 | for i in `seq 2 $n`; do join_cluster $i; done | |
460 | ||
461 | on_exit 'kill `cat *.pid`' | |
462 | for i in `seq $n`; do start_server $i; done | |
463 | for i in `seq $n`; do connect_server $i; done | |
464 | ||
465 | OVN_SB_DB=unix:s1.ovsdb | |
466 | for i in `seq 2 $n`; do | |
467 | OVN_SB_DB=$OVN_SB_DB,unix:s$i.ovsdb | |
468 | done | |
469 | export OVN_SB_DB | |
470 | ||
0f954f32 | 471 | n1=10 n2=5 n3=50 |
1b1d2e6d BP |
472 | echo "starting $n1*$n2 ovn-sbctl processes..." |
473 | for i in $(seq 0 $(expr $n1 - 1) ); do | |
5a0e4aec | 474 | (for j in $(seq $n2); do |
1b1d2e6d | 475 | : > $i-$j.running |
0f954f32 HZ |
476 | txn="add SB_Global . external_ids $i-$j=$i-$j" |
477 | for k in $(seq $n3); do | |
478 | txn="$txn -- add SB_Global . external_ids $i-$j-$k=$i-$j-$k" | |
479 | done | |
480 | run_as "ovn-sbctl($i-$j)" ovn-sbctl "-vPATTERN:console:ovn-sbctl($i-$j)|%D{%H:%M:%S}|%05N|%c|%p|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --no-leader-only $txn | |
5a0e4aec BP |
481 | status=$? |
482 | if test $status != 0; then | |
483 | echo "$i-$j exited with status $status" > $i-$j:$status | |
1b1d2e6d BP |
484 | fi |
485 | rm $i-$j.running | |
5a0e4aec BP |
486 | done |
487 | : > $i.done)& | |
1b1d2e6d BP |
488 | done |
489 | echo "...done" | |
1b1d2e6d BP |
490 | |
491 | echo "waiting for ovn-sbctl processes to exit..." | |
6c8dd8ca | 492 | # Use file instead of var because code inside "while" runs in a subshell. |
7ee9c6e0 | 493 | echo 0 > phase |
0f03ae37 | 494 | i=0 |
0f954f32 | 495 | (while :; do echo; sleep 0.1; done) | while read REPLY; do |
1b1d2e6d | 496 | printf "t=%2d s:" $i |
5a0e4aec | 497 | done=0 |
1b1d2e6d | 498 | for j in $(seq 0 $(expr $n1 - 1)); do |
5a0e4aec BP |
499 | if test -f $j.done; then |
500 | printf " $j" | |
501 | done=$(expr $done + 1) | |
1b1d2e6d | 502 | fi |
5a0e4aec BP |
503 | done |
504 | printf '\n' | |
505 | if test $done = $n1; then | |
506 | break | |
1b1d2e6d BP |
507 | fi |
508 | ||
7ee9c6e0 | 509 | case $(cat phase) in # ( |
5a0e4aec | 510 | 0) |
0f954f32 | 511 | if test $done -ge $(expr $n1 / 10); then |
5a0e4aec BP |
512 | if test $variant = kill; then |
513 | stop_server $victim | |
514 | else | |
515 | remove_server $victim | |
516 | fi | |
7ee9c6e0 | 517 | echo 1 > phase |
5a0e4aec | 518 | next=$(expr $i + 2) |
1b1d2e6d | 519 | fi |
5a0e4aec | 520 | ;; # ( |
1b1d2e6d | 521 | 1) |
5a0e4aec BP |
522 | if test $i -ge $next; then |
523 | if test $variant = kill; then | |
524 | start_server $victim | |
525 | connect_server $victim | |
526 | else | |
527 | add_server $victim | |
528 | fi | |
7ee9c6e0 | 529 | echo 2 > phase |
5a0e4aec BP |
530 | fi |
531 | ;; | |
532 | esac | |
533 | ||
534 | i=$(expr $i + 1) | |
1b1d2e6d BP |
535 | done |
536 | echo "...done" | |
7ee9c6e0 | 537 | AT_CHECK([if test $(cat phase) != 2; then exit 77; fi]) |
1b1d2e6d | 538 | |
dff60a1e BP |
539 | for i in $(seq 0 $(expr $n1 - 1) ); do |
540 | for j in `seq $n2`; do | |
541 | echo "$i-$j=$i-$j" | |
0f954f32 HZ |
542 | for k in `seq $n3`; do |
543 | echo "$i-$j-$k=$i-$j-$k" | |
544 | done | |
dff60a1e BP |
545 | done |
546 | done | sort > expout | |
b6c9325b | 547 | AT_CHECK([ovn-sbctl --timeout=30 --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get SB_Global . external-ids | tr ',' '\n' | sed 's/[[{}"" ]]//g' | sort], [0], [expout]) |
1b1d2e6d BP |
548 | |
549 | for i in `seq $n`; do | |
7ee9c6e0 | 550 | if test $i != $victim || test $(cat phase) != 1; then |
5a0e4aec BP |
551 | stop_server $i |
552 | fi | |
1b1d2e6d BP |
553 | done |
554 | ||
555 | # We ignore stdout because non-fatal warnings get printed there. | |
556 | AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore]) | |
557 | } | |
558 | OVS_END_SHELL_HELPERS | |
559 | ||
560 | AT_SETUP([OVSDB 3-server torture test - kill/restart leader]) | |
561 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
562 | ovsdb_torture_test 3 1 kill | |
563 | AT_CLEANUP | |
564 | AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1]) | |
565 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
566 | ovsdb_torture_test 3 2 kill | |
567 | AT_CLEANUP | |
568 | AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2]) | |
569 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
570 | ovsdb_torture_test 3 3 kill | |
571 | AT_CLEANUP | |
572 | AT_SETUP([OVSDB 5-server torture test - kill/restart leader]) | |
573 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
574 | ovsdb_torture_test 5 1 kill | |
575 | AT_CLEANUP | |
576 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1]) | |
577 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
578 | ovsdb_torture_test 5 2 kill | |
579 | AT_CLEANUP | |
580 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2]) | |
581 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
582 | ovsdb_torture_test 5 3 kill | |
583 | AT_CLEANUP | |
584 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3]) | |
585 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
586 | ovsdb_torture_test 5 4 kill | |
587 | AT_CLEANUP | |
588 | AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4]) | |
589 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
590 | ovsdb_torture_test 5 5 kill | |
591 | AT_CLEANUP | |
592 | ||
593 | AT_SETUP([OVSDB 3-server torture test - remove/re-add leader]) | |
594 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
595 | ovsdb_torture_test 3 1 remove | |
596 | AT_CLEANUP | |
597 | AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1]) | |
598 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
599 | ovsdb_torture_test 3 2 remove | |
600 | AT_CLEANUP | |
601 | AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2]) | |
602 | AT_KEYWORDS([ovsdb server positive unix cluster cluster3]) | |
603 | ovsdb_torture_test 3 3 remove | |
604 | AT_CLEANUP | |
605 | AT_SETUP([OVSDB 5-server torture test - remove/re-add leader]) | |
606 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
607 | ovsdb_torture_test 5 1 remove | |
608 | AT_CLEANUP | |
609 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1]) | |
610 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
611 | ovsdb_torture_test 5 2 remove | |
612 | AT_CLEANUP | |
613 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2]) | |
614 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
615 | ovsdb_torture_test 5 3 remove | |
616 | AT_CLEANUP | |
617 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3]) | |
618 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
619 | ovsdb_torture_test 5 4 remove | |
620 | AT_CLEANUP | |
621 | AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4]) | |
622 | AT_KEYWORDS([ovsdb server positive unix cluster cluster5]) | |
623 | ovsdb_torture_test 5 5 remove | |
624 | AT_CLEANUP |