tests/ovsdb-cluster.at

   1 OVS_START_SHELL_HELPERS
   2 # ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT TRANSACTION...
   3 ovsdb_check_cluster () {
   4     local n=$1 schema_func=$2 output=$3
   5     shift; shift; shift
   6
   7     $schema_func > schema
   8     schema=`ovsdb-tool schema-name schema`
   9     AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
  10     AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1])
  11     cid=`ovsdb-tool db-cid s1.db`
  12     for i in `seq 2 $n`; do
  13         AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft])
  14     done
  15
  16     on_exit 'kill `cat *.pid`'
  17     for i in `seq $n`; do
  18         AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
  19     done
  20     for i in `seq $n`; do
  21         AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema connected])
  22     done
  23
  24     for txn
  25     do
  26       AT_CHECK([ovsdb-client -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout])
  27       cat stdout >> output
  28     done
  29     AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output])
  30     for i in `seq $n`; do
  31         OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
  32     done
  33
  34     AT_CHECK([ovsdb-tool check-cluster s*.db])
  35 }
  36 OVS_END_SHELL_HELPERS
  37
  38 # Test a 1-server cluster.
  39 AT_BANNER([OVSDB - clustered transactions (1 server)])
  40 m4_define([OVSDB_CHECK_EXECUTION],
  41   [AT_SETUP([$1 - cluster of 1])
  42    AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5])
  43    ovsdb_check_cluster 1 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
  44    AT_CLEANUP])
  45 EXECUTION_EXAMPLES
  46
  47 # Test a 3-server cluster.
  48 AT_BANNER([OVSDB - clustered transactions (3 servers)])
  49 m4_define([OVSDB_CHECK_EXECUTION],
  50   [AT_SETUP([$1 - cluster of 3])
  51    AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5])
  52    ovsdb_check_cluster 3 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
  53    AT_CLEANUP])
  54 EXECUTION_EXAMPLES
  55
  56 # Test a 5-server cluster.
  57 AT_BANNER([OVSDB - clustered transactions (5 servers)])
  58 m4_define([OVSDB_CHECK_EXECUTION],
  59   [AT_SETUP([$1 - cluster of 5])
  60    AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5])
  61    ovsdb_check_cluster 5 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
  62    AT_CLEANUP])
  63 EXECUTION_EXAMPLES
  64 \f
  65
  66 AT_BANNER([OVSDB - disconnect from cluster])
  67
  68 OVS_START_SHELL_HELPERS
  69 # ovsdb_test_cluster_disconnect N_SERVERS LEADER_OR_FOLLOWER [CHECK_FLAPPING]
  70 # Test server disconnected from the cluster.
  71 # N_SERVERS: Number of servers in the cluster.
  72 # LEADER_OR_FOLLOWER: The role of the server that is disconnected from the
  73 #                     cluster: "leader" or "follower".
  74 # CHECK_FLAPPING: Whether to check if is_disconnected flapped. "yes", "no".
  75 ovsdb_test_cluster_disconnect () {
  76     n=$1
  77     leader_or_follower=$2
  78     check_flapping=$3
  79     schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
  80     ordinal_schema > schema
  81     AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
  82     cid=`ovsdb-tool db-cid s1.db`
  83     schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
  84     for i in `seq 2 $n`; do
  85         AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
  86     done
  87
  88     on_exit 'kill `cat *.pid`'
  89     for i in `seq $n`; do
  90         AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
  91     done
  92     for i in `seq $n`; do
  93         AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
  94     done
  95
  96     AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
  97           {"op": "insert",
  98            "table": "simple",
  99            "row": {"i": 1}}]]'], [0], [ignore], [ignore])
 100
 101     # When a node is disconnected from the cluster, the IDL should disconnect
 102     # and retry even if it uses a single remote, because the remote IP can be
 103     # a VIP on a load-balance. So we use single remote to test here.
 104     if test $leader_or_follower == "leader"; then
 105         target=1
 106         shutdown=`seq $(($n/2 + 1)) $n`
 107         cleanup=`seq $(($n/2))`
 108     else
 109         target=$n
 110
 111         # shutdown followers before the leader (s1) so that there is no chance for
 112         # s$n to become leader during the process.
 113         shutdown="`seq 2 $(($n/2 + 1))` 1"
 114         cleanup=`seq $(($n/2 + 2)) $n`
 115     fi
 116     echo shutdown=$shutdown
 117     echo cleanup=$cleanup
 118
 119     # Connect to $target.  Use "wait" to trigger a non-op transaction so
 120     # that test-ovsdb will not quit.
 121
 122     test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -v -t10 idl unix:s$target.ovsdb '[["idltest",
 123           {"op": "wait",
 124            "table": "simple",
 125            "where": [["i", "==", 1]],
 126            "columns": ["i"],
 127            "until": "==",
 128            "rows": [{"i": 1}]}]]' > test-ovsdb.log 2>&1 &
 129     echo $! > test-ovsdb.pid
 130
 131     OVS_WAIT_UNTIL([grep "000: i=1" test-ovsdb.log])
 132
 133     # Start collecting raft_is_connected logs for $target before shutting down
 134     # any servers.
 135     tail -f s$target.log > raft_is_connected.log &
 136     echo $! > tail.pid
 137
 138     # Shutdown the other servers so that $target is disconnected from the cluster.
 139     for i in $shutdown; do
 140         OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 141     done
 142
 143     # The test-ovsdb should detect the disconnect and retry.
 144     OVS_WAIT_UNTIL([grep disconnect test-ovsdb.log])
 145
 146     # The $target debug log should show raft_is_connected: false.
 147     OVS_WAIT_UNTIL([grep "raft_is_connected: false" raft_is_connected.log])
 148
 149     # Save the current count of "raft_is_connected: true"
 150     count_old=`grep "raft_is_connected: true" raft_is_connected.log | wc -l`
 151     echo count_old $count_old
 152
 153     if test X$check_flapping == X"yes"; then
 154         sleep 10
 155     fi
 156     # Make sure raft_is_connected didn't flap from false to true.
 157     count_new=`grep "raft_is_connected: true" raft_is_connected.log | wc -l`
 158     echo count_new $count_new
 159     AT_CHECK([test $count_new == $count_old])
 160
 161     for i in $cleanup; do
 162         OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 163     done
 164 }
 165 OVS_END_SHELL_HELPERS
 166
 167 AT_SETUP([OVSDB cluster - follower disconnect from cluster, single remote])
 168 AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
 169 ovsdb_test_cluster_disconnect 3 follower
 170 AT_CLEANUP
 171
 172 AT_SETUP([OVSDB cluster - leader disconnect from cluster, single remote])
 173 AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
 174 ovsdb_test_cluster_disconnect 3 leader
 175 AT_CLEANUP
 176
 177 AT_SETUP([OVSDB cluster - leader disconnect from cluster, check flapping])
 178 AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
 179 ovsdb_test_cluster_disconnect 5 leader yes
 180 AT_CLEANUP
 181
 182 \f
 183
 184 AT_BANNER([OVSDB cluster election timer change])
 185
 186 AT_SETUP([OVSDB cluster - election timer change])
 187 AT_KEYWORDS([ovsdb server positive unix cluster timer])
 188
 189 n=3
 190 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
 191 ordinal_schema > schema
 192 AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
 193 cid=`ovsdb-tool db-cid s1.db`
 194 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
 195 for i in `seq 2 $n`; do
 196     AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
 197 done
 198
 199 on_exit 'kill `cat *.pid`'
 200 for i in `seq $n`; do
 201     AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 202 done
 203 for i in `seq $n`; do
 204     AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
 205 done
 206
 207 # Change not allowed through follower.
 208 AT_CHECK([ovs-appctl -t "`pwd`"/s2 cluster/change-election-timer $schema_name 2000], [2], [], [ignore])
 209
 210 # Timer cannot be changed to bigger than 2x the original value.
 211 AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [2], [], [ignore])
 212
 213 AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 2000], [0], [dnl
 214 change of election timer initiated.
 215 ], [])
 216 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 2000"])
 217 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 2000"])
 218
 219 AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [0], [dnl
 220 change of election timer initiated.
 221 ], [])
 222 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"])
 223 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"])
 224
 225 # Latest timer should be used after restart
 226 for i in `seq $n`; do
 227     printf "\ns$i: stopping\n"
 228     OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 229 done
 230 for i in `seq $n`; do
 231     AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 232 done
 233 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"])
 234 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"])
 235
 236 AT_CLEANUP
 237
 238 \f
 239
 240 OVS_START_SHELL_HELPERS
 241 # ovsdb_cluster_failure_test SCHEMA_FUNC OUTPUT TRANSACTION...
 242 ovsdb_cluster_failure_test () {
 243     # Initial state: s1 is leader, s2 and s3 are followers
 244     remote_1=$1
 245     remote_2=$2
 246     crash_node=$3
 247     crash_command=$4
 248     if test "$crash_node" == "1"; then
 249         new_leader=$5
 250     fi
 251
 252     cp $top_srcdir/vswitchd/vswitch.ovsschema schema
 253     schema=`ovsdb-tool schema-name schema`
 254     AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl
 255 ovsdb|WARN|schema: changed 30 columns in 'Open_vSwitch' database from ephemeral to persistent, including 'status' column in 'Manager' table, because clusters do not support ephemeral columns
 256 ])
 257
 258     n=3
 259     join_cluster() {
 260         local i=$1
 261         others=
 262         for j in `seq 1 $n`; do
 263             if test $i != $j; then
 264                 others="$others unix:s$j.raft"
 265             fi
 266         done
 267         AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
 268     }
 269     start_server() {
 270         local i=$1
 271         printf "\ns$i: starting\n"
 272         AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 273     }
 274     connect_server() {
 275         local i=$1
 276         printf "\ns$i: waiting to connect to storage\n"
 277         AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
 278     }
 279     cid=`ovsdb-tool db-cid s1.db`
 280     for i in `seq 2 $n`; do join_cluster $i; done
 281
 282     on_exit 'kill `cat *.pid`'
 283     for i in `seq $n`; do start_server $i; done
 284     for i in `seq $n`; do connect_server $i; done
 285
 286     db=unix:s$remote_1.ovsdb,unix:s$remote_2.ovsdb
 287
 288     # To ensure $new_leader node the new leader, we delay election timer for
 289     # the other follower.
 290     if test -n "$new_leader"; then
 291         if test "$new_leader" == "2"; then
 292             delay_election_node=3
 293         else
 294             delay_election_node=2
 295         fi
 296         AT_CHECK([ovs-appctl -t "`pwd`"/s$delay_election_node cluster/failure-test delay-election], [0], [ignore])
 297     fi
 298     AT_CHECK([ovs-appctl -t "`pwd`"/s$crash_node cluster/failure-test $crash_command], [0], [ignore])
 299     AT_CHECK([ovs-vsctl -v --db="$db" --no-leader-only --no-shuffle-remotes --no-wait create QoS type=x], [0], [ignore], [ignore])
 300
 301     # Make sure that the node really crashed.
 302     AT_CHECK([ls s$crash_node.ovsdb], [2], [ignore], [ignore])
 303     # XXX: Client will fail if remotes contains unix socket that doesn't exist (killed).
 304     if test "$remote_1" = "$crash_node"; then
 305         db=unix:s$remote_2.ovsdb
 306     fi
 307     AT_CHECK([ovs-vsctl --db="$db" --no-leader-only --no-wait --columns=type --bare list QoS], [0], [x
 308 ])
 309 }
 310 OVS_END_SHELL_HELPERS
 311 AT_BANNER([OVSDB - cluster failure with pending transaction])
 312
 313 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-2 becomes leader])
 314 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 315 ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 2
 316 AT_CLEANUP
 317
 318 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-3 becomes leader])
 319 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 320 ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 3
 321 AT_CLEANUP
 322
 323 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-2 becomes leader])
 324 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 325 ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 2
 326 AT_CLEANUP
 327
 328 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-3 becomes leader])
 329 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 330 ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 3
 331 AT_CLEANUP
 332
 333 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-2 becomes leader])
 334 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 335 ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 2
 336 AT_CLEANUP
 337
 338 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-3 becomes leader])
 339 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 340 ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 3
 341 AT_CLEANUP
 342
 343 AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-2 becomes leader])
 344 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 345 ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 2
 346 AT_CLEANUP
 347
 348 AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-3 becomes leader])
 349 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 350 ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 3
 351 AT_CLEANUP
 352
 353 AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-2 becomes leader])
 354 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 355 # XXX: Detect and skip repeated transaction before enabling this test
 356 AT_CHECK([exit 77])
 357 ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 2
 358 AT_CLEANUP
 359
 360 AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-3 becomes leader])
 361 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 362 # XXX: Detect and skip repeated transaction before enabling this test
 363 AT_CHECK([exit 77])
 364 ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 3
 365 AT_CLEANUP
 366
 367 AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to follower-3])
 368 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 369 ovsdb_cluster_failure_test 2 3 2 crash-before-sending-execute-command-request
 370 AT_CLEANUP
 371
 372 AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to leader])
 373 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 374 ovsdb_cluster_failure_test 2 1 2 crash-before-sending-execute-command-request
 375 AT_CLEANUP
 376
 377 AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to follower-3])
 378 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 379 # XXX: Detect and skip repeated transaction before enabling this test
 380 AT_CHECK([exit 77])
 381 ovsdb_cluster_failure_test 2 3 2 crash-after-sending-execute-command-request
 382 AT_CLEANUP
 383
 384 AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to leader])
 385 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 386 # XXX: Detect and skip repeated transaction before enabling this test
 387 AT_CHECK([exit 77])
 388 ovsdb_cluster_failure_test 2 1 2 crash-after-sending-execute-command-request
 389 AT_CLEANUP
 390
 391 AT_SETUP([OVSDB cluster - txn on leader, follower-2 crash after receiving appendReq for the update])
 392 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 393 ovsdb_cluster_failure_test 1 1 2 crash-after-receiving-append-request-update
 394 AT_CLEANUP
 395
 396 AT_SETUP([OVSDB cluster - txn on follower-2, follower-3 crash after receiving appendReq for the update])
 397 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 398 ovsdb_cluster_failure_test 2 2 3 crash-after-receiving-append-request-update
 399 AT_CLEANUP
 400
 401 \f
 402 AT_BANNER([OVSDB - cluster tests])
 403
 404 # Torture test.
 405 OVS_START_SHELL_HELPERS
 406 ovsdb_torture_test () {
 407     local n=$1                  # Number of cluster members
 408     local victim=$2             # Cluster member to kill or remove
 409     local variant=$3            # 'kill' and restart or 'remove' and add
 410     cp $top_srcdir/vswitchd/vswitch.ovsschema schema
 411     schema=`ovsdb-tool schema-name schema`
 412     AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl
 413 ovsdb|WARN|schema: changed 30 columns in 'Open_vSwitch' database from ephemeral to persistent, including 'status' column in 'Manager' table, because clusters do not support ephemeral columns
 414 ])
 415
 416     join_cluster() {
 417         local i=$1
 418         others=
 419         for j in `seq 1 $n`; do
 420             if test $i != $j; then
 421                 others="$others unix:s$j.raft"
 422             fi
 423         done
 424         AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
 425     }
 426
 427     start_server() {
 428         local i=$1
 429         printf "\ns$i: starting\n"
 430         AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 431     }
 432     stop_server() {
 433         local i=$1
 434         printf "\ns$i: stopping\n"
 435         OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 436     }
 437     connect_server() {
 438         local i=$1
 439         printf "\ns$i: waiting to connect to storage\n"
 440         AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
 441     }
 442     remove_server() {
 443         local i=$1
 444         printf "\ns$i: removing from cluster\n"
 445         AT_CHECK([ovs-appctl -t "`pwd`"/s$i cluster/leave Open_vSwitch])
 446         printf "\ns$i: waiting for removal to complete\n"
 447         AT_CHECK([ovsdb_client_wait --log-file=remove$i.log unix:s$i.ovsdb $schema removed])
 448         stop_server $i
 449     }
 450     add_server() {
 451         local i=$1
 452         rm s$i.db
 453         join_cluster $i
 454         start_server $i
 455         connect_server $i
 456     }
 457
 458     cid=`ovsdb-tool db-cid s1.db`
 459     for i in `seq 2 $n`; do join_cluster $i; done
 460
 461     on_exit 'kill `cat *.pid`'
 462     for i in `seq $n`; do start_server $i; done
 463     for i in `seq $n`; do connect_server $i; done
 464
 465     db=unix:s1.ovsdb
 466     for i in `seq 2 $n`; do
 467         db=$db,unix:s$i.ovsdb
 468     done
 469
 470     n1=10 n2=5 n3=50
 471     echo "starting $n1*$n2 ovs-vsctl processes..."
 472     for i in $(seq 0 $(expr $n1 - 1) ); do
 473         (for j in $(seq $n2); do
 474              : > $i-$j.running
 475              txn="add Open_vSwitch . external_ids $i-$j=$i-$j"
 476              for k in $(seq $n3); do
 477                  txn="$txn -- add Open_vSwitch . external_ids $i-$j-$k=$i-$j-$k"
 478              done
 479              run_as "ovs-vsctl($i-$j)" ovs-vsctl "-vPATTERN:console:ovs-vsctl($i-$j)|%D{%H:%M:%S}|%05N|%c|%p|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --db="$db" --no-leader-only --no-wait $txn
 480              status=$?
 481              if test $status != 0; then
 482                  echo "$i-$j exited with status $status" > $i-$j:$status
 483              fi
 484              rm $i-$j.running
 485          done
 486          : > $i.done)&
 487     done
 488     echo "...done"
 489
 490     echo "waiting for ovs-vsctl processes to exit..."
 491     # Use file instead of var because code inside "while" runs in a subshell.
 492     echo 0 > phase
 493     i=0
 494     (while :; do echo; sleep 0.1; done) | while read REPLY; do
 495         printf "t=%2d s:" $i
 496         done=0
 497         for j in $(seq 0 $(expr $n1 - 1)); do
 498             if test -f $j.done; then
 499                 printf " $j"
 500                 done=$(expr $done + 1)
 501             fi
 502         done
 503         printf '\n'
 504         if test $done = $n1; then
 505             break
 506         fi
 507
 508         case $(cat phase) in # (
 509         0)
 510             if test $done -ge $(expr $n1 / 10); then
 511                 if test $variant = kill; then
 512                     stop_server $victim
 513                 else
 514                     remove_server $victim
 515                 fi
 516                 echo 1 > phase
 517                 next=$(expr $i + 2)
 518             fi
 519             ;; # (
 520         1)
 521             if test $i -ge $next; then
 522                 if test $variant = kill; then
 523                     start_server $victim
 524                     connect_server $victim
 525                 else
 526                     add_server $victim
 527                 fi
 528                 echo 2 > phase
 529             fi
 530             ;;
 531         esac
 532
 533         i=$(expr $i + 1)
 534     done
 535     echo "...done"
 536     AT_CHECK([if test $(cat phase) != 2; then exit 77; fi])
 537
 538     for i in $(seq 0 $(expr $n1 - 1) ); do
 539         for j in `seq $n2`; do
 540             echo "$i-$j=$i-$j"
 541             for k in `seq $n3`; do
 542                 echo "$i-$j-$k=$i-$j-$k"
 543             done
 544         done
 545     done | sort > expout
 546     AT_CHECK([ovs-vsctl --db="$db" --no-wait --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get Open_vSwitch . external-ids | tr ',' '\n' | sed 's/[[{}"" ]]//g' | sort], [0], [expout])
 547
 548     for i in `seq $n`; do
 549         if test $i != $victim || test $(cat phase) != 1; then
 550             stop_server $i
 551         fi
 552     done
 553
 554     # We ignore stdout because non-fatal warnings get printed there.
 555     AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore])
 556 }
 557 OVS_END_SHELL_HELPERS
 558
 559 AT_SETUP([OVSDB 3-server torture test - kill/restart leader])
 560 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
 561 ovsdb_torture_test 3 1 kill
 562 AT_CLEANUP
 563 AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1])
 564 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
 565 ovsdb_torture_test 3 2 kill
 566 AT_CLEANUP
 567 AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2])
 568 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
 569 ovsdb_torture_test 3 3 kill
 570 AT_CLEANUP
 571 AT_SETUP([OVSDB 5-server torture test - kill/restart leader])
 572 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 573 ovsdb_torture_test 5 1 kill
 574 AT_CLEANUP
 575 AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1])
 576 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 577 ovsdb_torture_test 5 2 kill
 578 AT_CLEANUP
 579 AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2])
 580 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 581 ovsdb_torture_test 5 3 kill
 582 AT_CLEANUP
 583 AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3])
 584 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 585 ovsdb_torture_test 5 4 kill
 586 AT_CLEANUP
 587 AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4])
 588 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 589 ovsdb_torture_test 5 5 kill
 590 AT_CLEANUP
 591
 592 AT_SETUP([OVSDB 3-server torture test - remove/re-add leader])
 593 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
 594 ovsdb_torture_test 3 1 remove
 595 AT_CLEANUP
 596 AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1])
 597 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
 598 ovsdb_torture_test 3 2 remove
 599 AT_CLEANUP
 600 AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2])
 601 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
 602 ovsdb_torture_test 3 3 remove
 603 AT_CLEANUP
 604 AT_SETUP([OVSDB 5-server torture test - remove/re-add leader])
 605 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 606 ovsdb_torture_test 5 1 remove
 607 AT_CLEANUP
 608 AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1])
 609 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 610 ovsdb_torture_test 5 2 remove
 611 AT_CLEANUP
 612 AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2])
 613 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 614 ovsdb_torture_test 5 3 remove
 615 AT_CLEANUP
 616 AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3])
 617 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 618 ovsdb_torture_test 5 4 remove
 619 AT_CLEANUP
 620 AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4])
 621 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 622 ovsdb_torture_test 5 5 remove
 623 AT_CLEANUP