tests/ovsdb-cluster.at

   1 OVS_START_SHELL_HELPERS
   2 # ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT TRANSACTION...
   3 ovsdb_check_cluster () {
   4     local n=$1 schema_func=$2 output=$3
   5     shift; shift; shift
   6
   7     $schema_func > schema
   8     schema=`ovsdb-tool schema-name schema`
   9     AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
  10     AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1])
  11     cid=`ovsdb-tool db-cid s1.db`
  12     for i in `seq 2 $n`; do
  13         AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft])
  14     done
  15
  16     on_exit 'kill `cat *.pid`'
  17     for i in `seq $n`; do
  18         AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
  19     done
  20     for i in `seq $n`; do
  21         AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema connected])
  22     done
  23
  24     for txn
  25     do
  26       AT_CHECK([ovsdb-client -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout])
  27       cat stdout >> output
  28     done
  29     AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output])
  30     for i in `seq $n`; do
  31         OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
  32     done
  33
  34     AT_CHECK([ovsdb-tool check-cluster s*.db])
  35 }
  36 OVS_END_SHELL_HELPERS
  37
  38 # Test a 1-server cluster.
  39 AT_BANNER([OVSDB - clustered transactions (1 server)])
  40 m4_define([OVSDB_CHECK_EXECUTION],
  41   [AT_SETUP([$1 - cluster of 1])
  42    AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5])
  43    ovsdb_check_cluster 1 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
  44    AT_CLEANUP])
  45 EXECUTION_EXAMPLES
  46
  47 # Test a 3-server cluster.
  48 AT_BANNER([OVSDB - clustered transactions (3 servers)])
  49 m4_define([OVSDB_CHECK_EXECUTION],
  50   [AT_SETUP([$1 - cluster of 3])
  51    AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5])
  52    ovsdb_check_cluster 3 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
  53    AT_CLEANUP])
  54 EXECUTION_EXAMPLES
  55
  56 # Test a 5-server cluster.
  57 AT_BANNER([OVSDB - clustered transactions (5 servers)])
  58 m4_define([OVSDB_CHECK_EXECUTION],
  59   [AT_SETUP([$1 - cluster of 5])
  60    AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5])
  61    ovsdb_check_cluster 5 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
  62    AT_CLEANUP])
  63 EXECUTION_EXAMPLES
  64 \f
  65
  66 AT_BANNER([OVSDB - disconnect from cluster])
  67
  68 OVS_START_SHELL_HELPERS
  69 # ovsdb_test_cluster_disconnect N_SERVERS LEADER_OR_FOLLOWER [CHECK_FLAPPING]
  70 # Test server disconnected from the cluster.
  71 # N_SERVERS: Number of servers in the cluster.
  72 # LEADER_OR_FOLLOWER: The role of the server that is disconnected from the
  73 #                     cluster: "leader" or "follower".
  74 # CHECK_FLAPPING: Whether to check if is_disconnected flapped. "yes", "no".
  75 ovsdb_test_cluster_disconnect () {
  76     n=$1
  77     leader_or_follower=$2
  78     check_flapping=$3
  79     schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
  80     ordinal_schema > schema
  81     AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
  82     cid=`ovsdb-tool db-cid s1.db`
  83     schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
  84     for i in `seq 2 $n`; do
  85         AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
  86     done
  87
  88     on_exit 'kill `cat *.pid`'
  89     for i in `seq $n`; do
  90         AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
  91     done
  92     for i in `seq $n`; do
  93         AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
  94     done
  95
  96     AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
  97           {"op": "insert",
  98            "table": "simple",
  99            "row": {"i": 1}}]]'], [0], [ignore], [ignore])
 100
 101     # When a node is disconnected from the cluster, the IDL should disconnect
 102     # and retry even if it uses a single remote, because the remote IP can be
 103     # a VIP on a load-balance. So we use single remote to test here.
 104     if test $leader_or_follower == "leader"; then
 105         target=1
 106         shutdown=`seq $(($n/2 + 1)) $n`
 107         cleanup=`seq $(($n/2))`
 108     else
 109         target=$n
 110
 111         # shutdown followers before the leader (s1) so that there is no chance for
 112         # s$n to become leader during the process.
 113         shutdown="`seq 2 $(($n/2 + 1))` 1"
 114         cleanup=`seq $(($n/2 + 2)) $n`
 115     fi
 116     echo shutdown=$shutdown
 117     echo cleanup=$cleanup
 118
 119     # Connect to $target.  Use "wait" to trigger a non-op transaction so
 120     # that test-ovsdb will not quit.
 121
 122     test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -v -t10 idl unix:s$target.ovsdb '[["idltest",
 123           {"op": "wait",
 124            "table": "simple",
 125            "where": [["i", "==", 1]],
 126            "columns": ["i"],
 127            "until": "==",
 128            "rows": [{"i": 1}]}]]' > test-ovsdb.log 2>&1 &
 129     echo $! > test-ovsdb.pid
 130
 131     OVS_WAIT_UNTIL([grep "000: i=1" test-ovsdb.log])
 132
 133     # Start collecting raft_is_connected logs for $target before shutting down
 134     # any servers.
 135     tail -f s$target.log > raft_is_connected.log &
 136     echo $! > tail.pid
 137
 138     # Shutdown the other servers so that $target is disconnected from the cluster.
 139     for i in $shutdown; do
 140         OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 141     done
 142
 143     # The test-ovsdb should detect the disconnect and retry.
 144     OVS_WAIT_UNTIL([grep disconnect test-ovsdb.log])
 145
 146     # The $target debug log should show raft_is_connected: false.
 147     OVS_WAIT_UNTIL([grep "raft_is_connected: false" raft_is_connected.log])
 148
 149     # Save the current count of "raft_is_connected: true"
 150     count_old=`grep "raft_is_connected: true" raft_is_connected.log | wc -l`
 151     echo count_old $count_old
 152
 153     if test X$check_flapping == X"yes"; then
 154         sleep 10
 155     fi
 156     # Make sure raft_is_connected didn't flap from false to true.
 157     count_new=`grep "raft_is_connected: true" raft_is_connected.log | wc -l`
 158     echo count_new $count_new
 159     AT_CHECK([test $count_new == $count_old])
 160
 161     for i in $cleanup; do
 162         OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 163     done
 164 }
 165 OVS_END_SHELL_HELPERS
 166
 167 AT_SETUP([OVSDB cluster - follower disconnect from cluster, single remote])
 168 AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
 169 ovsdb_test_cluster_disconnect 3 follower
 170 AT_CLEANUP
 171
 172 AT_SETUP([OVSDB cluster - leader disconnect from cluster, single remote])
 173 AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
 174 ovsdb_test_cluster_disconnect 3 leader
 175 AT_CLEANUP
 176
 177 AT_SETUP([OVSDB cluster - leader disconnect from cluster, check flapping])
 178 AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
 179 ovsdb_test_cluster_disconnect 5 leader yes
 180 AT_CLEANUP
 181
 182 AT_SETUP([OVSDB cluster - initial status should be disconnected])
 183 AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
 184
 185 n=3
 186 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
 187 ordinal_schema > schema
 188 AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
 189 cid=`ovsdb-tool db-cid s1.db`
 190 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
 191 for i in `seq 2 $n`; do
 192     AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
 193 done
 194
 195 on_exit 'kill `cat *.pid`'
 196 for i in `seq $n`; do
 197     AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 198 done
 199 for i in `seq $n`; do
 200     AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
 201 done
 202
 203 # Stop all servers, and start the s1 only, to test initial connection status
 204 # when there is no leader yet.
 205 for i in `seq 1 $n`; do
 206     OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 207 done
 208 i=1
 209 AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 210
 211 # The initial status should be disconnected. So wait should fail.
 212 AT_CHECK([ovsdb_client_wait --timeout=1 unix:s$i.ovsdb $schema_name connected], [142], [ignore], [ignore])
 213 OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 214
 215 AT_CLEANUP
 216
 217 \f
 218
 219 AT_BANNER([OVSDB cluster election timer change])
 220
 221 AT_SETUP([OVSDB cluster - election timer change])
 222 AT_KEYWORDS([ovsdb server positive unix cluster timer])
 223
 224 n=3
 225 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
 226 ordinal_schema > schema
 227 AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
 228 cid=`ovsdb-tool db-cid s1.db`
 229 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
 230 for i in `seq 2 $n`; do
 231     AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
 232 done
 233
 234 on_exit 'kill `cat *.pid`'
 235 for i in `seq $n`; do
 236     AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 237 done
 238 for i in `seq $n`; do
 239     AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
 240 done
 241
 242 # Change not allowed through follower.
 243 AT_CHECK([ovs-appctl -t "`pwd`"/s2 cluster/change-election-timer $schema_name 2000], [2], [], [ignore])
 244
 245 # Timer cannot be changed to bigger than 2x the original value.
 246 AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [2], [], [ignore])
 247
 248 AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 2000], [0], [dnl
 249 change of election timer initiated.
 250 ], [])
 251 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 2000"])
 252 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 2000"])
 253
 254 AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [0], [dnl
 255 change of election timer initiated.
 256 ], [])
 257 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"])
 258 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"])
 259
 260 # Latest timer should be used after restart
 261 for i in `seq $n`; do
 262     printf "\ns$i: stopping\n"
 263     OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 264 done
 265 for i in `seq $n`; do
 266     AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 267 done
 268 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"])
 269 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"])
 270
 271 # Wait until cluster is ready
 272 for i in `seq $n`; do
 273     OVS_WAIT_WHILE([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "Leader: unknown"])
 274 done
 275
 276 # Latest timer should be restored after DB compact and restart.
 277 # This is to test the install_snapshot RPC.
 278
 279 # Compact online
 280 for i in `seq $n`; do
 281     AT_CHECK([ovs-appctl -t "`pwd`"/s$i ovsdb-server/compact])
 282 done
 283
 284 for i in `seq $n`; do
 285     printf "\ns$i: stopping\n"
 286     OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 287 done
 288 for i in `seq $n`; do
 289     AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 290 done
 291 for i in `seq $n`; do
 292     OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "Election timer: 4000"])
 293 done
 294
 295 # Wait until cluster is ready
 296 for i in `seq $n`; do
 297     OVS_WAIT_WHILE([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "Leader: unknown"])
 298 done
 299
 300 # Newly joined member should use latest timer value
 301 AT_CHECK([ovsdb-tool join-cluster s4.db $schema_name unix:s4.raft unix:s1.raft])
 302 AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s4.log --pidfile=s4.pid --unixctl=s4 --remote=punix:s4.ovsdb s4.db])
 303 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s4 cluster/status $schema_name | grep "Election timer: 4000"])
 304 # for i in `seq 10`; do
 305 #     ovs-appctl -t "`pwd`"/s4 cluster/status $schema_name
 306 #     sleep 1
 307 # done
 308
 309 AT_CLEANUP
 310
 311 \f
 312 AT_BANNER([OVSDB cluster install snapshot RPC])
 313
 314 AT_SETUP([OVSDB cluster - install snapshot RPC])
 315 AT_KEYWORDS([ovsdb server positive unix cluster snapshot])
 316
 317 n=3
 318 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
 319 ordinal_schema > schema
 320 AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
 321 cid=`ovsdb-tool db-cid s1.db`
 322 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
 323 for i in `seq 2 $n`; do
 324     AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
 325 done
 326
 327 on_exit 'kill `cat *.pid`'
 328 for i in `seq $n`; do
 329     AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 330 done
 331 for i in `seq $n`; do
 332     AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
 333 done
 334
 335 # Kill one follower (s2) and write some data to cluster, so that the follower is falling behind
 336 printf "\ns2: stopping\n"
 337 OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s2], [s2.pid])
 338
 339 AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
 340       {"op": "insert",
 341        "table": "simple",
 342        "row": {"i": 1}}]]'], [0], [ignore], [ignore])
 343
 344 # Compact leader online to generate snapshot
 345 AT_CHECK([ovs-appctl -t "`pwd`"/s1 ovsdb-server/compact])
 346
 347 # Start the follower s2 again.
 348 AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s2.log --pidfile=s2.pid --unixctl=s2 --remote=punix:s2.ovsdb s2.db])
 349 AT_CHECK([ovsdb_client_wait unix:s2.ovsdb $schema_name connected])
 350
 351 # A client transaction through s2. During this transaction, there will be a
 352 # install_snapshot RPC because s2 detects it is behind and s1 doesn't have the
 353 # pre_log_index requested by s2 because it is already compacted.
 354 # After the install_snapshot RPC process, the transaction through s2 should
 355 # succeed.
 356 AT_CHECK([ovsdb-client transact unix:s2.ovsdb '[["idltest",
 357       {"op": "insert",
 358        "table": "simple",
 359        "row": {"i": 1}}]]'], [0], [ignore], [ignore])
 360
 361 for i in `seq $n`; do
 362     OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 363 done
 364
 365 AT_CLEANUP
 366
 367 \f
 368
 369 OVS_START_SHELL_HELPERS
 370 # ovsdb_cluster_failure_test SCHEMA_FUNC OUTPUT TRANSACTION...
 371 ovsdb_cluster_failure_test () {
 372     # Initial state: s1 is leader, s2 and s3 are followers
 373     remote_1=$1
 374     remote_2=$2
 375     crash_node=$3
 376     crash_command=$4
 377     if test "$crash_node" == "1"; then
 378         new_leader=$5
 379     fi
 380
 381     cp $top_srcdir/vswitchd/vswitch.ovsschema schema
 382     schema=`ovsdb-tool schema-name schema`
 383     AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl
 384 ovsdb|WARN|schema: changed 30 columns in 'Open_vSwitch' database from ephemeral to persistent, including 'status' column in 'Manager' table, because clusters do not support ephemeral columns
 385 ])
 386
 387     n=3
 388     join_cluster() {
 389         local i=$1
 390         others=
 391         for j in `seq 1 $n`; do
 392             if test $i != $j; then
 393                 others="$others unix:s$j.raft"
 394             fi
 395         done
 396         AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
 397     }
 398     start_server() {
 399         local i=$1
 400         printf "\ns$i: starting\n"
 401         AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 402     }
 403     connect_server() {
 404         local i=$1
 405         printf "\ns$i: waiting to connect to storage\n"
 406         AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
 407     }
 408     cid=`ovsdb-tool db-cid s1.db`
 409     for i in `seq 2 $n`; do join_cluster $i; done
 410
 411     on_exit 'kill `cat *.pid`'
 412     for i in `seq $n`; do start_server $i; done
 413     for i in `seq $n`; do connect_server $i; done
 414
 415     db=unix:s$remote_1.ovsdb,unix:s$remote_2.ovsdb
 416
 417     # To ensure $new_leader node the new leader, we delay election timer for
 418     # the other follower.
 419     if test -n "$new_leader"; then
 420         if test "$new_leader" == "2"; then
 421             delay_election_node=3
 422         else
 423             delay_election_node=2
 424         fi
 425         AT_CHECK([ovs-appctl -t "`pwd`"/s$delay_election_node cluster/failure-test delay-election], [0], [ignore])
 426     fi
 427     AT_CHECK([ovs-appctl -t "`pwd`"/s$crash_node cluster/failure-test $crash_command], [0], [ignore])
 428     AT_CHECK([ovs-vsctl -v --db="$db" --no-leader-only --no-shuffle-remotes --no-wait create QoS type=x], [0], [ignore], [ignore])
 429
 430     # Make sure that the node really crashed.
 431     AT_CHECK([ls s$crash_node.ovsdb], [2], [ignore], [ignore])
 432     # XXX: Client will fail if remotes contains unix socket that doesn't exist (killed).
 433     if test "$remote_1" = "$crash_node"; then
 434         db=unix:s$remote_2.ovsdb
 435     fi
 436     AT_CHECK([ovs-vsctl --db="$db" --no-leader-only --no-wait --columns=type --bare list QoS], [0], [x
 437 ])
 438 }
 439 OVS_END_SHELL_HELPERS
 440 AT_BANNER([OVSDB - cluster failure with pending transaction])
 441
 442 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-2 becomes leader])
 443 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 444 ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 2
 445 AT_CLEANUP
 446
 447 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-3 becomes leader])
 448 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 449 ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 3
 450 AT_CLEANUP
 451
 452 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-2 becomes leader])
 453 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 454 ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 2
 455 AT_CLEANUP
 456
 457 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-3 becomes leader])
 458 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 459 ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 3
 460 AT_CLEANUP
 461
 462 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-2 becomes leader])
 463 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 464 ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 2
 465 AT_CLEANUP
 466
 467 AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-3 becomes leader])
 468 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 469 ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 3
 470 AT_CLEANUP
 471
 472 AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-2 becomes leader])
 473 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 474 ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 2
 475 AT_CLEANUP
 476
 477 AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-3 becomes leader])
 478 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 479 ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 3
 480 AT_CLEANUP
 481
 482 AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-2 becomes leader])
 483 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 484 # XXX: Detect and skip repeated transaction before enabling this test
 485 AT_CHECK([exit 77])
 486 ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 2
 487 AT_CLEANUP
 488
 489 AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-3 becomes leader])
 490 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 491 # XXX: Detect and skip repeated transaction before enabling this test
 492 AT_CHECK([exit 77])
 493 ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 3
 494 AT_CLEANUP
 495
 496 AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to follower-3])
 497 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 498 ovsdb_cluster_failure_test 2 3 2 crash-before-sending-execute-command-request
 499 AT_CLEANUP
 500
 501 AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to leader])
 502 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 503 ovsdb_cluster_failure_test 2 1 2 crash-before-sending-execute-command-request
 504 AT_CLEANUP
 505
 506 AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to follower-3])
 507 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 508 # XXX: Detect and skip repeated transaction before enabling this test
 509 AT_CHECK([exit 77])
 510 ovsdb_cluster_failure_test 2 3 2 crash-after-sending-execute-command-request
 511 AT_CLEANUP
 512
 513 AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to leader])
 514 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 515 # XXX: Detect and skip repeated transaction before enabling this test
 516 AT_CHECK([exit 77])
 517 ovsdb_cluster_failure_test 2 1 2 crash-after-sending-execute-command-request
 518 AT_CLEANUP
 519
 520 AT_SETUP([OVSDB cluster - txn on leader, follower-2 crash after receiving appendReq for the update])
 521 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 522 ovsdb_cluster_failure_test 1 1 2 crash-after-receiving-append-request-update
 523 AT_CLEANUP
 524
 525 AT_SETUP([OVSDB cluster - txn on follower-2, follower-3 crash after receiving appendReq for the update])
 526 AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
 527 ovsdb_cluster_failure_test 2 2 3 crash-after-receiving-append-request-update
 528 AT_CLEANUP
 529
 530 \f
 531 AT_SETUP([OVSDB cluster - competing candidates])
 532 AT_KEYWORDS([ovsdb server negative unix cluster competing-candidates])
 533
 534 n=3
 535 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
 536 ordinal_schema > schema
 537 AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
 538 cid=`ovsdb-tool db-cid s1.db`
 539 schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
 540 for i in `seq 2 $n`; do
 541     AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
 542 done
 543
 544 on_exit 'kill `cat *.pid`'
 545 for i in `seq $n`; do
 546     AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 547 done
 548 for i in `seq $n`; do
 549     AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
 550 done
 551
 552 # We need to simulate the situation when 2 candidates starts election with same
 553 # term.
 554 #
 555 # Before triggering leader election, tell follower s2 don't send vote request (simulating
 556 # vote-request lost or not handled in time), and tell follower s3 to delay
 557 # election timer to make sure s3 doesn't send vote-request before s2 enters
 558 # term 2.
 559 AT_CHECK([ovs-appctl -t "`pwd`"/s2 cluster/failure-test dont-send-vote-request], [0], [ignore])
 560 AT_CHECK([ovs-appctl -t "`pwd`"/s3 cluster/failure-test delay-election], [0], [ignore])
 561
 562 # Restart leader, which will become follower, and both old followers will start
 563 # election as candidate. The new follower (old leader) will vote one of them,
 564 # and the other candidate should step back as follower as again.
 565 kill -9 `cat s1.pid`
 566 AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s1.log --pidfile=s1.pid --unixctl=s1 --remote=punix:s1.ovsdb s1.db])
 567
 568 # Tell s1 to delay election timer so that it won't start election before s3
 569 # becomes candidate.
 570 AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/failure-test delay-election], [0], [ignore])
 571
 572 OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Term: 2"])
 573
 574 for i in `seq $n`; do
 575     OVS_WAIT_WHILE([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "candidate"])
 576     AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
 577 done
 578
 579 for i in `seq $n`; do
 580     OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 581 done
 582
 583 AT_CLEANUP
 584
 585 \f
 586 AT_BANNER([OVSDB - cluster tests])
 587
 588 # Torture test.
 589 OVS_START_SHELL_HELPERS
 590 ovsdb_torture_test () {
 591     local n=$1                  # Number of cluster members
 592     local victim=$2             # Cluster member to kill or remove
 593     local variant=$3            # 'kill' and restart or 'remove' and add
 594     cp $top_srcdir/vswitchd/vswitch.ovsschema schema
 595     schema=`ovsdb-tool schema-name schema`
 596     AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl
 597 ovsdb|WARN|schema: changed 30 columns in 'Open_vSwitch' database from ephemeral to persistent, including 'status' column in 'Manager' table, because clusters do not support ephemeral columns
 598 ])
 599
 600     join_cluster() {
 601         local i=$1
 602         others=
 603         for j in `seq 1 $n`; do
 604             if test $i != $j; then
 605                 others="$others unix:s$j.raft"
 606             fi
 607         done
 608         AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
 609     }
 610
 611     start_server() {
 612         local i=$1
 613         printf "\ns$i: starting\n"
 614         AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
 615     }
 616     stop_server() {
 617         local i=$1
 618         printf "\ns$i: stopping\n"
 619         OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
 620     }
 621     connect_server() {
 622         local i=$1
 623         printf "\ns$i: waiting to connect to storage\n"
 624         AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
 625     }
 626     remove_server() {
 627         local i=$1
 628         printf "\ns$i: removing from cluster\n"
 629         AT_CHECK([ovs-appctl -t "`pwd`"/s$i cluster/leave Open_vSwitch])
 630         printf "\ns$i: waiting for removal to complete\n"
 631         AT_CHECK([ovsdb_client_wait --log-file=remove$i.log unix:s$i.ovsdb $schema removed])
 632         stop_server $i
 633     }
 634     add_server() {
 635         local i=$1
 636         rm s$i.db
 637         join_cluster $i
 638         start_server $i
 639         connect_server $i
 640     }
 641
 642     cid=`ovsdb-tool db-cid s1.db`
 643     for i in `seq 2 $n`; do join_cluster $i; done
 644
 645     on_exit 'kill `cat *.pid`'
 646     for i in `seq $n`; do start_server $i; done
 647     for i in `seq $n`; do connect_server $i; done
 648
 649     db=unix:s1.ovsdb
 650     for i in `seq 2 $n`; do
 651         db=$db,unix:s$i.ovsdb
 652     done
 653
 654     n1=10 n2=5 n3=50
 655     echo "starting $n1*$n2 ovs-vsctl processes..."
 656     for i in $(seq 0 $(expr $n1 - 1) ); do
 657         (for j in $(seq $n2); do
 658              : > $i-$j.running
 659              txn="add Open_vSwitch . external_ids $i-$j=$i-$j"
 660              for k in $(seq $n3); do
 661                  txn="$txn -- add Open_vSwitch . external_ids $i-$j-$k=$i-$j-$k"
 662              done
 663              run_as "ovs-vsctl($i-$j)" ovs-vsctl "-vPATTERN:console:ovs-vsctl($i-$j)|%D{%H:%M:%S}|%05N|%c|%p|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --db="$db" --no-leader-only --no-wait $txn
 664              status=$?
 665              if test $status != 0; then
 666                  echo "$i-$j exited with status $status" > $i-$j:$status
 667              fi
 668              rm $i-$j.running
 669          done
 670          : > $i.done)&
 671     done
 672     echo "...done"
 673
 674     echo "waiting for ovs-vsctl processes to exit..."
 675     # Use file instead of var because code inside "while" runs in a subshell.
 676     echo 0 > phase
 677     i=0
 678     (while :; do echo; sleep 0.1; done) | while read REPLY; do
 679         printf "t=%2d s:" $i
 680         done=0
 681         for j in $(seq 0 $(expr $n1 - 1)); do
 682             if test -f $j.done; then
 683                 printf " $j"
 684                 done=$(expr $done + 1)
 685             fi
 686         done
 687         printf '\n'
 688         if test $done = $n1; then
 689             break
 690         fi
 691
 692         case $(cat phase) in # (
 693         0)
 694             if test $done -ge $(expr $n1 / 10); then
 695                 if test $variant = kill; then
 696                     stop_server $victim
 697                 else
 698                     remove_server $victim
 699                 fi
 700                 echo 1 > phase
 701                 next=$(expr $i + 2)
 702             fi
 703             ;; # (
 704         1)
 705             if test $i -ge $next; then
 706                 if test $variant = kill; then
 707                     start_server $victim
 708                     connect_server $victim
 709                 else
 710                     add_server $victim
 711                 fi
 712                 echo 2 > phase
 713             fi
 714             ;;
 715         esac
 716
 717         i=$(expr $i + 1)
 718     done
 719     echo "...done"
 720     AT_CHECK([if test $(cat phase) != 2; then exit 77; fi])
 721
 722     for i in $(seq 0 $(expr $n1 - 1) ); do
 723         for j in `seq $n2`; do
 724             echo "$i-$j=$i-$j"
 725             for k in `seq $n3`; do
 726                 echo "$i-$j-$k=$i-$j-$k"
 727             done
 728         done
 729     done | sort > expout
 730     AT_CHECK([ovs-vsctl --db="$db" --no-wait --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get Open_vSwitch . external-ids | tr ',' '\n' | sed 's/[[{}"" ]]//g' | sort], [0], [expout])
 731
 732     for i in `seq $n`; do
 733         if test $i != $victim || test $(cat phase) != 1; then
 734             stop_server $i
 735         fi
 736     done
 737
 738     # We ignore stdout because non-fatal warnings get printed there.
 739     AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore])
 740 }
 741 OVS_END_SHELL_HELPERS
 742
 743 AT_SETUP([OVSDB 3-server torture test - kill/restart leader])
 744 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
 745 ovsdb_torture_test 3 1 kill
 746 AT_CLEANUP
 747 AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1])
 748 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
 749 ovsdb_torture_test 3 2 kill
 750 AT_CLEANUP
 751 AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2])
 752 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
 753 ovsdb_torture_test 3 3 kill
 754 AT_CLEANUP
 755 AT_SETUP([OVSDB 5-server torture test - kill/restart leader])
 756 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 757 ovsdb_torture_test 5 1 kill
 758 AT_CLEANUP
 759 AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1])
 760 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 761 ovsdb_torture_test 5 2 kill
 762 AT_CLEANUP
 763 AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2])
 764 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 765 ovsdb_torture_test 5 3 kill
 766 AT_CLEANUP
 767 AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3])
 768 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 769 ovsdb_torture_test 5 4 kill
 770 AT_CLEANUP
 771 AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4])
 772 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 773 ovsdb_torture_test 5 5 kill
 774 AT_CLEANUP
 775
 776 AT_SETUP([OVSDB 3-server torture test - remove/re-add leader])
 777 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
 778 ovsdb_torture_test 3 1 remove
 779 AT_CLEANUP
 780 AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1])
 781 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
 782 ovsdb_torture_test 3 2 remove
 783 AT_CLEANUP
 784 AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2])
 785 AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
 786 ovsdb_torture_test 3 3 remove
 787 AT_CLEANUP
 788 AT_SETUP([OVSDB 5-server torture test - remove/re-add leader])
 789 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 790 ovsdb_torture_test 5 1 remove
 791 AT_CLEANUP
 792 AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1])
 793 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 794 ovsdb_torture_test 5 2 remove
 795 AT_CLEANUP
 796 AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2])
 797 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 798 ovsdb_torture_test 5 3 remove
 799 AT_CLEANUP
 800 AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3])
 801 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 802 ovsdb_torture_test 5 4 remove
 803 AT_CLEANUP
 804 AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4])
 805 AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
 806 ovsdb_torture_test 5 5 remove
 807 AT_CLEANUP